From bf1b6482ce3535ef7e7b3f77879def12ff454c0c Mon Sep 17 00:00:00 2001 From: makefu Date: Tue, 22 Dec 2015 19:36:19 +0100 Subject: mv makefu->krebs 3 buildbot --- shared/2configs/buildbot-standalone.nix | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 shared/2configs/buildbot-standalone.nix (limited to 'shared/2configs/buildbot-standalone.nix') diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix new file mode 100644 index 000000000..adf44cada --- /dev/null +++ b/shared/2configs/buildbot-standalone.nix @@ -0,0 +1,31 @@ +{ lib, config, pkgs, ... }: +let + pkgs-unst = import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-unstable.tar.gz) {}; +in { + nixpkgs.config.packageOverrides = pkgs: { + buildbot = pkgs-unst.buildbot; + buildbot-slave = pkgs-unst.buildbot-slave; + }; + networking.firewall.allowedTCPPorts = [ 8010 ]; + krebs.buildbot.master = { + enable = true; + irc = { + enable = true; + server = "cd.retiolum"; + channel = "retiolum"; + allowForce = true; + }; + extraConfig = '' + c['buildbotURL'] = "http://${config.krebs.build.host.name}:8010/" + ''; + }; + + krebs.buildbot.slave = { + enable = true; + masterhost = "localhost"; + username = "testslave"; + password = "krebspass"; + packages = with pkgs;[ git nix ]; + extraEnviron = { NIX_PATH="nixpkgs=${toString }"; }; + }; +} -- cgit v1.2.3 From 3adf78473d2deff0b991d7222e928fa2888529f6 Mon Sep 17 00:00:00 2001 From: makefu Date: Thu, 24 Dec 2015 00:02:59 +0100 Subject: k 3 buildbot.master: refactor see buildbot-standalone.nix in shared/2configs for the current buildbot config --- shared/2configs/buildbot-standalone.nix | 107 ++++++++++++++++++++++++++++++-- 1 file changed, 103 insertions(+), 4 deletions(-) (limited to 'shared/2configs/buildbot-standalone.nix') diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix index adf44cada..baab059c9 100644 --- a/shared/2configs/buildbot-standalone.nix +++ b/shared/2configs/buildbot-standalone.nix @@ -8,16 +8,115 @@ in { }; networking.firewall.allowedTCPPorts = [ 8010 ]; krebs.buildbot.master = { + slaves = { + testslave = "krebspass"; + testslave2 = "krebspass"; + }; + change_source.stockholm = '' + stockholm_repo = 'http://cgit.gum/stockholm' + cs.append(changes.GitPoller( + stockholm_repo, + workdir='stockholm-poller', branch='master', + project='stockholm', + pollinterval=120)) + ''; + scheduler = { + force-scheduler = '' + sched.append(schedulers.ForceScheduler( + name="force", + builderNames=["full-tests"])) + ''; + fast-tests-scheduler = '' + # test the master real quick + sched.append(schedulers.SingleBranchScheduler( + change_filter=util.ChangeFilter(branch="master"), + name="fast-master-test", + builderNames=["fast-tests"])) + ''; + full-master-scheduler = '' + # files everyone depends on or are part of the share branch + def shared_files(change): + r =re.compile("^((krebs|shared)/.*|Makefile|default.nix)") + for file in change.files: + if r.match(file): + return True + return False + + sched.append(schedulers.SingleBranchScheduler( + change_filter=util.ChangeFilter(branch="master"), + fileIsImportant=shared_files, + name="full-master-test", + builderNames=["full-tests"])) + ''; + }; + builder_pre = '' + # prepare grab_repo step for stockholm + stockholm_repo = "http://cgit.gum.retiolum/stockholm" + grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental') + + env = {"LOGNAME": "shared", "NIX_REMOTE": "daemon"} + + # prepare nix-shell + # the dependencies which are used by the test script + deps = [ "gnumake", "jq","nix","rsync", + "(import {}).pkgs.test.infest-cac-centos7" ] + # TODO: --pure , prepare ENV in nix-shell command: + # SSL_CERT_FILE,LOGNAME,NIX_REMOTE + nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ] + + # prepare addShell function + def addShell(factory,**kwargs): + factory.addStep(steps.ShellCommand(**kwargs)) + ''; + builder = { + fast-tests = '' + f = util.BuildFactory() + f.addStep(grab_repo) + addShell(f,name="centos7-eval",env=env, + command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"]) + + addShell(f,name="wolf-eval",env=env, + command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"]) + + addShell(f,name="eval-cross-check",env=env, + command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"]) + + bu.append(util.BuilderConfig(name="fast-tests", + slavenames=slavenames, + factory=f)) + ''; + slow-tests = '' + s = util.BuildFactory() + s.addStep(grab_repo) + + # slave needs 2 files: + # * cac.json + # * retiolum + for file in ["cac.json", "retiolum.rsa_key.priv"]: + s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/{}".format(file), + slavedest=file)) + + addShell(s, name="infest-cac-centos7",env=env, + sigtermTime=60, # SIGTERM 1 minute before SIGKILL + timeout=5400, # 1.5h timeout + command=nixshell + ["infest-cac-centos7"]) + + bu.append(util.BuilderConfig(name="full-tests", + slavenames=slavenames, + factory=s)) + ''; + }; enable = true; + web = { + enable = true; + }; irc = { enable = true; + nick = "shared-buildbot"; server = "cd.retiolum"; - channel = "retiolum"; + channels = [ "retiolum" ]; allowForce = true; }; - extraConfig = '' - c['buildbotURL'] = "http://${config.krebs.build.host.name}:8010/" - ''; }; krebs.buildbot.slave = { -- cgit v1.2.3 From 1ff8d0b8c5c6f631d71408eeff61778d90f9789f Mon Sep 17 00:00:00 2001 From: makefu Date: Thu, 24 Dec 2015 20:50:23 +0100 Subject: s 2 buildbot: add treestabletimer --- shared/2configs/buildbot-standalone.nix | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'shared/2configs/buildbot-standalone.nix') diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix index baab059c9..51c600329 100644 --- a/shared/2configs/buildbot-standalone.nix +++ b/shared/2configs/buildbot-standalone.nix @@ -6,11 +6,11 @@ in { buildbot = pkgs-unst.buildbot; buildbot-slave = pkgs-unst.buildbot-slave; }; - networking.firewall.allowedTCPPorts = [ 8010 ]; + networking.firewall.allowedTCPPorts = [ 8010 9989 ]; krebs.buildbot.master = { slaves = { testslave = "krebspass"; - testslave2 = "krebspass"; + omo = "krebspass"; }; change_source.stockholm = '' stockholm_repo = 'http://cgit.gum/stockholm' @@ -33,7 +33,7 @@ in { name="fast-master-test", builderNames=["fast-tests"])) ''; - full-master-scheduler = '' + test-cac-infest-master = '' # files everyone depends on or are part of the share branch def shared_files(change): r =re.compile("^((krebs|shared)/.*|Makefile|default.nix)") @@ -45,6 +45,7 @@ in { sched.append(schedulers.SingleBranchScheduler( change_filter=util.ChangeFilter(branch="master"), fileIsImportant=shared_files, + treeStableTimer=60*60, # master was stable for the last hour name="full-master-test", builderNames=["full-tests"])) ''; -- cgit v1.2.3 From d574c0ef78f7572aec88e484d3ff6256247e878c Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 30 Dec 2015 01:38:33 +0100 Subject: m 3 buildbot/master: add secrets --- shared/2configs/buildbot-standalone.nix | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'shared/2configs/buildbot-standalone.nix') diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix index 51c600329..28d1eef2e 100644 --- a/shared/2configs/buildbot-standalone.nix +++ b/shared/2configs/buildbot-standalone.nix @@ -8,6 +8,9 @@ in { }; networking.firewall.allowedTCPPorts = [ 8010 9989 ]; krebs.buildbot.master = { + secrets = [ + "cac.json" + ]; slaves = { testslave = "krebspass"; omo = "krebspass"; @@ -93,9 +96,8 @@ in { # slave needs 2 files: # * cac.json # * retiolum - for file in ["cac.json", "retiolum.rsa_key.priv"]: - s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/{}".format(file), - slavedest=file)) + s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/cac.json", slavedest="cac.json")) + s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/retiolum-ci.rsa_key.priv", slavedest="retiolum.rsa_key.priv")) addShell(s, name="infest-cac-centos7",env=env, sigtermTime=60, # SIGTERM 1 minute before SIGKILL -- cgit v1.2.3 From a8cb9d41bdefe8a5dc72ca76b3ebc5b4047ddf65 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 30 Dec 2015 02:45:47 +0100 Subject: s 1 test-all-krebs-modules: init --- shared/2configs/buildbot-standalone.nix | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) (limited to 'shared/2configs/buildbot-standalone.nix') diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix index 28d1eef2e..22e9861cc 100644 --- a/shared/2configs/buildbot-standalone.nix +++ b/shared/2configs/buildbot-standalone.nix @@ -8,12 +8,9 @@ in { }; networking.firewall.allowedTCPPorts = [ 8010 9989 ]; krebs.buildbot.master = { - secrets = [ - "cac.json" - ]; + secrets = [ "retiolum-ci.rsa_key.priv" "cac.json" ]; slaves = { testslave = "krebspass"; - omo = "krebspass"; }; change_source.stockholm = '' stockholm_repo = 'http://cgit.gum/stockholm' @@ -85,6 +82,18 @@ in { addShell(f,name="eval-cross-check",env=env, command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"]) + addShell(f,name="instaniate-test-all-modules",env=env, + command=nixshell + \ + ["touch retiolum.rsa_key.priv; \ + nix-instantiate --eval -A \ + users.shared.test-all-krebs-modules.system \ + -I stockholm=. \ + -I secrets=. '' \ + --argstr current-date lol \ + --argstr current-user-name shared \ + --argstr current-host-name lol \ + --strict --json"]) + bu.append(util.BuilderConfig(name="fast-tests", slavenames=slavenames, factory=f)) -- cgit v1.2.3 From 2bee6a0f6d71fbb1e1f02bd0df396bc38d3b6f67 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 30 Dec 2015 04:43:16 +0100 Subject: s 2 buildbot-standalone: cosmetics --- shared/2configs/buildbot-standalone.nix | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'shared/2configs/buildbot-standalone.nix') diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix index 22e9861cc..c62f8920c 100644 --- a/shared/2configs/buildbot-standalone.nix +++ b/shared/2configs/buildbot-standalone.nix @@ -73,16 +73,16 @@ in { fast-tests = '' f = util.BuildFactory() f.addStep(grab_repo) - addShell(f,name="centos7-eval",env=env, + addShell(f,name="deploy-eval-centos7",env=env, command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"]) - addShell(f,name="wolf-eval",env=env, + addShell(f,name="deploy-eval-wolf",env=env, command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"]) - addShell(f,name="eval-cross-check",env=env, + addShell(f,name="deploy-eval-cross-check",env=env, command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"]) - addShell(f,name="instaniate-test-all-modules",env=env, + addShell(f,name="instantiate-test-all-modules",env=env, command=nixshell + \ ["touch retiolum.rsa_key.priv; \ nix-instantiate --eval -A \ @@ -110,7 +110,7 @@ in { addShell(s, name="infest-cac-centos7",env=env, sigtermTime=60, # SIGTERM 1 minute before SIGKILL - timeout=5400, # 1.5h timeout + timeout=7200, # 2h command=nixshell + ["infest-cac-centos7"]) bu.append(util.BuilderConfig(name="full-tests", @@ -137,6 +137,7 @@ in { username = "testslave"; password = "krebspass"; packages = with pkgs;[ git nix ]; + # all nix commands will need a working nixpkgs installation extraEnviron = { NIX_PATH="nixpkgs=${toString }"; }; }; } -- cgit v1.2.3 From ca9e1700ef0deac0b71d4c3e2a6d1ee0a0ccbc42 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 30 Dec 2015 14:47:40 +0100 Subject: s 1 minimal-deploy: init test --- shared/2configs/buildbot-standalone.nix | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'shared/2configs/buildbot-standalone.nix') diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix index c62f8920c..6ffd7fe8a 100644 --- a/shared/2configs/buildbot-standalone.nix +++ b/shared/2configs/buildbot-standalone.nix @@ -94,6 +94,17 @@ in { --argstr current-host-name lol \ --strict --json"]) + addShell(f,name="instantiate-test-minimal-deploy",env=env, + command=nixshell + \ + ["nix-instantiate --eval -A \ + users.shared.test-minimal-deploy.system \ + -I stockholm=. \ + -I secrets=. '' \ + --argstr current-date lol \ + --argstr current-user-name shared \ + --argstr current-host-name lol \ + --strict --json"]) + bu.append(util.BuilderConfig(name="fast-tests", slavenames=slavenames, factory=f)) -- cgit v1.2.3 From 5d15b95ac200359392d9a86a68905c2162404904 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 30 Dec 2015 23:37:02 +0100 Subject: s 2 buildbot: add short tree timeout before trying a test --- shared/2configs/buildbot-standalone.nix | 1 + 1 file changed, 1 insertion(+) (limited to 'shared/2configs/buildbot-standalone.nix') diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix index 6ffd7fe8a..544b54dde 100644 --- a/shared/2configs/buildbot-standalone.nix +++ b/shared/2configs/buildbot-standalone.nix @@ -30,6 +30,7 @@ in { # test the master real quick sched.append(schedulers.SingleBranchScheduler( change_filter=util.ChangeFilter(branch="master"), + treeStableTimer=10, #only test the latest push name="fast-master-test", builderNames=["fast-tests"])) ''; -- cgit v1.2.3 From ff945f40b1c3fdb69a5016911ab48462e00cf536 Mon Sep 17 00:00:00 2001 From: makefu Date: Thu, 7 Jan 2016 08:05:05 +0100 Subject: s 2 buildbot: up cac timeout to 3h --- shared/2configs/buildbot-standalone.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'shared/2configs/buildbot-standalone.nix') diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix index 544b54dde..3275189a5 100644 --- a/shared/2configs/buildbot-standalone.nix +++ b/shared/2configs/buildbot-standalone.nix @@ -122,7 +122,7 @@ in { addShell(s, name="infest-cac-centos7",env=env, sigtermTime=60, # SIGTERM 1 minute before SIGKILL - timeout=7200, # 2h + timeout=10800, # 3h command=nixshell + ["infest-cac-centos7"]) bu.append(util.BuilderConfig(name="full-tests", -- cgit v1.2.3 [cgit] Unable to lock slot /tmp/cgit/f6000000.lock: No such file or directory (2)