ci: build steps dynamically, cleanup
This commit is contained in:
lassulus
parent
90774818a3
commit
16d306c784
|
|
@ -9,7 +9,7 @@
|
|||
<stockholm/krebs>
|
||||
<stockholm/krebs/2configs>
|
||||
|
||||
<stockholm/krebs/2configs/buildbot-all.nix>
|
||||
<stockholm/krebs/2configs/buildbot-stockholm.nix>
|
||||
<stockholm/krebs/2configs/gitlab-runner-shackspace.nix>
|
||||
<stockholm/krebs/2configs/binary-cache/nixos.nix>
|
||||
<stockholm/krebs/2configs/ircd.nix>
|
||||
|
|
@ -20,6 +20,5 @@
|
|||
|
||||
boot.isContainer = true;
|
||||
networking.useDHCP = false;
|
||||
krebs.ci.stockholmSrc = "http://cgit.prism.r/stockholm";
|
||||
environment.variables.NIX_REMOTE = "daemon";
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +0,0 @@
|
|||
with import <stockholm/lib>;
|
||||
{ lib, config, pkgs, ... }:
|
||||
{
|
||||
networking.firewall.allowedTCPPorts = [ 80 8010 9989 ];
|
||||
krebs.ci.enable = true;
|
||||
krebs.ci.treeStableTimer = 1;
|
||||
krebs.ci.hosts = filter (getAttr "ci") (attrValues config.krebs.hosts);
|
||||
}
|
||||
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
with import <stockholm/lib>;
|
||||
{ lib, config, pkgs, ... }:
|
||||
{
|
||||
imports = [
|
||||
<stockholm/krebs/2configs/repo-sync.nix>
|
||||
];
|
||||
|
||||
networking.firewall.allowedTCPPorts = [ 80 8010 9989 ];
|
||||
krebs.ci.enable = true;
|
||||
krebs.ci.treeStableTimer = 120;
|
||||
krebs.ci.hosts = [ config.krebs.build.host ];
|
||||
}
|
||||
178
krebs/2configs/buildbot-stockholm.nix
Normal file
178
krebs/2configs/buildbot-stockholm.nix
Normal file
|
|
@ -0,0 +1,178 @@
|
|||
{ config, pkgs, ... }: with import <stockholm/lib>;
|
||||
|
||||
let
|
||||
|
||||
hostname = config.networking.hostName;
|
||||
|
||||
in
|
||||
{
|
||||
networking.firewall.allowedTCPPorts = [ 80 ];
|
||||
services.nginx = {
|
||||
enable = true;
|
||||
virtualHosts.build = {
|
||||
serverAliases = [ "build.${hostname}.r" ];
|
||||
locations."/".extraConfig = ''
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
proxy_pass http://127.0.0.1:${toString config.krebs.buildbot.master.web.port};
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
krebs.buildbot.master = {
|
||||
slaves = {
|
||||
testslave = "lasspass";
|
||||
};
|
||||
change_source.stockholm = ''
|
||||
stockholm_repo = 'http://cgit.prism.r/stockholm'
|
||||
cs.append(
|
||||
changes.GitPoller(
|
||||
stockholm_repo,
|
||||
workdir='stockholm-poller', branches=True,
|
||||
project='stockholm',
|
||||
pollinterval=10
|
||||
)
|
||||
)
|
||||
'';
|
||||
scheduler = {
|
||||
auto-scheduler = ''
|
||||
sched.append(
|
||||
schedulers.SingleBranchScheduler(
|
||||
change_filter=util.ChangeFilter(branch_re=".*"),
|
||||
treeStableTimer=60,
|
||||
name="build-all-branches",
|
||||
builderNames=[
|
||||
"hosts",
|
||||
]
|
||||
)
|
||||
)
|
||||
'';
|
||||
force-scheduler = ''
|
||||
sched.append(
|
||||
schedulers.ForceScheduler(
|
||||
name="hosts",
|
||||
builderNames=[
|
||||
"hosts",
|
||||
]
|
||||
)
|
||||
)
|
||||
'';
|
||||
};
|
||||
builder_pre = ''
|
||||
# prepare grab_repo step for stockholm
|
||||
grab_repo = steps.Git(
|
||||
repourl=stockholm_repo,
|
||||
mode='full',
|
||||
)
|
||||
'';
|
||||
builder = {
|
||||
hosts = ''
|
||||
from buildbot import interfaces
|
||||
from buildbot.steps.shell import ShellCommand
|
||||
|
||||
class StepToStartMoreSteps(ShellCommand):
|
||||
def __init__(self, **kwargs):
|
||||
ShellCommand.__init__(self, **kwargs)
|
||||
|
||||
def addBuildSteps(self, steps_factories):
|
||||
for sf in steps_factories:
|
||||
step = interfaces.IBuildStepFactory(sf).buildStep()
|
||||
step.setBuild(self.build)
|
||||
step.setBuildSlave(self.build.slavebuilder.slave)
|
||||
step_status = self.build.build_status.addStepWithName(step.name)
|
||||
step.setStepStatus(step_status)
|
||||
self.build.steps.append(step)
|
||||
|
||||
def start(self):
|
||||
props = self.build.getProperties()
|
||||
hosts = json.loads(props.getProperty('hosts_json'))
|
||||
for host in hosts:
|
||||
user = hosts[host]['owner']
|
||||
|
||||
self.addBuildSteps([steps.ShellCommand(
|
||||
name=str(host),
|
||||
env={
|
||||
"NIX_PATH": "secrets=/var/src/stockholm/null:stockholm=./:/var/src",
|
||||
"NIX_REMOTE": "daemon",
|
||||
"dummy_secrets": "true",
|
||||
},
|
||||
command=[
|
||||
"nix-shell", "-I", "stockholm=.", "--run", " ".join(["test",
|
||||
"--user={}".format(user),
|
||||
"--system={}".format(host),
|
||||
"--force-populate",
|
||||
"--target=$LOGNAME@${config.krebs.build.host.name}$HOME/{}".format(user),
|
||||
])
|
||||
],
|
||||
timeout=90001,
|
||||
workdir='build', # TODO figure out why we need this?
|
||||
)])
|
||||
|
||||
ShellCommand.start(self)
|
||||
|
||||
|
||||
f = util.BuildFactory()
|
||||
f.addStep(grab_repo)
|
||||
|
||||
f.addStep(steps.SetPropertyFromCommand(
|
||||
env={
|
||||
"NIX_PATH": "secrets=/var/src/stockholm/null:stockholm=./:/var/src",
|
||||
"NIX_REMOTE": "daemon",
|
||||
},
|
||||
name="get_hosts",
|
||||
command=["nix-instantiate", "--json", "--strict", "--eval", "-E", """
|
||||
with import <nixpkgs> {};
|
||||
let
|
||||
eval-config = cfg:
|
||||
import <nixpkgs/nixos/lib/eval-config.nix> {
|
||||
modules = [
|
||||
(import cfg)
|
||||
];
|
||||
}
|
||||
;
|
||||
|
||||
system = eval-config ./krebs/1systems/hotdog/config.nix; # TODO put a better config here
|
||||
|
||||
ci-systems = lib.filterAttrs (_: v: v.ci) system.config.krebs.hosts;
|
||||
|
||||
filtered-attrs = lib.mapAttrs ( n: v: {
|
||||
owner = v.owner.name;
|
||||
}) ci-systems;
|
||||
|
||||
in filtered-attrs
|
||||
"""],
|
||||
property="hosts_json"
|
||||
))
|
||||
f.addStep(StepToStartMoreSteps(command=["echo"])) # TODO remove dummy command from here
|
||||
|
||||
bu.append(
|
||||
util.BuilderConfig(
|
||||
name="hosts",
|
||||
slavenames=slavenames,
|
||||
factory=f
|
||||
)
|
||||
)
|
||||
'';
|
||||
};
|
||||
enable = true;
|
||||
web.enable = true;
|
||||
irc = {
|
||||
enable = true;
|
||||
nick = "build|${hostname}";
|
||||
server = "irc.r";
|
||||
channels = [ "noise" "xxx" ];
|
||||
allowForce = true;
|
||||
};
|
||||
extraConfig = ''
|
||||
c['buildbotURL'] = "http://build.${hostname}.r/"
|
||||
'';
|
||||
};
|
||||
|
||||
krebs.buildbot.slave = {
|
||||
enable = true;
|
||||
masterhost = "localhost";
|
||||
username = "testslave";
|
||||
password = "lasspass";
|
||||
packages = with pkgs; [ gnumake jq nix populate ];
|
||||
};
|
||||
}
|
||||
|
|
@ -1,192 +0,0 @@
|
|||
{ config, pkgs, ... }:
|
||||
with import <stockholm/lib>;
|
||||
let
|
||||
cfg = config.krebs.ci;
|
||||
|
||||
hostname = config.networking.hostName;
|
||||
in
|
||||
{
|
||||
options.krebs.ci = {
|
||||
enable = mkEnableOption "krebs continous integration";
|
||||
stockholmSrc = mkOption {
|
||||
type = types.str;
|
||||
default = "http://cgit.${hostname}.r/stockholm";
|
||||
};
|
||||
treeStableTimer = mkOption {
|
||||
type = types.int;
|
||||
default = 10;
|
||||
description = "how long to wait until we test changes (in minutes)";
|
||||
};
|
||||
hosts = mkOption {
|
||||
type = types.listOf types.host;
|
||||
default = [];
|
||||
description = ''
|
||||
List of hosts that should be build
|
||||
'';
|
||||
};
|
||||
tests = mkOption {
|
||||
type = types.listOf types.str;
|
||||
default = [];
|
||||
description = ''
|
||||
List of tests that should be build
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
config = mkIf cfg.enable {
|
||||
services.nginx = {
|
||||
enable = true;
|
||||
virtualHosts.build = {
|
||||
serverAliases = [ "build.${hostname}.r" ];
|
||||
locations."/".extraConfig = ''
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
proxy_pass http://127.0.0.1:${toString config.krebs.buildbot.master.web.port};
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
krebs.buildbot.master = {
|
||||
slaves = {
|
||||
testslave = "lasspass";
|
||||
};
|
||||
change_source.stockholm = ''
|
||||
stockholm_repo = '${cfg.stockholmSrc}'
|
||||
cs.append(
|
||||
changes.GitPoller(
|
||||
stockholm_repo,
|
||||
workdir='stockholm-poller', branches=True,
|
||||
project='stockholm',
|
||||
pollinterval=10
|
||||
)
|
||||
)
|
||||
'';
|
||||
scheduler = {
|
||||
build-scheduler = ''
|
||||
sched.append(
|
||||
schedulers.SingleBranchScheduler(
|
||||
change_filter=util.ChangeFilter(branch_re=".*"),
|
||||
treeStableTimer=${toString cfg.treeStableTimer}*60,
|
||||
name="build-all-branches",
|
||||
builderNames=[
|
||||
${optionalString (cfg.hosts != []) ''"hosts",''}
|
||||
${optionalString (cfg.tests != []) ''"tests",''}
|
||||
]
|
||||
)
|
||||
)
|
||||
'';
|
||||
force-scheduler = ''
|
||||
sched.append(
|
||||
schedulers.ForceScheduler(
|
||||
name="force",
|
||||
builderNames=[
|
||||
${optionalString (cfg.hosts != []) ''"hosts",''}
|
||||
${optionalString (cfg.tests != []) ''"tests",''}
|
||||
]
|
||||
)
|
||||
)
|
||||
'';
|
||||
};
|
||||
builder_pre = ''
|
||||
# prepare grab_repo step for stockholm
|
||||
grab_repo = steps.Git(
|
||||
repourl=stockholm_repo,
|
||||
mode='full'
|
||||
)
|
||||
|
||||
# prepare addShell function
|
||||
def addShell(factory,**kwargs):
|
||||
factory.addStep(steps.ShellCommand(**kwargs))
|
||||
'';
|
||||
builder = {
|
||||
hosts = mkIf (cfg.hosts != []) ''
|
||||
f = util.BuildFactory()
|
||||
f.addStep(grab_repo)
|
||||
|
||||
def build_host(user, host):
|
||||
addShell(f,
|
||||
name="{}".format(host),
|
||||
env={
|
||||
"NIX_PATH": "secrets=/var/src/stockholm/null:/var/src",
|
||||
"NIX_REMOTE": "daemon",
|
||||
"dummy_secrets": "true",
|
||||
},
|
||||
command=[
|
||||
"nix-shell", "-I", "stockholm=.", "--run", " ".join(["test",
|
||||
"--user={}".format(user),
|
||||
"--system={}".format(host),
|
||||
"--force-populate",
|
||||
"--target=$LOGNAME@${config.krebs.build.host.name}$HOME/{}".format(user),
|
||||
])
|
||||
],
|
||||
timeout=90001
|
||||
)
|
||||
|
||||
${concatMapStringsSep "\n" (host:
|
||||
"build_host(\"${host.owner.name}\", \"${host.name}\")"
|
||||
) cfg.hosts}
|
||||
|
||||
bu.append(
|
||||
util.BuilderConfig(
|
||||
name="hosts",
|
||||
slavenames=slavenames,
|
||||
factory=f
|
||||
)
|
||||
)
|
||||
'';
|
||||
tests = mkIf (cfg.tests != []) ''
|
||||
f = util.BuildFactory()
|
||||
f.addStep(grab_repo)
|
||||
|
||||
def run_test(test):
|
||||
addShell(f,
|
||||
name="{}".format(test),
|
||||
env={
|
||||
"NIX_PATH": "secrets=/var/src/stockholm/null:/var/src",
|
||||
"NIX_REMOTE": "daemon",
|
||||
"dummy_secrets": "true",
|
||||
},
|
||||
command=[
|
||||
"nix-build", "-I", "stockholm=.", "krebs/0tests",
|
||||
"-A", "{}".format(test)
|
||||
],
|
||||
timeout=90001
|
||||
)
|
||||
|
||||
${concatMapStringsSep "\n" (test:
|
||||
"run_test(\"${test}\")"
|
||||
) cfg.tests}
|
||||
|
||||
bu.append(
|
||||
util.BuilderConfig(
|
||||
name="tests",
|
||||
slavenames=slavenames,
|
||||
factory=f
|
||||
)
|
||||
)
|
||||
'';
|
||||
};
|
||||
enable = true;
|
||||
web.enable = true;
|
||||
irc = {
|
||||
enable = true;
|
||||
nick = "build|${hostname}";
|
||||
server = "irc.r";
|
||||
channels = [ "xxx" "noise" ];
|
||||
allowForce = true;
|
||||
};
|
||||
extraConfig = ''
|
||||
c['buildbotURL'] = "http://build.${hostname}.r/"
|
||||
'';
|
||||
};
|
||||
|
||||
krebs.buildbot.slave = {
|
||||
enable = true;
|
||||
masterhost = "localhost";
|
||||
username = "testslave";
|
||||
password = "lasspass";
|
||||
packages = with pkgs; [ gnumake jq nix populate ];
|
||||
};
|
||||
|
||||
};
|
||||
}
|
||||
|
|
@ -13,7 +13,6 @@ let
|
|||
./buildbot/master.nix
|
||||
./buildbot/slave.nix
|
||||
./build.nix
|
||||
./ci.nix
|
||||
./current.nix
|
||||
./exim.nix
|
||||
./exim-retiolum.nix
|
||||
|
|
|
|||
Loading…
Reference in a new issue