Merge remote-tracking branch 'lass/master'

Makefile

@@ -1,126 +0,0 @@
-stockholm ?= .
-
-export HOSTNAME ?= $(shell cat /proc/sys/kernel/hostname)
-
-export STOCKHOLM_VERSION ?= $(shell \
-	version=git.$$(git describe --always --dirty); \
-	case $$version in (*-dirty) version=$$version@$$HOSTNAME; esac; \
-	date=$$(date +%y.%m); \
-	printf '%s' "$$date.$$version"; \
-)
-
-system ?= $(HOSTNAME)
-$(if $(system),,$(error unbound variable: system))
-
-nixos-config ?= $(stockholm)/$(LOGNAME)/1systems/$(system)/config.nix
-ifneq ($(words $(wildcard $(nixos-config))),1)
-$(error bad nixos-config: $(nixos-config))
-endif
-
-# target = [target_user@]target_host[:target_port][/target_path]
-ifdef target
-_target_user != echo $(target) | sed -n 's/@.*//p'
-_target_path != echo $(target) | sed -n 's/^[^/]*//p'
-_target_port != echo $(target) | sed -En 's|^.*:([^/]*)(/.*)?$$|\1|p'
-_target_host != echo $(target) | sed -En 's/^(.*@)?([^:/]*).*/\2/p'
-ifneq ($(_target_host),)
-$(if $(target_host),$(error cannot define both, target_host and host in target))
-target_host ?= $(_target_host)
-endif
-ifneq ($(_target_user),)
-$(if $(target_user),$(error cannot define both, target_user and user in target))
-target_user ?= $(_target_user)
-endif
-ifneq ($(_target_port),)
-$(if $(target_port),$(error cannot define both, target_port and port in target))
-target_port ?= $(_target_port)
-endif
-ifneq ($(_target_path),)
-$(if $(target_path),$(error cannot define both, target_path and path in target))
-target_path ?= $(_target_path)
-endif
-endif
-
-target_host ?= $(system)
-target_user ?= root
-target_port ?= 22
-target_path ?= /var/src
-
-$(if $(target_host),,$(error unbound variable: target_host))
-$(if $(target_user),,$(error unbound variable: target_user))
-$(if $(target_port),,$(error unbound variable: target_port))
-$(if $(target_path),,$(error unbound variable: target_path))
-
-whatsupnix = \
-	if type whatsupnix >/dev/null 2>&1; then \
-	  whatsupnix $(1); \
-	else \
-	  cat; \
-	fi
-
-build = \
-	nix-build \
-		-Q \
-		--no-out-link \
-		--show-trace \
-		-I nixos-config=$(nixos-config) \
-		-I stockholm=$(stockholm) \
-		-E "with import <stockholm>; $(1)" \
-		$(2) \
-	|& $(call whatsupnix)
-
-evaluate = \
-	nix-instantiate \
-		--eval \
-		--readonly-mode \
-		--show-trace \
-		-I nixos-config=$(nixos-config) \
-		-I stockholm=$(stockholm) \
-		-E "let eval = import <stockholm>; in with eval; $(1)" \
-		$(2)
-
-ifeq ($(MAKECMDGOALS),)
-$(error No goals specified)
-endif
-
-# usage: make deploy system=foo [target=bar]
-# usage: make test system=foo target=bar
-deploy test:
-ifdef target
-	nix-shell --run '$@ --system=$(system) --target=$(target)'
-else
-	nix-shell --run '$@ --system=$(system)'
-endif
-
-# usage: make populate system=foo
-populate: populate-target = \
-	$(target_user)@$(target_host):$(target_port)$(target_path)
-ifeq ($(debug),true)
-populate: populate-flags += --debug
-endif
-ifneq ($(ssh),)
-populate: populate-flags += --ssh=$(ssh)
-endif
-populate:
-	nix-shell --run 'get-source $(LOGNAME)/1systems/$(system)/source.nix' \
-	populate $(populate-target) $(populate-flags)
-
-# usage: make pkgs.populate
-pkgs:;@$(error no package selected)
-pkgs.%:;@$(call build,$@)
-
-# usage: make LOGNAME=krebs system=wolf eval.config.krebs.build.host.name
-eval eval.:;@$(call evaluate,$${expr-eval})
-eval.%:;@$(call evaluate,$@)
-
-# usage: make install system=foo [target_host=bar]
-install: ssh ?= ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
-install:
-	$(ssh) $(target_user)@$(target_host) -p $(target_port) \
-		env target_path=$(target_path) \
-			sh -s prepare < krebs/4lib/infest/prepare.sh
-	$(MAKE) populate target_path=/mnt$(target_path)
-	$(ssh) $(target_user)@$(target_host) -p $(target_port) \
-		env NIXOS_CONFIG=$(target_path)/nixos-config \
-				STOCKHOLM_VERSION="$$STOCKHOLM_VERSION" \
-			nixos-install

krebs/1systems/hotdog/config.nix

@@ -8,6 +8,8 @@
   imports = [
     <stockholm/krebs>
     <stockholm/krebs/2configs>
+
+    <stockholm/krebs/2configs/shared-buildbot.nix>
   ];
 
   krebs.build.host = config.krebs.hosts.hotdog;

krebs/1systems/puyak/config.nix

@@ -7,7 +7,6 @@
     <stockholm/krebs/2configs/secret-passwords.nix>
     <stockholm/krebs/2configs/hw/x220.nix>
 
-    <stockholm/krebs/2configs/repo-sync.nix>
     <stockholm/krebs/2configs/shared-buildbot.nix>
     <stockholm/krebs/2configs/stats/puyak-client.nix>
   ];

krebs/1systems/wolf/config.nix

@@ -12,7 +12,6 @@ in
     <stockholm/krebs/2configs/save-diskspace.nix>
 
     <stockholm/krebs/2configs/graphite.nix>
-    <stockholm/krebs/2configs/repo-sync.nix>
     <stockholm/krebs/2configs/shared-buildbot.nix>
 
     <stockholm/krebs/2configs/shack/worlddomination.nix>

krebs/2configs/repo-sync.nix

@@ -17,7 +17,7 @@ let
           verbose = false;
           channel = "#retiolum";
           server = "ni.r";
-          branches = [ "newest" ];
+          branches = [ "master" ];
         };
       });
     };
@@ -55,7 +55,7 @@ let
         };
         latest = {
           url = "${mirror}${name}";
-          ref = "heads/newest";
+          ref = "heads/master";
         };
       };
       krebs.git = defineRepo name true;

krebs/2configs/shared-buildbot.nix

@@ -1,183 +1,18 @@
 { lib, config, pkgs, ... }:
-# The buildbot config is self-contained and currently provides a way
+{
-# to test "krebs" configuration (infrastructure to be used by every krebsminister).
+  imports = [
+    <stockholm/krebs/2configs/repo-sync.nix>
+  ];
 
-# You can add your own test, test steps as required. Deploy the config on a
-# krebs host like wolf and everything should be fine.
-
-# TODO for all users schedule a build for fast tests
-let
-  hostname = config.networking.hostName;
-in {
-  # due to the fact that we actually build stuff on the box via the daemon,
-  # /nix/store should be cleaned up automatically as well
-  services.nginx = {
-    enable = true;
-    virtualHosts.build = {
-      serverAliases = [ "build.${hostname}.r" ];
-      locations."/".extraConfig = ''
-        proxy_set_header Upgrade $http_upgrade;
-        proxy_set_header Connection "upgrade";
-        proxy_pass http://127.0.0.1:${toString config.krebs.buildbot.master.web.port};
-      '';
-    };
-  };
-
-  nix.gc.automatic = true;
-  nix.gc.dates = "05:23";
   networking.firewall.allowedTCPPorts = [ 80 8010 9989 ];
-
+  krebs.ci.enable = true;
-  krebs.buildbot.master = let
+  krebs.ci.users.krebs ={
-    stockholm-mirror-url = "http://cgit.${hostname}.r/stockholm" ;
+    all = true;
-  in {
+    hosts = [
-    slaves = {
+      "test-arch"
-      testslave =  "krebspass";
+      "test-centos6"
-    };
+      "test-centos7"
-    change_source.stockholm = ''
+      "test-all-krebs-modules"
-  stockholm_repo = '${stockholm-mirror-url}'
+    ];
-  cs.append(changes.GitPoller(
-          stockholm_repo,
-          workdir='stockholm-poller', branches=True,
-          project='stockholm',
-          pollinterval=60))
-    '';
-    scheduler = {
-        force-scheduler = ''
-  sched.append(schedulers.ForceScheduler(
-                              name="force",
-                              builderNames=[
-                              #  "full-tests",
-                                "fast-tests",
-                                "build-local"
-                              ]))
-        '';
-        fast-tests-scheduler = ''
-  # test everything real quick
-  sched.append(schedulers.AnyBranchScheduler(
-                              treeStableTimer=10,
-                              name="fast-all-branches",
-                              builderNames=["fast-tests"]))
-        '';
-        test-cac-infest-master = ''
-  # files everyone depends on or are part of the share branch
-  def shared_files(change):
-    r =re.compile("^(krebs/.*|Makefile|default.nix|shell.nix)")
-    for file in change.files:
-      if r.match(file):
-        return True
-    return False
-
-  sched.append(schedulers.SingleBranchScheduler(
-                              change_filter=util.ChangeFilter(branch="master"),
-                              fileIsImportant=shared_files,
-                              treeStableTimer=60*60, # master was stable for the last hour
-                              name="full-master",
-                              builderNames=[
-                              #  "full-tests",
-                                "build-local"
-                              ]))
-        '';
-    };
-    builder_pre = ''
-  # prepare grab_repo step for stockholm
-  grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
-
-  env = {
-    "LOGNAME": "krebs",
-    "NIX_REMOTE": "daemon",
-    "dummy_secrets": "true",
-  }
-
-  # prepare nix-shell
-  # the dependencies which are used by the test script
-  deps = [ "gnumake", "jq", "nix",
-            "(import <stockholm>).pkgs.populate",
-            "(import <stockholm>).pkgs.test.infest-cac-centos7" ]
-  # TODO: --pure , prepare ENV in nix-shell command:
-  #                   SSL_CERT_FILE,LOGNAME,NIX_REMOTE
-  nixshell = ["nix-shell",
-                "-I", "stockholm=.",
-                "-I", "nixpkgs=/var/src/nixpkgs",
-                "-p" ] + deps + [ "--run" ]
-
-  # prepare addShell function
-  def addShell(factory,**kwargs):
-    factory.addStep(steps.ShellCommand(**kwargs))
-    '';
-    builder = {
-      fast-tests = ''
-        f = util.BuildFactory()
-        f.addStep(grab_repo)
-
-        for i in [ "test-minimal-deploy", "test-all-krebs-modules", "wolf", "test-centos7" ]:
-          addShell(f,name="build-{}".format(i),env=env,
-                  command=nixshell + \
-                      ["mkdir -p /tmp/testbuild/$LOGNAME && touch /tmp/testbuild/$LOGNAME/.populate; \
-                        make \
-                            test \
-                            target=$LOGNAME@${config.krebs.build.host.name}/tmp/testbuild/$LOGNAME \
-                            method=eval \
-                            system={}".format(i)])
-
-        bu.append(util.BuilderConfig(name="fast-tests",
-              slavenames=slavenames,
-              factory=f))
-
-            '';
-      # this build will try to build against local nixpkgs
-      # TODO change to do a 'local' populate and use the retrieved nixpkgs
-      build-local = ''
-  f = util.BuildFactory()
-  f.addStep(grab_repo)
-
-
-  bu.append(util.BuilderConfig(name="build-local",
-        slavenames=slavenames,
-        factory=f))
-      '';
-#      slow-tests = ''
-#  s = util.BuildFactory()
-#  s.addStep(grab_repo)
-#
-#  # slave needs 2 files:
-#  # * cac.json
-#  # * retiolum
-#  s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/cac.json", slavedest="cac.json"))
-#  s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/retiolum-ci.rsa_key.priv", slavedest="retiolum.rsa_key.priv"))
-#  addShell(s, name="infest-cac-centos7",env=env,
-#              sigtermTime=60,           # SIGTERM 1 minute before SIGKILL
-#              timeout=10800,             # 3h
-#              command=nixshell + ["infest-cac-centos7"])
-#
-#  bu.append(util.BuilderConfig(name="full-tests",
-#        slavenames=slavenames,
-#        factory=s))
-#      '';
-    };
-    enable = true;
-    web = {
-      enable = true;
-    };
-    irc = {
-      enable = true;
-      nick = "${hostname}bot";
-      server = "ni.r";
-      channels = [ "retiolum" ];
-      allowForce = true;
-    };
-    extraConfig = ''
-      c['buildbotURL'] = "http://build.${hostname}.r/"
-    '';
-  };
-
-  krebs.buildbot.slave = {
-    enable = true;
-    masterhost = "localhost";
-    username = "testslave";
-    password = "krebspass";
-    packages = with pkgs; [ gnumake jq nix populate ];
-    # all nix commands will need a working nixpkgs installation
-    extraEnviron = {
-      NIX_PATH="nixpkgs=/var/src/nixpkgs:nixos-config=./krebs/1systems/${hostname}/config.nix:stockholm=./"; };
   };
 }

krebs/3modules/buildbot/master.nix

@@ -2,22 +2,6 @@
 
 with import <stockholm/lib>;
 let
-  # https://github.com/NixOS/nixpkgs/issues/14026
-  nixpkgs-fix = import (pkgs.fetchgit {
-    url = https://github.com/nixos/nixpkgs;
-    rev = "e026b5c243ea39810826e68362718f5d703fb5d0";
-    sha256 = "11lqd480bi6xbi7xbh4krrxmbp6a6iafv1d0q3sj461al0x0has8";
-  }) {};
-
-  buildbot = nixpkgs-fix.buildbot.overrideDerivation (old: { 
-  postUnpack = "sourceRoot=\${sourceRoot}/master";
-  patches = [];
-  src = pkgs.fetchFromGitHub { 
-    owner = "krebscode";
-    repo = "buildbot-classic";
-    rev = "5b4f5f6f1";
-    sha256 = "1j3xn1gjzvsf90jvfmyln71fzlhjx642ivrqf47zfxpkacljja93"; };});
-
   buildbot-master-config = pkgs.writeText "buildbot-master.cfg" ''
     # -*- python -*-
     from buildbot.plugins import *
@@ -364,7 +348,7 @@ let
           set -efux
           if [ ! -e ${workdir} ];then
             mkdir -p ${workdir}
-            ${buildbot}/bin/buildbot create-master -r -l 10 -f ${workdir}
+            ${pkgs.buildbot-classic}/bin/buildbot create-master -r -l 10 -f ${workdir}
           fi
           # always override the master.cfg
           cp ${buildbot-master-config} ${workdir}/master.cfg
@@ -373,18 +357,18 @@ let
           ${ concatMapStringsSep "\n"
             (f: "cp ${secretsdir}/${f} ${workdir}/${f}" ) cfg.secrets }
           # sanity
-          ${buildbot}/bin/buildbot checkconfig ${workdir}
+          ${pkgs.buildbot-classic}/bin/buildbot checkconfig ${workdir}
 
           # TODO: maybe upgrade? not sure about this
           #       normally we should write buildbot.tac by our own
-          # ${buildbot}/bin/buildbot upgrade-master ${workdir}
+          # ${pkgs.buildbot-classic}/bin/buildbot upgrade-master ${workdir}
 
           chmod 700 -R ${workdir}
           chown buildbotMaster:buildbotMaster -R ${workdir}
         '';
-        ExecStart = "${buildbot}/bin/buildbot start ${workdir}";
+        ExecStart = "${pkgs.buildbot-classic}/bin/buildbot start ${workdir}";
-        ExecStop = "${buildbot}/bin/buildbot stop ${workdir}";
+        ExecStop = "${pkgs.buildbot-classic}/bin/buildbot stop ${workdir}";
-        ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}";
+        ExecReload = "${pkgs.buildbot-classic}/bin/buildbot reconfig ${workdir}";
         PrivateTmp = "true";
         User = "buildbotMaster";
         Restart = "always";

krebs/3modules/buildbot/slave.nix

@@ -2,20 +2,6 @@
 
 with import <stockholm/lib>;
 let
-  # https://github.com/NixOS/nixpkgs/issues/14026
-  nixpkgs-fix = import (pkgs.fetchgit {
-    url = https://github.com/nixos/nixpkgs;
-    rev = "e026b5c243ea39810826e68362718f5d703fb5d0";
-    sha256 = "11lqd480bi6xbi7xbh4krrxmbp6a6iafv1d0q3sj461al0x0has8";
-  }) {};
-  pkg = nixpkgs-fix.buildbot-slave.overrideDerivation (old: { 
-  postUnpack = "sourceRoot=\${sourceRoot}/slave";
-  patches = [];
-  src = pkgs.fetchFromGitHub { 
-    owner = "krebscode";
-    repo = "buildbot-classic";
-    rev = "5b4f5f6f1";
-    sha256 = "1j3xn1gjzvsf90jvfmyln71fzlhjx642ivrqf47zfxpkacljja93"; };});
   buildbot-slave-init = pkgs.writeText "buildbot-slave.tac" ''
     import os
 
@@ -166,7 +152,6 @@ let
         workdir = shell.escape cfg.workDir;
         contact = shell.escape cfg.contact;
         description = shell.escape cfg.description;
-        buildbot = pkg;
         # TODO:make this
       in {
         PermissionsStartOnly = true;
@@ -183,8 +168,8 @@ let
           chown buildbotSlave:buildbotSlave -R ${workdir}
           chmod 700 -R ${workdir}
         '';
-        ExecStart = "${buildbot}/bin/buildslave start ${workdir}";
+        ExecStart = "${pkgs.buildbot-classic-slave}/bin/buildslave start ${workdir}";
-        ExecStop = "${buildbot}/bin/buildslave stop ${workdir}";
+        ExecStop = "${pkgs.buildbot-classic-slave}/bin/buildslave stop ${workdir}";
         PrivateTmp = "true";
         User = "buildbotSlave";
         Restart = "always";

krebs/3modules/ci.nix

@@ -0,0 +1,175 @@
+{ config, pkgs, ... }:
+with import <stockholm/lib>;
+let
+  cfg = config.krebs.ci;
+
+  hostname = config.networking.hostName;
+in
+{
+  options.krebs.ci = {
+    enable = mkEnableOption "krebs continous integration";
+    users = mkOption {
+      type = with types; attrsOf (submodule {
+        options = {
+          all = mkOption {
+            type = bool;
+            default = true;
+          };
+          hosts = mkOption {
+            type = listOf str;
+            default = [];
+          };
+        };
+      });
+      example = {
+        lass.all = true;
+        krebs = {
+          all = true;
+          hosts = [
+            "test-all-krebs-modules"
+            "test-arch"
+          ];
+        };
+      };
+      default = {};
+    };
+  };
+
+  config = mkIf cfg.enable {
+    services.nginx = {
+      enable = true;
+      virtualHosts.build = {
+        serverAliases = [ "build.${hostname}.r" ];
+        locations."/".extraConfig = ''
+          proxy_set_header Upgrade $http_upgrade;
+          proxy_set_header Connection "upgrade";
+          proxy_pass http://127.0.0.1:${toString config.krebs.buildbot.master.web.port};
+        '';
+      };
+    };
+
+    nix.gc.automatic = true;
+    nix.gc.dates = "05:23";
+
+    krebs.buildbot.master = {
+      slaves = {
+        testslave = "lasspass";
+      };
+      change_source.stockholm = ''
+        stockholm_repo = 'http://cgit.${hostname}.r/stockholm'
+        cs.append(
+            changes.GitPoller(
+                stockholm_repo,
+                workdir='stockholm-poller', branches=True,
+                project='stockholm',
+                pollinterval=10
+            )
+        )
+      '';
+      scheduler = {
+        build-scheduler = ''
+          # build all hosts
+          sched.append(
+                schedulers.SingleBranchScheduler(
+                    change_filter=util.ChangeFilter(branch_re=".*"),
+                    treeStableTimer=10,
+                    name="build-all-branches",
+                    builderNames=[
+                        "build-hosts"
+                    ]
+                )
+          )
+        '';
+        force-scheduler = ''
+          sched.append(
+              schedulers.ForceScheduler(
+                    name="force",
+                    builderNames=[
+                        "build-hosts"
+                    ]
+              )
+          )
+        '';
+      };
+      builder_pre = ''
+        # prepare grab_repo step for stockholm
+        grab_repo = steps.Git(
+            repourl=stockholm_repo,
+            mode='full'
+        )
+
+        # prepare addShell function
+        def addShell(factory,**kwargs):
+          factory.addStep(steps.ShellCommand(**kwargs))
+      '';
+      builder = {
+        build-hosts = ''
+          f = util.BuildFactory()
+          f.addStep(grab_repo)
+
+          def build_host(user, host):
+              addShell(f,
+                  name="{}".format(host),
+                  env={
+                    "NIX_PATH": "secrets=/var/src/stockholm/null:/var/src",
+                    "NIX_REMOTE": "daemon",
+                    "dummy_secrets": "true",
+                  },
+                  command=[
+                    "nix-shell", "--run",
+                    "test --user={} --system={} --target=$LOGNAME@${config.krebs.build.host.name}$HOME/{}".format(user, host, user)
+                  ]
+              )
+
+          ${let
+            user-hosts = mapAttrs (user: a: let
+              managed-hosts = attrNames (filterAttrs (_: h: (h.owner.name == user) && h.managed) config.krebs.hosts);
+              defined-hosts = a.hosts;
+            in
+              defined-hosts ++ (optionals a.all managed-hosts)
+            ) cfg.users;
+
+          in
+            concatStringsSep "\n" (
+              (mapAttrsToList (user: hosts:
+                concatMapStringsSep "\n" (host:
+                  "build_host(\"${user}\", \"${host}\")"
+                ) hosts
+              ) user-hosts)
+            )
+          }
+
+          bu.append(
+              util.BuilderConfig(
+                  name="build-hosts",
+                  slavenames=slavenames,
+                  factory=f
+              )
+          )
+
+        '';
+      };
+      enable = true;
+      web.enable = true;
+      irc = {
+        enable = true;
+        nick = "build|${hostname}";
+        server = "ni.r";
+        channels = [ "retiolum" "noise" ];
+        allowForce = true;
+      };
+      extraConfig = ''
+        c['buildbotURL'] = "http://build.${hostname}.r/"
+      '';
+    };
+
+    krebs.buildbot.slave = {
+      enable = true;
+      masterhost = "localhost";
+      username = "testslave";
+      password = "lasspass";
+      packages = with pkgs; [ gnumake jq nix populate ];
+    };
+
+  };
+}

krebs/3modules/default.nix

@@ -12,6 +12,7 @@ let
       ./buildbot/master.nix
       ./buildbot/slave.nix
       ./build.nix
+      ./ci.nix
       ./current.nix
       ./exim.nix
       ./exim-retiolum.nix

krebs/3modules/krebs/default.nix

@@ -32,12 +32,15 @@ in {
   hosts = {
     hotdog = {
       owner = config.krebs.users.krebs;
+      managed = true;
       nets = {
         retiolum = {
           ip4.addr = "10.243.77.3";
           ip6.addr = "42:0:0:0:0:0:77:3";
           aliases = [
             "hotdog.r"
+            "build.hotdog.r"
+            "cgit.hotdog.r"
           ];
           tinc.pubkey = ''
             -----BEGIN RSA PUBLIC KEY-----
@@ -56,6 +59,7 @@ in {
     };
     puyak = {
       owner = config.krebs.users.krebs;
+      managed = true;
       nets = {
         retiolum = {
           ip4.addr = "10.243.77.2";
@@ -82,6 +86,7 @@ in {
     };
     wolf = {
       owner = config.krebs.users.krebs;
+      managed = true;
       nets = {
         shack = {
           ip4.addr =  "10.42.2.150" ;
@@ -120,6 +125,11 @@ in {
     krebs = {
       pubkey = "lol"; # TODO krebs.users.krebs.pubkey should be unnecessary
     };
+    hotdog-repo-sync = {
+      name = "hotdog-repo-sync";
+      mail = "spam@krebsco.de";
+      pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILzTvaR3QqOD3oEEGHQzg/sRnNbKJnZYcV9htDvXmu53";
+    };
     puyak-repo-sync = {
       name = "puyak-repo-sync";
       mail = "spam@krebsco.de";

krebs/3modules/lass/default.nix

@@ -255,7 +255,7 @@ with import <stockholm/lib>;
       nets = rec {
         retiolum = {
           ip4.addr = "10.243.133.114";
-          ip6.addr = "42:0000:0000:0000:0000:0000:d15f:1214";
+          ip6.addr = "42:0:0:0:0:0:1ca0:1205";
           aliases = [
             "icarus.r"
             "cgit.icarus.r"
@@ -276,6 +276,32 @@ with import <stockholm/lib>;
       ssh.privkey.path = <secrets/ssh.id_ed25519>;
       ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOPgQIMYiyD4/Co+nlOQWEzCKssemOEXAY/lbIZZaMhj";
     };
+    daedalus = {
+      cores = 2;
+      nets = rec {
+        retiolum = {
+          ip4.addr = "10.243.133.115";
+          ip6.addr = "42:0:0:0:0:0:daed:a105";
+          aliases = [
+            "daedalus.r"
+            "cgit.daedalus.r"
+          ];
+          tinc.pubkey = ''
+            -----BEGIN RSA PUBLIC KEY-----
+            MIIBCgKCAQEAzlIJfYIoQGXishIQGFNOcaVoeelqy7a731FJ+VfrqeR8WURQ6D+8
+            5hz7go+l3Z7IhTc/HbpGFJ5QJJNFSuSpLfZVyi+cKAUVheTivIniHFIRw37JbJ4+
+            qWTlVe3uvOiZ0cA9S6LrbzqAUTLbH0JlWj36mvGIPICDr9YSEkIUKbenxjJlIpX8
+            ECEBm8RU1aq3PUo/cVjmpqircynVJBbRCXZiHoxyLXNmh23d0fCPCabEYWhJhgaR
+            arkYRls5A14HGMI52F3ehnhED3k0mU8/lb4OzYgk34FjuZGmyRWIfrEKnqL4Uu2w
+            3pmEvswG1WYG/3+YE80C5OpCE4BUKAzYSwIDAQAB
+            -----END RSA PUBLIC KEY-----
+          '';
+        };
+      };
+      secure = true;
+      ssh.privkey.path = <secrets/ssh.id_ed25519>;
+      ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAq5Ovdcsljr5dOl7+2sQNKpGpdX0SlOIuCZKEiWEp8g";
+    };
     iso = {
       cores = 1;
       managed = false;

krebs/4lib/infest/prepare.sh

@@ -1,8 +1,8 @@
 #! /bin/sh
 set -efu
 
-nix_url=https://nixos.org/releases/nix/nix-1.10/nix-1.10-x86_64-linux.tar.bz2
+nix_url=https://nixos.org/releases/nix/nix-1.11.13/nix-1.11.13-x86_64-linux.tar.bz2
-nix_sha256=504f7a3a85fceffb8766ae5e1005de9e02e489742f5a63cc3e7552120b138bf4
+nix_sha256=c11411d52d8ad1ce3a68410015487282fd4651d3abefbbb13fa1f7803a2f60de
 
 prepare() {(
   if test -e /etc/os-release; then
@@ -14,10 +14,6 @@ prepare() {(
         ;;
       centos)
         case $VERSION_ID in
-          6)
-            prepare_centos "$@"
-            exit
-            ;;
           7)
             prepare_centos "$@"
             exit
@@ -51,13 +47,6 @@ prepare() {(
         esac
         ;;
     esac
-  elif test -e /etc/centos-release; then
-    case $(cat /etc/centos-release) in
-      'CentOS release 6.5 (Final)')
-        prepare_centos "$@"
-        exit
-        ;;
-    esac
   fi
   echo "$0 prepare: unknown OS" >&2
   exit -1
@@ -217,7 +206,7 @@ prepare_common() {(
   mkdir -p bin
   rm -f bin/nixos-install
   cp "$(type -p nixos-install)" bin/nixos-install
-  sed -i "s@^NIX_PATH=\"[^\"]*\"@NIX_PATH=$target_path@" bin/nixos-install
+  sed -i "s@NIX_PATH=\"[^\"]*\"@NIX_PATH=$target_path@" bin/nixos-install
 
   if ! grep -q '^PATH.*#krebs' .bashrc; then
     echo '. /root/.nix-profile/etc/profile.d/nix.sh' >> .bashrc

krebs/5pkgs/simple/buildbot-classic-slave/default.nix

@@ -0,0 +1,21 @@
+{ coreutils, fetchgit, fetchFromGitHub, buildbot-classic, python2Packages, ... }:
+
+python2Packages.buildPythonApplication {
+  name = "buildbot-classic-slave-0.8.12";
+  namePrefix = "";
+
+  src = buildbot-classic.src;
+  postUnpack = "sourceRoot=\${sourceRoot}/slave";
+
+  patchPhase = ''
+    substituteInPlace buildslave/scripts/logwatcher.py --replace /usr/bin/tail ${coreutils}/bin/tail
+  '';
+
+  propagatedBuildInputs = [ python2Packages.twisted ];
+  doCheck = false;
+
+  postInstall = ''
+    mkdir -p "$out/share/man/man1"
+    cp docs/buildslave.1 "$out/share/man/man1"
+  '';
+}

krebs/5pkgs/simple/buildbot-classic/default.nix

@@ -0,0 +1,47 @@
+{ fetchgit, fetchFromGitHub, python2Packages, ... }:
+let
+  # https://github.com/NixOS/nixpkgs/issues/14026
+  nixpkgs-fix = import (fetchgit {
+    url = https://github.com/nixos/nixpkgs;
+    rev = "e026b5c243ea39810826e68362718f5d703fb5d0";
+    sha256 = "11lqd480bi6xbi7xbh4krrxmbp6a6iafv1d0q3sj461al0x0has8";
+  }) {};
+
+in nixpkgs-fix.buildPythonApplication {
+  name = "buildbot-classic-0.8.12";
+  namePrefix = "";
+  patches = [];
+
+  src = fetchFromGitHub {
+    owner = "krebscode";
+    repo = "buildbot-classic";
+    rev = "5b4f5f6f1";
+    sha256 = "1j3xn1gjzvsf90jvfmyln71fzlhjx642ivrqf47zfxpkacljja93";
+  };
+  postUnpack = "sourceRoot=\${sourceRoot}/master";
+
+  patchPhase =
+    # The code insists on /usr/bin/tail, /usr/bin/make, etc.
+    '' echo "patching erroneous absolute path references..."
+       for i in $(find -name \*.py)
+       do
+         sed -i "$i" \
+             -e "s|/usr/bin/python|$(type -P python)|g ; s|/usr/bin/||g"
+       done
+
+      sed -i 's/==/>=/' setup.py
+    '';
+
+  propagatedBuildInputs = [
+    python2Packages.jinja2
+    python2Packages.twisted
+    nixpkgs-fix.pythonPackages.dateutil_1_5
+    nixpkgs-fix.pythonPackages.sqlalchemy_migrate_0_7
+  ];
+  doCheck = false;
+  postInstall = ''
+    mkdir -p "$out/share/man/man1"
+    cp docs/buildbot.1 "$out/share/man/man1"
+  '';
+}
+

krebs/5pkgs/simple/populate/default.nix

@@ -13,12 +13,12 @@ in
 
 stdenv.mkDerivation rec {
   name = "populate";
-  version = "1.2.0";
+  version = "1.2.1";
 
   src = fetchgit {
     url = http://cgit.ni.krebsco.de/populate;
     rev = "refs/tags/v${version}";
-    sha256 = "0q3110hkkxn9bc3a63xbx1hyd1fpzz4wrck4lng3j5a9i1y1jm07";
+    sha256 = "13viizpmjkqxc3d9mk2bzspgnal07bma5m9lc90mcdlk36na3pkq";
   };
 
   phases = [

krebs/5pkgs/writers.nix

@@ -144,9 +144,14 @@ with import <stockholm/lib>;
 
       env = filevars // { passAsFile = attrNames filevars; };
     in
+      # Use a subshell because <nixpkgs/stdenv/generic/setup.sh>'s genericBuild
+      # sources (or evaluates) the buildCommand and we don't want to modify its
+      # shell.  In particular, exitHandler breaks in multiple ways with set -u.
       pkgs.runCommand name env /* sh */ ''
-        set -efu
+        (
-        ${concatMapStringsSep "\n" (getAttr "install") files}
+          set -efu
+          ${concatMapStringsSep "\n" (getAttr "install") files}
+        )
       '';
 
     writeHaskell =

lass/1systems/daedalus/config.nix

@@ -0,0 +1,35 @@
+{ config, pkgs, ... }:
+
+{
+  imports = [
+    <stockholm/lass>
+    <stockholm/lass/2configs/hw/x220.nix>
+    <stockholm/lass/2configs/boot/stock-x220.nix>
+
+    <stockholm/lass/2configs/mouse.nix>
+    <stockholm/lass/2configs/retiolum.nix>
+    <stockholm/lass/2configs/git.nix>
+    <stockholm/lass/2configs/exim-retiolum.nix>
+    <stockholm/lass/2configs/baseX.nix>
+    <stockholm/lass/2configs/browsers.nix>
+    <stockholm/lass/2configs/programs.nix>
+    <stockholm/lass/2configs/fetchWallpaper.nix>
+    <stockholm/lass/2configs/backups.nix>
+    <stockholm/lass/2configs/games.nix>
+  ];
+
+  krebs.build.host = config.krebs.hosts.daedalus;
+
+  fileSystems = {
+    "/bku" = {
+      device = "/dev/mapper/pool-bku";
+      fsType = "btrfs";
+      options = ["defaults" "noatime" "ssd" "compress=lzo"];
+    };
+  };
+
+  services.udev.extraRules = ''
+    SUBSYSTEM=="net", ATTR{address}=="00:24:d7:f0:e8:c8", NAME="wl0"
+    SUBSYSTEM=="net", ATTR{address}=="f0:de:f1:8f:8a:78", NAME="et0"
+  '';
+}

lass/1systems/daedalus/source.nix

@@ -0,0 +1,4 @@
+import <stockholm/lass/source.nix> {
+  name = "daedalus";
+  secure = true;
+}

lass/2configs/boot/stock-x220.nix

@@ -0,0 +1,8 @@
+{ ... }:
+
+{
+  boot = {
+    loader.systemd-boot.enable = true;
+    loader.efi.canTouchEfiVariables = true;
+  };
+}

lass/2configs/buildbot-standalone.nix

@@ -9,9 +9,11 @@ let
     ControlPersist 4h
   '';
 
+  hostname = config.networking.hostName;
+
 in {
   config.services.nginx.virtualHosts.build = {
-    serverAliases = [ "build.prism.r" ];
+    serverAliases = [ "build.${hostname}.r" ];
     locations."/".extraConfig = ''
       proxy_set_header Upgrade $http_upgrade;
       proxy_set_header Connection "upgrade";
@@ -20,7 +22,7 @@ in {
   };
 
   config.krebs.buildbot.master = let
-    stockholm-mirror-url = http://cgit.prism.r/stockholm ;
+    stockholm-mirror-url = "http://cgit.${hostname}.r/stockholm";
   in {
     slaves = {
       testslave = "lasspass";
@@ -109,13 +111,13 @@ in {
     web.enable = true;
     irc = {
       enable = true;
-      nick = "buildbot-lass";
+      nick = "build|${hostname}";
       server = "ni.r";
       channels = [ "retiolum" "noise" ];
       allowForce = true;
     };
     extraConfig = ''
-      c['buildbotURL'] = "http://build.prism.r/"
+      c['buildbotURL'] = "http://build.${hostname}.r/"
     '';
   };
 

lass/2configs/exim-smarthost.nix

@@ -34,6 +34,7 @@ with import <stockholm/lib>;
       { from = "irgendwas@lassul.us"; to = lass.mail; }
       { from = "polo@lassul.us"; to = lass.mail; }
       { from = "shack@lassul.us"; to = lass.mail; }
+      { from = "nix@lassul.us"; to = lass.mail; }
     ];
     system-aliases = [
       { from = "mailer-daemon"; to = "postmaster"; }

lib/types.nix

@@ -36,6 +36,7 @@ rec {
           If true, then the host's configuration is defined in stockholm.
         '';
         type = bool;
+        default = false;
       };
 
       owner = mkOption {

shell.nix

@@ -6,7 +6,8 @@ let
   # high level commands
   #
 
-  # usage: deploy [--user=USER] --system=SYSTEM [--target=TARGET]
+  # usage: deploy [--force-populate] [--user=USER]
+  #               --system=SYSTEM [--target=TARGET]
   cmds.deploy = pkgs.writeDash "cmds.deploy" ''
     set -efu
 
@@ -15,11 +16,55 @@ let
     \test -n "''${target-}" || target=$system
     \test -n "''${user-}" || user=$LOGNAME
     . ${init.env}
+    . ${init.proxy}
 
     exec ${utils.deploy}
   '';
 
-  # usage: test [--user=USER] --system=SYSTEM --target=TARGET
+  # usage: install [--force-populate] [--user=USER]
+  #                --system=SYSTEM --target=TARGET
+  cmds.install = pkgs.writeBash "cmds.install" ''
+    set -efu
+
+    command=install
+    . ${init.args}
+    \test -n "''${user-}" || user=$LOGNAME
+    . ${init.env}
+
+    if \test "''${using_proxy-}" != true; then
+      ${pkgs.openssh}/bin/ssh \
+          -o StrictHostKeyChecking=no \
+          -o UserKnownHostsFile=/dev/null \
+          "$target_user@$target_host" -p "$target_port" \
+          env target_path=$(quote "$target_path") \
+              sh -s prepare < ${./krebs/4lib/infest/prepare.sh}
+              # TODO inline prepare.sh?
+    fi
+
+    . ${init.proxy}
+
+    # Reset PATH because we need access to nixos-install.
+    # TODO provide nixos-install instead of relying on prepare.sh
+    export PATH="$OLD_PATH"
+
+    # these variables get defined by nix-shell (i.e. nix-build) from
+    # XDG_RUNTIME_DIR and reference the wrong directory (/run/user/0),
+    # which only exists on / and not at /mnt.
+    export NIX_BUILD_TOP=/tmp
+    export TEMPDIR=/tmp
+    export TEMP=/tmp
+    export TMPDIR=/tmp
+    export TMP=/tmp
+    export XDG_RUNTIME_DIR=/tmp
+
+    export NIXOS_CONFIG="$target_path/nixos-config"
+
+    cd
+    exec nixos-install
+  '';
+
+  # usage: test [--force-populate] [--user=USER]
+  #             --system=SYSTEM --target=TARGET
   cmds.test = pkgs.writeDash "cmds.test" /* sh */ ''
     set -efu
 
@@ -29,6 +74,7 @@ let
     . ${init.args}
     \test -n "''${user-}" || user=$LOGNAME
     . ${init.env}
+    . ${init.proxy}
 
     exec ${utils.build} config.system.build.toplevel
   '';
@@ -99,11 +145,13 @@ let
   init.args = pkgs.writeText "init.args" /* sh */ ''
     args=$(${pkgs.utillinux}/bin/getopt -n "$command" -s sh \
         -o s:t:u: \
-        -l system:,target:,user: \
+        -l force-populate,system:,target:,user: \
         -- "$@")
     if \test $? != 0; then exit 1; fi
     eval set -- "$args"
+    force_populate=false;
     while :; do case $1 in
+      --force-populate) force_populate=true; shift;;
       -s|--system) system=$2; shift 2;;
       -t|--target) target=$2; shift 2;;
       -u|--user) user=$2; shift 2;;
@@ -114,9 +162,6 @@ let
   '';
 
   init.env = pkgs.writeText "init.env" /* sh */ ''
-    source=''${source-$user/1systems/$system/source.nix}
-
-    export source
     export system
     export target
     export user
@@ -129,38 +174,35 @@ let
     export target_port="$(echo $target_object | ${pkgs.jq}/bin/jq -r .port)"
     export target_path="$(echo $target_object | ${pkgs.jq}/bin/jq -r .path)"
     export target_local="$(echo $target_object | ${pkgs.jq}/bin/jq -r .local)"
+  '';
 
+  init.proxy = pkgs.writeText "init.proxy" /* sh */ ''
     if \test "''${using_proxy-}" != true; then
-      ${init.env.populate}
+
+      source_file=$user/1systems/$system/source.nix
+      source=$(get-source "$source_file")
+      qualified_target=$target_user@$target_host:$target_port$target_path
+      if test "$force_populate" = true; then
+        echo "$source" | populate --force "$qualified_target"
+      else
+        echo "$source" | populate "$qualified_target"
+      fi
+
       if \test "$target_local" != true; then
-        exec ${init.env.proxy} "$command" "$@"
+        exec ${pkgs.openssh}/bin/ssh \
+            "$target_user@$target_host" -p "$target_port" \
+            cd "$target_path/stockholm" \; \
+            NIX_PATH=$(quote "$target_path") \
+            STOCKHOLM_VERSION=$(quote "$STOCKHOLM_VERSION") \
+            nix-shell --run "$(quote "
+              system=$(quote "$system") \
+              target=$(quote "$target") \
+              using_proxy=true \
+              $(quote "$command" "$@")
+            ")"
       fi
     fi
-  '' // {
+  '';
-    populate = pkgs.writeDash "init.env.populate" ''
-      set -efu
-      _source=$(get-source "$source")
-      echo $_source |
-      ${pkgs.populate}/bin/populate \
-          "$target_user@$target_host:$target_port$target_path" \
-        >&2
-      unset _source
-    '';
-    proxy = pkgs.writeDash "init.env.proxy" ''
-      set -efu
-      exec ${pkgs.openssh}/bin/ssh \
-        "$target_user@$target_host" -p "$target_port" \
-        cd "$target_path/stockholm" \; \
-        NIX_PATH=$(quote "$target_path") \
-        STOCKHOLM_VERSION=$(quote "$STOCKHOLM_VERSION") \
-        nix-shell --run "$(quote "
-          system=$(quote "$system") \
-          target=$(quote "$target") \
-          using_proxy=true \
-          $(quote "$@")
-        ")"
-    '';
-  };
 
   utils.build = pkgs.writeDash "utils.build" ''
     set -efu
@@ -201,9 +243,13 @@ let
 in pkgs.stdenv.mkDerivation {
   name = "stockholm";
   shellHook = /* sh */ ''
+    export OLD_PATH="$PATH"
     export NIX_PATH=stockholm=$PWD:nixpkgs=${toString <nixpkgs>}
-    export NIX_REMOTE=daemon
+    if test -e /nix/var/nix/daemon-socket/socket; then
+      export NIX_REMOTE=daemon
+    fi
     export PATH=${lib.makeBinPath [
+      pkgs.populate
       shell.cmdspkg
     ]}