Merge remote-tracking branch 'gum/master'

56 files changed, 1365 insertions, 599 deletions

diff --git a/Makefile b/Makefile
index aefd17147..5b898c54c 100644
--- a/Makefile
+++ b/Makefile
@@ -35,7 +35,7 @@ ifeq ($(filter),json)
 else
 	filter() { cat; }
 endif
-	nix-instantiate \
+	result=$$(nix-instantiate \
 		$${extraArgs-} \
 		--eval \
 		-A "$$get" \
@@ -45,8 +45,9 @@ endif
 		--argstr current-host-name "$$HOSTNAME" \
 		--argstr current-user-name "$$LOGNAME" \
 		$${system+--argstr system "$$system"} \
-		$${target+--argstr target "$$target"} \
-		| filter
+		$${target+--argstr target "$$target"})
+	echo "$$result" | filter
+
 else
 $(error unbound variable: system[s])
 endif
diff --git a/krebs/3modules/Reaktor.nix b/krebs/3modules/Reaktor.nix
index 0fca52203..92400139c 100644
--- a/krebs/3modules/Reaktor.nix
+++ b/krebs/3modules/Reaktor.nix
@@ -9,6 +9,7 @@ let
       ${cfg.overrideConfig}
       '' else ""}
       ## Extra Config
+      ${concatStringsSep "\n" (map (plug: plug.config) cfg.plugins)}
       ${cfg.extraConfig}
     '';
   cfg = config.krebs.Reaktor;
@@ -35,7 +36,6 @@ let
       '';
     };
 
-
     overrideConfig = mkOption {
       default = null;
       type = types.nullOr types.str;
@@ -44,6 +44,9 @@ let
         Reaktor default cfg can be retrieved via `reaktor get-config`
       '';
     };
+    plugins = mkOption {
+      default = [pkgs.ReaktorPlugins.nixos-version];
+    };
     extraConfig = mkOption {
       default = "";
       type = types.string;
@@ -51,6 +54,14 @@ let
         configuration appended to the default or overridden configuration
       '';
     };
+
+    workdir = mkOption {
+      default = "/var/lib/Reaktor";
+      type = types.str;
+      description = ''
+        Reaktor working directory
+      '';
+    };
     extraEnviron = mkOption {
       default = {};
       type = types.attrsOf types.str;
@@ -59,12 +70,17 @@ let
           REAKTOR_HOST
           REAKTOR_PORT
           REAKTOR_STATEDIR
-          REAKTOR_CHANNELS
 
           debug and nickname can be set separately via the Reaktor api
       '';
     };
-
+    channels = mkOption {
+      default = [ "#krebs" ];
+      type = types.listOf types.str;
+      description = ''
+        Channels the Reaktor should connect to at startup.
+      '';
+    };
     debug = mkOption {
       default = false;
       description = ''
@@ -79,7 +95,7 @@ let
       name = "Reaktor";
       uid = genid name;
       description = "Reaktor user";
-      home = "/var/lib/Reaktor";
+      home = cfg.workdir;
       createHome = true;
     };
 
@@ -101,6 +117,9 @@ let
         GIT_SSL_CAINFO = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
         REAKTOR_NICKNAME = cfg.nickname;
         REAKTOR_DEBUG = (if cfg.debug  then "True" else "False");
+        REAKTOR_CHANNELS = lib.concatStringsSep "," cfg.channels;
+        state_dir = cfg.workdir;
+
         } // cfg.extraEnviron;
       serviceConfig= {
         ExecStartPre = pkgs.writeScript "Reaktor-init" ''
diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
new file mode 100644
index 000000000..74385a433
--- /dev/null
+++ b/krebs/3modules/buildbot/master.nix
@@ -0,0 +1,385 @@
+{ config, pkgs, lib, ... }:
+
+with lib;
+let
+  buildbot = pkgs.buildbot;
+  buildbot-master-config = pkgs.writeText "buildbot-master.cfg" ''
+    # -*- python -*-
+    from buildbot.plugins import *
+    import re
+    import json
+    c = BuildmasterConfig = {}
+
+    c['slaves'] = []
+    slaves = json.loads('${builtins.toJSON cfg.slaves}')
+    slavenames = [ s for s in slaves ]
+    for k,v in slaves.items():
+      c['slaves'].append(buildslave.BuildSlave(k, v))
+
+    # TODO: configure protocols?
+    c['protocols'] = {'pb': {'port': 9989}}
+
+    ####### Build Inputs
+    c['change_source'] = cs = []
+
+    ${ concatStringsSep "\n"
+    (mapAttrsToList (n: v: ''
+        #### Change_Source: Begin of ${n}
+        ${v}
+        #### Change_Source: End of ${n}
+      '') cfg.change_source )}
+
+    ####### Build Scheduler
+    c['schedulers'] = sched = []
+
+    ${ concatStringsSep "\n"
+    (mapAttrsToList (n: v: ''
+        #### Schedulers: Begin of ${n}
+        ${v}
+        #### Schedulers: End of ${n}
+      '') cfg.scheduler )}
+
+    ###### Builder
+    c['builders'] = bu = []
+    
+    # Builder Pre: Begin
+    ${cfg.builder_pre}
+    # Builder Pre: End
+
+    ${ concatStringsSep "\n"
+    (mapAttrsToList (n: v: ''
+        #### Builder: Begin of ${n}
+        ${v}
+        #### Builder: End of ${n}
+      '') cfg.builder )}
+
+
+    ####### Status
+    c['status'] = st = []
+
+    # If you want to configure this url, override with extraConfig
+    c['buildbotURL'] = "http://${config.networking.hostName}:${toString cfg.web.port}/"
+
+    ${optionalString (cfg.web.enable) ''
+      from buildbot.status import html
+      from buildbot.status.web import authz, auth
+      authz_cfg=authz.Authz(
+          auth=auth.BasicAuth([ ("${cfg.web.username}","${cfg.web.password}") ]),
+          # TODO: configure harder
+          gracefulShutdown = False,
+          forceBuild = 'auth',
+          forceAllBuilds = 'auth',
+          pingBuilder = False,
+          stopBuild = 'auth',
+          stopAllBuilds = 'auth',
+          cancelPendingBuild = 'auth'
+      )
+      # TODO: configure krebs.nginx
+      st.append(html.WebStatus(http_port=${toString cfg.web.port}, authz=authz_cfg))
+      ''}
+
+    ${optionalString (cfg.irc.enable) ''
+      from buildbot.status import words
+      irc = words.IRC("${cfg.irc.server}", "${cfg.irc.nick}",
+                      channels=${builtins.toJSON cfg.irc.channels},
+                      notify_events={
+                        'success': 1,
+                        'failure': 1,
+                        'exception': 1,
+                        'successToFailure': 1,
+                        'failureToSuccess': 1,
+                      }${optionalString cfg.irc.allowForce ",allowForce=True"})
+      c['status'].append(irc)
+      ''}
+
+    ${ concatStringsSep "\n"
+    (mapAttrsToList (n: v: ''
+        #### Status: Begin of ${n}
+        ${v}
+        #### Status: End of ${n}
+      '') cfg.status )}
+
+    ####### PROJECT IDENTITY
+    c['title'] = "${cfg.title}"
+    c['titleURL'] = "http://krebsco.de"
+
+
+    ####### DB URL
+    # TODO: configure
+    c['db'] = {
+        'db_url' : "sqlite:///state.sqlite",
+    }
+    ${cfg.extraConfig}
+    '';
+
+  cfg = config.krebs.buildbot.master;
+
+  api = {
+    enable = mkEnableOption "Buildbot Master";
+    title = mkOption {
+      default = "Buildbot CI";
+      type = types.str;
+      description = ''
+        Title of the Buildbot Installation
+      '';
+    };
+    workDir = mkOption {
+      default = "/var/lib/buildbot/master";
+      type = types.str;
+      description = ''
+        Path to build bot master directory.
+        Will be created on startup.
+      '';
+    };
+
+    secrets = mkOption {
+      default = [];
+      type = types.listOf types.str;
+      example = [ "cac.json" ];
+      description = ''
+        List of all the secrets in <secrets> which should be copied into the
+        buildbot master directory.
+      '';
+    };
+
+    slaves = mkOption {
+      default = {};
+      type = types.attrsOf types.str;
+      description = ''
+        Attrset of slavenames with their passwords
+        slavename = slavepassword
+      '';
+    };
+
+    change_source = mkOption {
+      default = {};
+      type = types.attrsOf types.str;
+      example = {
+        stockholm = ''
+          cs.append(changes.GitPoller(
+                  'http://cgit.gum/stockholm',
+                  workdir='stockholm-poller', branch='master',
+                  project='stockholm',
+                  pollinterval=120))
+        '';
+      };
+      description = ''
+        Attrset of all the change_sources which should be configured.
+        It will be directly included into the master configuration.
+
+        At the end an change object should be appended to <literal>cs</literal>
+      '';
+    };
+
+    scheduler = mkOption {
+      default = {};
+      type = types.attrsOf types.str;
+      example = {
+        force-scheduler = ''
+          sched.append(schedulers.ForceScheduler(
+                                      name="force",
+                                      builderNames=["full-tests"]))
+        '';
+      };
+      description = ''
+        Attrset of all the schedulers which should be configured.
+        It will be directly included into the master configuration.
+
+        At the end an change object should be appended to <literal>sched</literal>
+      '';
+    };
+
+    builder_pre = mkOption {
+      default = "";
+      type = types.lines;
+      example = ''
+        grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
+      '';
+      description = ''
+        some code before the builders are being assembled.
+        can be used to define functions used by multiple builders
+      '';
+    };
+
+    builder = mkOption {
+      default = {};
+      type = types.attrsOf types.str;
+      example = {
+        fast-test = ''
+        '';
+      };
+      description = ''
+        Attrset of all the builder which should be configured.
+        It will be directly included into the master configuration.
+
+        At the end an change object should be appended to <literal>bu</literal>
+      '';
+    };
+
+    status = mkOption {
+      default = {};
+      type = types.attrsOf types.str;
+      description = ''
+        Attrset of all the extra status which should be configured.
+        It will be directly included into the master configuration.
+
+        At the end an change object should be appended to <literal>st</literal>
+
+        Right now IRC and Web status can be configured by setting
+        <literal>buildbot.master.irc.enable</literal> and
+        <literal>buildbot.master.web.enable</literal>
+      '';
+    };
+
+    # Configurable Stati
+    web = mkOption {
+      default = {};
+      type = types.submodule ({ config2, ... }: {
+        options = {
+          enable = mkEnableOption "Buildbot Master Web Status";
+          username = mkOption {
+            default = "krebs";
+            type = types.str;
+            description = ''
+              username for web authentication
+            '';
+          };
+          hostname = mkOption {
+            default = config.networking.hostName;
+            type = types.str;
+            description = ''
+              web interface Hostname
+            '';
+          };
+          password = mkOption {
+            default = "bob";
+            type = types.str;
+            description = ''
+              password for web authentication
+            '';
+          };
+          port = mkOption {
+            default = 8010;
+            type = types.int;
+            description = ''
+              port for buildbot web status
+            '';
+          };
+        };
+      });
+    };
+
+    irc = mkOption {
+      default = {};
+      type = types.submodule ({ config, ... }: {
+        options = {
+          enable = mkEnableOption "Buildbot Master IRC Status";
+          channels = mkOption {
+            default = [ "nix-buildbot-meetup" ];
+            type = with types; listOf str;
+            description = ''
+              irc channels the bot should connect to
+            '';
+          };
+          allowForce = mkOption {
+            default = false;
+            type = types.bool;
+            description = ''
+              Determines if builds can be forced via IRC
+            '';
+          };
+          nick = mkOption {
+            default = "nix-buildbot";
+            type = types.str;
+            description = ''
+              nickname for IRC
+            '';
+          };
+          server = mkOption {
+            default = "irc.freenode.net";
+            type = types.str;
+            description = ''
+              Buildbot Status IRC Server to connect to
+            '';
+          };
+        };
+      });
+    };
+
+    extraConfig = mkOption {
+      default = "";
+      type = types.lines;
+      description = ''
+        extra config appended to the generated master.cfg
+      '';
+    };
+  };
+
+  imp = {
+
+    users.extraUsers.buildbotMaster = {
+      uid = genid "buildbotMaster";
+      description = "Buildbot Master";
+      home = cfg.workDir;
+      createHome = false;
+    };
+
+    users.extraGroups.buildbotMaster = {
+      gid = 672626386;
+    };
+
+    systemd.services.buildbotMaster = {
+      description = "Buildbot Master";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      # TODO: add extra dependencies to master like svn and cvs
+      path = [ pkgs.git ];
+      environment = {
+        SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
+      };
+      serviceConfig = let
+        workdir="${lib.shell.escape cfg.workDir}";
+        secretsdir="${lib.shell.escape (toString <secrets>)}";
+      in {
+        PermissionsStartOnly = true;
+        Type = "forking";
+        PIDFile = "${workdir}/twistd.pid";
+        # TODO: maybe also prepare buildbot.tac?
+        ExecStartPre = pkgs.writeScript "buildbot-master-init" ''
+          #!/bin/sh
+          set -efux
+          if [ ! -e ${workdir} ];then
+            mkdir -p ${workdir}
+            ${buildbot}/bin/buildbot create-master -r -l 10 -f ${workdir}
+          fi
+          # always override the master.cfg
+          cp ${buildbot-master-config} ${workdir}/master.cfg
+
+          # copy secrets
+          ${ concatMapStringsSep "\n"
+            (f: "cp ${secretsdir}/${f} ${workdir}/${f}" ) cfg.secrets }
+          # sanity
+          ${buildbot}/bin/buildbot checkconfig ${workdir}
+
+          # TODO: maybe upgrade? not sure about this
+          #       normally we should write buildbot.tac by our own
+          # ${buildbot}/bin/buildbot upgrade-master ${workdir}
+
+          chmod 700 -R ${workdir}
+          chown buildbotMaster:buildbotMaster -R ${workdir}
+        '';
+        ExecStart = "${buildbot}/bin/buildbot start ${workdir}";
+        ExecStop = "${buildbot}/bin/buildbot stop ${workdir}";
+        ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}";
+        PrivateTmp = "true";
+        User = "buildbotMaster";
+        Restart = "always";
+        RestartSec = "10";
+      };
+    };
+  };
+in
+{
+  options.krebs.buildbot.master = api;
+  config = mkIf cfg.enable imp;
+}
diff --git a/makefu/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix
index 7c9ea79c0..0e7796d8a 100644
--- a/makefu/3modules/buildbot/slave.nix
+++ b/krebs/3modules/buildbot/slave.nix
@@ -39,7 +39,7 @@ let
     s.setServiceParent(application)
     '';
   default-packages = [ pkgs.git pkgs.bash ];
-  cfg = config.makefu.buildbot.slave;
+  cfg = config.krebs.buildbot.slave;
 
   api = {
     enable = mkEnableOption "Buildbot Slave";
@@ -144,6 +144,7 @@ let
       path = default-packages ++ cfg.packages;
 
       environment = {
+          SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
           NIX_REMOTE="daemon";
       } // cfg.extraEnviron;
 
@@ -180,6 +181,6 @@ let
   };
 in
 {
-  options.makefu.buildbot.slave = api;
+  options.krebs.buildbot.slave = api;
   config = mkIf cfg.enable imp;
 }
diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix
index deccecc48..ba1f425d9 100644
--- a/krebs/3modules/default.nix
+++ b/krebs/3modules/default.nix
@@ -10,6 +10,8 @@ let
       ./backup.nix
       ./bepasty-server.nix
       ./build.nix
+      ./buildbot/master.nix
+      ./buildbot/slave.nix
       ./current.nix
       ./exim-retiolum.nix
       ./exim-smarthost.nix
diff --git a/krebs/3modules/makefu/default.nix b/krebs/3modules/makefu/default.nix
index 1970a0777..31516d591 100644
--- a/krebs/3modules/makefu/default.nix
+++ b/krebs/3modules/makefu/default.nix
@@ -83,6 +83,9 @@ with lib;
             '';
         };
       };
+      ssh.privkey.path = <secrets/ssh_host_ed25519_key>;
+      ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHDM0E608d/6rGzXqGbNSuMb2RlCojCJSiiz6QcPOC2G root@pornocauster";
+
     };
 
     vbob = {
@@ -108,6 +111,8 @@ with lib;
             '';
         };
       };
+      ssh.privkey.path = <secrets/ssh_host_ed25519_key>;
+      ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICPLTMl+thSq77cjYa2XF7lz5fA7JMftrLo8Dy/OBXSg root@nixos";
     };
     flap = rec {
       cores = 1;
@@ -238,6 +243,8 @@ with lib;
           '';
         };
       };
+      ssh.privkey.path = <secrets/ssh_host_ed25519_key>;
+      ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH4Tjx9qK6uWtxT1HCpeC0XvDZKO/kaPygyKatpAqU6I root@wry";
     };
     filepimp = rec {
       cores = 1;
@@ -287,6 +294,8 @@ with lib;
             '';
         };
       };
+      ssh.privkey.path = <secrets/ssh_host_ed25519_key>;
+      ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIujMZ3ZFxKpWeB/cjfKfYRr77+VRZk0Eik+92t03NoA root@servarch";
     };
     gum = rec {
       cores = 1;
@@ -327,6 +336,8 @@ with lib;
             '';
         };
       };
+      ssh.privkey.path = <secrets/ssh_host_ed25519_key>;
+      ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIcxWFEPzke/Sdd9qNX6rSJgXal8NmINYajpFCxXfYdj root@gum";
     };
   };
   users = addNames rec {
diff --git a/krebs/3modules/shared/default.nix b/krebs/3modules/shared/default.nix
index b332676c6..518e46587 100644
--- a/krebs/3modules/shared/default.nix
+++ b/krebs/3modules/shared/default.nix
@@ -7,6 +7,7 @@ let
     "test-arch"
     "test-centos6"
     "test-centos7"
+    "test-all-krebs-modules"
   ] (name: {
     inherit name;
     cores = 1;
diff --git a/krebs/5pkgs/Reaktor/plugins.nix b/krebs/5pkgs/Reaktor/plugins.nix
new file mode 100644
index 000000000..7490be4ca
--- /dev/null
+++ b/krebs/5pkgs/Reaktor/plugins.nix
@@ -0,0 +1,120 @@
+{ stdenv, lib, pkgs, makeWrapper }:
+
+rec {
+  # Begin API
+  buildBaseReaktorPlugin = { name
+                        , config # python extra configuration for plugin
+                        , phases ? []
+                        , ... } @ attrs:
+    stdenv.mkDerivation (attrs // {
+      name = "Reaktor-plugin-" + name;
+      isReaktorPlugin = true;
+    });
+
+  buildSimpleReaktorPlugin = name: { script
+                        , path ? []
+                        , env ? {}
+                        , append_rule ? false # append the rule instead of insert
+                        , pattern ? ""
+                        , ... } @ attrs:
+    let
+      path_env = { "PATH" = lib.makeSearchPath "bin" (path ++ [ pkgs.coreutils ]); };
+      src_dir = pkgs.substituteAll ( {
+        inherit name;
+        dir = "bin";
+        isExecutable = true;
+        src = script;
+      });
+      src_file = "${src_dir}/bin/${name}";
+      config = ''
+        public_commands.${if append_rule then "append(" else "insert(0," }{
+          'capname' : "${name}",
+          'pattern' : ${if pattern == "" then
+                          ''indirect_pattern.format("${name}")'' else