Merge remote-tracking branch 'prism/master'

38 files changed, 435 insertions, 965 deletions

diff --git a/krebs/1systems/hotdog/config.nix b/krebs/1systems/hotdog/config.nix
index 9f1ac9134..84eaeaa19 100644
--- a/krebs/1systems/hotdog/config.nix
+++ b/krebs/1systems/hotdog/config.nix
@@ -10,6 +10,7 @@
     <stockholm/krebs/2configs/ircd.nix>
     <stockholm/krebs/2configs/reaktor2.nix>
     <stockholm/krebs/2configs/wiki.nix>
+    <stockholm/krebs/2configs/acme.nix>
 
     ## shackie irc bot
     <stockholm/krebs/2configs/shack/reaktor.nix>
diff --git a/krebs/2configs/acme.nix b/krebs/2configs/acme.nix
new file mode 100644
index 000000000..056aa7ae4
--- /dev/null
+++ b/krebs/2configs/acme.nix
@@ -0,0 +1,67 @@
+# generate intermediate certificate with generate-krebs-intermediate-ca
+{ config, lib, pkgs, ... }: let
+  domain = "ca.r";
+in {
+  security.acme = {
+    acceptTerms = true; # kinda pointless since we never use upstream
+    email = "spam@krebsco.de";
+    certs.${domain}.server = "https://${domain}:1443/acme/acme/directory"; # use 1443 here cause bootstrapping loop
+  };
+  networking.firewall.allowedTCPPorts = [ 80 443 ];
+  services.nginx = {
+    enable = true;
+    recommendedProxySettings = true;
+    virtualHosts.${domain} = {
+      addSSL = true;
+      enableACME = true;
+      locations."/" = {
+        proxyPass = "https://localhost:1443";
+      };
+      locations."= /ca.crt".alias = ../6assets/krebsAcmeCA.crt;
+    };
+  };
+  krebs.secret.files.krebsAcme = {
+    path = "/var/lib/step-ca/intermediate_ca.key";
+    owner.name = "root";
+    mode = "1444";
+    source-path = builtins.toString <secrets> + "/acme_ca.key";
+  };
+  services.step-ca = {
+    enable = true;
+    intermediatePasswordFile = "/dev/null";
+    address = "0.0.0.0";
+    port = 1443;
+    settings = {
+      root = pkgs.writeText "root.crt" config.krebs.ssl.rootCA;
+      crt = pkgs.writeText "intermediate.crt" config.krebs.ssl.intermediateCA;
+      key = "/var/lib/step-ca/intermediate_ca.key";
+      dnsNames = [ domain ];
+      logger.format = "text";
+      db = {
+        type = "badger";
+        dataSource = "/var/lib/step-ca/db";
+      };
+      authority = {
+        provisioners = [{
+          type = "ACME";
+          name = "acme";
+          forceCN = true;
+        }];
+        claims = {
+          maxTLSCertDuration = "2160h";
+          defaultTLSCertDuration = "2160h";
+        };
+        backdate = "1m0s";
+      };
+      tls = {
+        cipherSuites = [
+          "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256"
+          "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256"
+        ];
+        minVersion = 1.2;
+        maxVersion = 1.3;
+        renegotiation = false;
+      };
+    };
+  };
+}
diff --git a/krebs/2configs/buildbot-stockholm.nix b/krebs/2configs/buildbot-stockholm.nix
index 43a38a9f8..9fc6a79e5 100644
--- a/krebs/2configs/buildbot-stockholm.nix
+++ b/krebs/2configs/buildbot-stockholm.nix
@@ -6,11 +6,13 @@
     enable = true;
     virtualHosts.build = {
       serverAliases = [ "build.r" "build.${config.networking.hostName}.r" ];
-      locations."/".extraConfig = ''
-        proxy_set_header Upgrade $http_upgrade;
-        proxy_set_header Connection "upgrade";
-        proxy_pass http://127.0.0.1:${toString config.krebs.buildbot.master.web.port};
-      '';
+      locations."/" = {
+        proxyPass = "http://127.0.0.1:${toString config.services.buildbot-master.port}";
+        proxyWebsockets = true;
+        extraConfig = ''
+          proxy_read_timeout 3600s;
+        '';
+      };
     };
   };
   krebs.ci = {
@@ -18,25 +20,20 @@
     repos = {
       disko.urls = [
         "http://cgit.gum.r/disko"
-        "http://cgit.hotdog.r/disko"
         "http://cgit.ni.r/disko"
         "http://cgit.prism.r/disko"
       ];
       krops.urls = [
-        "http://cgit.hotdog.r/krops"
         "http://cgit.ni.r/krops"
         "http://cgit.prism.r/krops"
         "https://github.com/krebs/krops.git"
       ];
       nix_writers.urls = [
-        "http://cgit.hotdog.r/nix-writers"
         "http://cgit.ni.r/nix-writers"
         "http://cgit.prism.r/nix-writers"
       ];
       stockholm.urls = [
-        "http://cgit.enklave.r/stockholm"
         "http://cgit.gum.r/stockholm"
-        "http://cgit.hotdog.r/stockholm"
         "http://cgit.ni.r/stockholm"
         "http://cgit.prism.r/stockholm"
       ];
diff --git a/krebs/2configs/default.nix b/krebs/2configs/default.nix
index 8a84d4465..9200d41fe 100644
--- a/krebs/2configs/default.nix
+++ b/krebs/2configs/default.nix
@@ -4,18 +4,14 @@ with import <stockholm/lib>;
 {
   imports = [
     ./backup.nix
-    (let ca-bundle = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; in {
-      environment.variables = {
-        CURL_CA_BUNDLE = ca-bundle;
-        GIT_SSL_CAINFO = ca-bundle;
-        SSL_CERT_FILE = ca-bundle;
-      };
-    })
   ];
   krebs.announce-activation.enable = true;
   krebs.enable = true;
   krebs.tinc.retiolum.enable = mkDefault true;
 
+  # trust krebs ACME CA
+  krebs.ssl.trustIntermediate = true;
+
   krebs.build.user = mkDefault config.krebs.users.krebs;
 
   networking.hostName = config.krebs.build.host.name;
diff --git a/krebs/2configs/shack/ssh-keys.nix b/krebs/2configs/shack/ssh-keys.nix
index 50bb93809..80957f3a5 100644
--- a/krebs/2configs/shack/ssh-keys.nix
+++ b/krebs/2configs/shack/ssh-keys.nix
@@ -7,6 +7,7 @@
     config.krebs.users.raute.pubkey
     config.krebs.users.ulrich.pubkey
     config.krebs.users.xq.pubkey
+    config.krebs.users.xkey.pubkey
     "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAEAQDb9NPa2Hf51afcG1H13UPbE5E02J8aC9a1sGCRls592wAVlQbmojYR1jWDPA2m32Bsyv0ztqi81zDyndWWZPQVJVBk00VjYBcgk6D5ifqoAuWLzfuHJPWZGOvBf/U74/LNFNUkj1ywjneK7HYTRPXrRBBfBSQNmQzkvue7s599L2vdueZKyjNsMpx2m6nm2SchaMuDskSQut/168JgU1l4M8BeT68Bo4WdelhBYnhSI1a59FGkgdu2SCjyighLQRy2sOH3ksnkHWENPkA+wwQOlKl7R3DsEybrNd4NU9FSwFDyDmdhfv5gJp8UGSFdjAwx43+8zM5t5ruZ25J0LnVb0PuTuRA00UsW83MkLxFpDQLrQV08tlsY6iGrqxP67C3VJ6t4v6oTp7/vaRLhEFc1PhOLh+sZ18o8MLO+e2rGmHGHQnSKfBOLUvDMGa4jb01XBGjdnIXLOkVo79YR5jZn7jJb2gTZ95OD6bWSDADoURSuwuLa7kh4ti1ItAKuhkIvbuky3rRVvQEc92kJ6aNUswIUXJa0K2ibbIY6ycKAA3Ljksl3Mm9KzOn6yc/i/lSF+SOrTGhabPJigKkIoqKIwnV5IU3gkfsxPQJOBMPqHDGAOeYQe3WpWedEPYuhQEczw4exMb9TkNE96F71PzuQPJDl5sPAWyPLeMKpy5XbfRiF2by4nxN3ZIQvjtoyVkjNV+qM0q0yKBzLxuRAEQOZ2yCEaBudZQkQiwHD97H2vu4SRQ/2aOie1XiOnmdbQRDZSO3BsoDK569K1w+gDfSnqY7zVUMj6tw+uKx6Gstck5lbvYMtdWKsfPv/pDM8eyIVFLL93dKTX+ertcQj6xDwLfOiNubE5ayFXhYkjwImV6NgfBuq+3hLK0URP2rPlOZbbZTQ0WlKD6CCRZPMSZCU9oD2zYfqpvRArBUcdkAwGePezORkfJQLE6mYEJp6pdFkJ/IeFLbO6M0lZVlfnpzAC9kjjkMCRofZUETcFSppyTImCbgo3+ok59/PkNU5oavBXyW80ue2tWHr08HX/QALNte3UITmIIlU6SFMCPMWJqadK1eDPWfJ4H4iDXRNn3D5wqN++iMloKvpaj0wieqXLY4+YfvNTNr177OU48GEWW8DnoEkbpwsCbjPxznGDQhdDqdYyMY/fDgRQReKITvKYGHRzesGysw5cKsp9LEfXD0R6WE2TeiiENla5AWzTgXJB0AyZEcOiIfqOgT9Nr9S8q5gc/BdA7P+jhGGJgEHhV3dVlfIZ7pmZc27Yu7UTQ0lbAKWqcMSTOdne+QL6ILzbvLrQwdvax4tQdm5opfU16SrOox1AMwAbkdq84z6uJqYVx3cUXfMJgTyDNrVv3or root@plattenschwein" # for backup
     "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC1Lx5MKtVjB/Ef6LpEiIAgVwY5xKQFdHuLQR+odQO4cAgxj1QaIXGN0moixY52DebVQhAtiCNiFZ83uJyOj8kmu30yuXwtSOQeqziA859qMJKZ4ZcYdKvbXwnf2Chm5Ck/0FvtpjTWHIZAogwP1wQto/lcqHOjrTAnZeJfQuHTswYUSnmUU5zdsEZ9HidDPUc2Gv0wkBNd+KMQyOZl0HkaxHWvn0h4KK4hYZisOpeTfXJxD87bo+Eg4LL2vvnHW6dF6Ygrbd/0XRMsRRI8OAReVBUoJn7IE1wwAl/FpblNmhaF9hlL7g7hR1ADvaWMMw0e8SSzW6Y+oIa8qFQL6wR1 gitlab-builder" # for being deployed by gitlab ci
   ];
diff --git a/krebs/2configs/wiki.nix b/krebs/2configs/wiki.nix
index 9a18b8dff..40d946f7d 100644
--- a/krebs/2configs/wiki.nix
+++ b/krebs/2configs/wiki.nix
@@ -29,6 +29,7 @@ in
 {
   services.gollum = {
     enable = true;
+    address = "::1";
     extraConfig = ''
       Gollum::Hook.register(:post_commit, :hook_id) do |committer, sha1|
         system('${pushCgit}')
@@ -38,16 +39,20 @@ in
 
   systemd.services.gollum.environment.LC_ALL = "en_US.UTF-8";
 
-  networking.firewall.allowedTCPPorts = [ 80 ];
+  networking.firewall.allowedTCPPorts = [ 80 443 ];
+  security.acme.certs."wiki.r".server = config.krebs.ssl.acmeURL;
   services.nginx = {
     enable = true;
-    virtualHosts.wiki = {
-      serverAliases = [ "wiki.r" "wiki.${config.networking.hostName}.r" ];
-      locations."/".extraConfig = ''
-        proxy_set_header Upgrade $http_upgrade;
-        proxy_set_header Connection "upgrade";
-        proxy_pass http://127.0.0.1:${toString config.services.gollum.port};
-      '';
+    virtualHosts."wiki.r" = {
+      enableACME = true;
+      addSSL = true;
+      locations."/" = {
+        proxyPass = "http://[::1]:${toString config.services.gollum.port}";
+        proxyWebsockets = true;
+        extraConfig = ''
+          proxy_set_header Host $host;
+        '';
+      };
     };
   };
 
diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
deleted file mode 100644
index c30f31e31..000000000
--- a/krebs/3modules/buildbot/master.nix
+++ /dev/null
@@ -1,382 +0,0 @@
-{ config, pkgs, lib, ... }:
-
-with import <stockholm/lib>;
-let
-  buildbot-master-config = pkgs.writeText "buildbot-master.cfg" ''
-    # -*- python -*-
-    from buildbot.plugins import *
-    import re
-    import json
-    c = BuildmasterConfig = {}
-
-    c['slaves'] = []
-    slaves = json.loads('${builtins.toJSON cfg.slaves}')
-    slavenames = [ s for s in slaves ]
-    for k,v in slaves.items():
-      c['slaves'].append(buildslave.BuildSlave(k, v))
-
-    # TODO: configure protocols?
-    c['protocols'] = {'pb': {'port': 9989}}
-
-    ####### Build Inputs
-    c['change_source'] = cs = []
-
-    ${ concatStringsSep "\n"
-    (mapAttrsToList (n: v: ''
-        #### Change_Source: Begin of ${n}
-        ${v}
-        #### Change_Source: End of ${n}
-      '') cfg.change_source )}
-
-    ####### Build Scheduler
-    c['schedulers'] = sched = []
-
-    ${ concatStringsSep "\n"
-    (mapAttrsToList (n: v: ''
-        #### Schedulers: Begin of ${n}
-        ${v}
-        #### Schedulers: End of ${n}
-      '') cfg.scheduler )}
-
-    ###### Builder
-    c['builders'] = bu = []
-    
-    # Builder Pre: Begin
-    ${cfg.builder_pre}
-    # Builder Pre: End
-
-    ${ concatStringsSep "\n"
-    (mapAttrsToList (n: v: ''
-        #### Builder: Begin of ${n}
-        ${v}
-        #### Builder: End of ${n}
-      '') cfg.builder )}
-
-
-    ####### Status
-    c['status'] = st = []
-
-    # If you want to configure this url, override with extraConfig
-    c['buildbotURL'] = "http://${config.networking.hostName}:${toString cfg.web.port}/"
-
-    ${optionalString (cfg.web.enable) ''
-      from buildbot.status import html
-      from buildbot.status.web import authz, auth
-      authz_cfg=authz.Authz(
-          auth=auth.BasicAuth([ ("${cfg.web.username}","${cfg.web.password}") ]),
-          # TODO: configure harder
-          gracefulShutdown = False,
-          forceBuild = 'auth',
-          forceAllBuilds = 'auth',
-          pingBuilder = False,
-          stopBuild = 'auth',
-          stopAllBuilds = 'auth',
-          cancelPendingBuild = 'auth'
-      )
-      # TODO: configure krebs.nginx
-      st.append(html.WebStatus(http_port=${toString cfg.web.port}, authz=authz_cfg))
-      ''}
-
-    ${optionalString (cfg.irc.enable) ''
-      from buildbot.status import words
-      irc = words.IRC("${cfg.irc.server}", "${cfg.irc.nick}",
-                      channels=${builtins.toJSON cfg.irc.channels},
-                      notify_events={
-                        'started': 1,
-                        'success': 1,
-                        'failure': 1,
-                        'exception': 1,
-                        'successToFailure': 1,
-                        'failureToSuccess': 1,
-                      }${optionalString cfg.irc.allowForce ",allowForce=True"})
-      c['status'].append(irc)
-      ''}
-
-    ${ concatStringsSep "\n"
-    (mapAttrsToList (n: v: ''
-        #### Status: Begin of ${n}
-        ${v}
-        #### Status: End of ${n}
-      '') cfg.status )}
-
-    ####### PROJECT IDENTITY
-    c['title'] = "${cfg.title}"
-    c['titleURL'] = "http://krebsco.de"
-
-
-    ####### DB URL
-    # TODO: configure
-    c['db'] = {
-        'db_url' : "sqlite:///state.sqlite",
-    }
-    ${cfg.extraConfig}
-    '';
-
-  cfg = config.krebs.buildbot.master;
-
-  api = {
-    enable = mkEnableOption "Buildbot Master";
-    title = mkOption {
-      default = "Buildbot CI";
-      type = types.str;
-      description = ''
-        Title of the Buildbot Installation
-      '';
-    };
-    workDir = mkOption {
-      default = "/var/lib/buildbot/master";
-      type = types.str;
-      description = ''
-        Path to build bot master directory.
-        Will be created on startup.
-      '';
-    };
-
-    secrets = mkOption {
-      default = [];
-      type = types.listOf types.str;
-      example = [ "cac.json" ];
-      description = ''
-        List of all the secrets in ‹secrets› which should be copied into the
-        buildbot master directory.
-      '';
-    };
-
-    slaves = mkOption {
-      default = {};
-      type = types.attrsOf types.str;
-      description = ''
-        Attrset of slavenames with their passwords
-        slavename = slavepassword
-      '';
-    };
-
-    change_source = mkOption {
-      default = {};
-      type = types.attrsOf types.str;
-      example = {
-        stockholm = ''
-          cs.append(changes.GitPoller(
-                  'http://cgit.gum/stockholm',
-                  workdir='stockholm-poller', branch='master',
-                  project='stockholm',
-                  pollinterval=120))
-        '';
-      };
-      description = ''
-        Attrset of all the change_sources which should be configured.
-        It will be directly included into the master configuration.
-
-        At the end an change object should be appended to <literal>cs</literal>
-      '';
-    };
-
-    scheduler = mkOption {
-      default = {};
-      type = types.attrsOf types.str;
-      example = {
-        force-scheduler = ''
-          sched.append(schedulers.ForceScheduler(
-                                      name="force",
-                                      builderNames=["full-tests"]))
-        '';
-      };
-      description = ''
-        Attrset of all the schedulers which should be configured.
-        It will be directly included into the master configuration.
-
-        At the end an change object should be appended to <literal>sched</literal>
-      '';
-    };
-
-    builder_pre = mkOption {
-      default = "";
-      type = types.lines;
-      example = ''
-        grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
-      '';
-      description = ''
-        some code before the builders are being assembled.
-        can be used to define functions used by multiple builders
-      '';
-    };
-
-    builder = mkOption {
-      default = {};
-      type = types.attrsOf types.str;
-      example = {
-        fast-test = ''
-        '';
-      };
-      description = ''
-        Attrset of all the builder which should be configured.
-        It will be directly included into the master configuration.
-
-        At the end an change object should be appended to <literal>bu</literal>
-      '';
-    };
-
-    status = mkOption {
-      default = {};
-      type = types.attrsOf types.str;
-      description = ''
-        Attrset of all the extra status which should be configured.
-        It will be directly included into the master configuration.
-
-        At the end an change object should be appended to <literal>st</literal>
-
-        Right now IRC and Web status can be configured by setting
-        <literal>buildbot.master.irc.enable</literal> and
-        <literal>buildbot.master.web.enable</literal>
-      '';
-    };
-
-    # Configurable Stati
-    web = mkOption {
-      default = {};
-      type = types.submodule ({ config2, ... }: {
-        options = {
-          enable = mkEnableOption "Buildbot Master Web Status";
-          username = mkOption {
-            default = "krebs";
-            type = types.str;
-            description = ''
-              username for web authentication
-            '';
-          };
-          hostname = mkOption {
-            default = config.networking.hostName;
-            type = types.str;
-            description = ''
-              web interface Hostname
-            '';
-          };
-          password = mkOption {
-            default = "bob";
-            type = types.str;
-            description = ''
-              password for web authentication
-            '';
-          };
-          port = mkOption {
-            default = 8010;
-            type = types.int;
-            description = ''
-              port for buildbot web status
-            '';
-          };
-        };
-      });
-    };
-
-    irc = mkOption {
-      default = {};
-      type = types.submodule ({ config, ... }: {
-        options = {
-          enable = mkEnableOption "Buildbot Master IRC Status";
-          channels = mkOption {
-            default = [ "nix-buildbot-meetup" ];
-            type = with types; listOf str;
-            description = ''
-              irc channels the bot should connect to
-            '';
-          };
-          allowForce = mkOption {
-            default = false;
-            type = types.bool;
-            description = ''
-              Determines if builds can be forced via IRC
-            '';
-          };
-          nick = mkOption {
-            default = "nix-buildbot";
-            type = types.str;
-            description = ''
-              nickname for IRC
-            '';
-          };
-          server = mkOption {
-            default = "irc.freenode.net";
-            type = types.str;
-            description = ''
-              Buildbot Status IRC Server to connect to
-            '';
-          };
-        };
-      });
-    };
-
-    extraConfig = mkOption {
-      default = "";
-      type = types.lines;
-      description = ''
-        extra config appended to the generated master.cfg
-      '';
-    };
-  };
-
-  imp = {
-
-    users.extraUsers.buildbotMaster = {
-      uid = genid "buildbotMaster";
-      group = "buildbotMaster";
-      description = "Buildbot Master";
-      home = cfg.workDir;
-      createHome = false;
-      isSystemUser = true;
-    };
-
-    users.extraGroups.buildbotMaster = {
-      gid = 672626386;
-    };
-
-    systemd.services.buildbotMaster = {
-      description = "Buildbot Master";
-      after = [ "network.target" ];
-      wantedBy = [ "multi-user.target" ];
-      # TODO: add extra dependencies to master like svn and cvs
-      path = [ pkgs.git ];
-      environment = {
-        SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
-      };
-      serviceConfig = let
-        workdir = shell.escape cfg.workDir;
-        secretsdir = shell.escape (toString <secrets>);
-      in {
-        PermissionsStartOnly = true;
-        # TODO: maybe also prepare buildbot.tac?
-        ExecStartPre = pkgs.writeDash "buildbot-master-init" ''
-          set -efux
-          if [ ! -e ${workdir} ];then
-            mkdir -p ${workdir}
-            ${pkgs.buildbot-classic}/bin/buildbot create-master -r -l 10 -f ${workdir}
-          fi
-          # always override the master.cfg
-          cp ${buildbot-master-config} ${workdir}/master.cfg
-
-          # copy secrets
-          ${ concatMapStringsSep "\n"
-            (f: "cp ${secretsdir}/${f} ${workdir}/${f}" ) cfg.secrets }
-          # sanity
-          ${pkgs.buildbot-classic}/bin/buildbot checkconfig ${workdir}
-
-          # TODO: maybe upgrade? not sure about this
-          #       normally we should write buildbot.tac by our own
-          # ${pkgs.buildbot-classic}/bin/buildbot upgrade-master ${workdir}
-
-          chmod 700 ${workdir}
-          chown buildbotMaster:buildbotMaster -R ${workdir}
-        '';
-        ExecStart = "${pkgs.buildbot-classic}/bin/buildbot start --nodaemon ${workdir}";
-        PrivateTmp = "true";
-        User = "buildbotMaster";
-        Restart = "always";
-        RestartSec = "10";
-      };
-    };
-  };
-in
-{
-  options.krebs.buildbot.master = api;
-  config = lib.mkIf cfg.enable imp;
-}
diff --git a/krebs/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix
deleted file mode 100644
index f97b50def..000000000
--- a/krebs/3modules/buildbot/slave.nix
+++ /dev/null
@@ -1,186 +0,0 @@
-{ config, pkgs, lib, ... }:
-
-with import <stockholm/lib>;
-let
-  buildbot-slave-init = pkgs.writeText "buildbot-slave.tac" ''
-    import os
-
-    from buildslave.bot import BuildSlave
-    from twisted.application import service
-
-    basedir = '${cfg.workDir}'
-    rotateLength = 10000000
-    maxRotatedFiles = 10
-
-    application = service.Application('buildslave')
-
-    from twisted.python.logfile import LogFile
-    from twisted.python.log import ILogObserver, FileLogObserver
-    logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength,
-                                  maxRotatedFiles=maxRotatedFiles)
-    application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
-
-    buildmaster_host = '${cfg.masterhost}'
-    # TODO: masterport?
-    port = 9989
-    slavename = '${cfg.username}'
-    passwd = '${cfg.password}'
-    keepalive = 600
-    usepty = 0
-    umask = None
-    maxdelay = 300
-    allow_shutdown = None
-
-    ${cfg.extraConfig}
-
-    s = BuildSlave(buildmaster_host, port, slavename, passwd, basedir,
-                  keepalive, usepty, umask=umask, maxdelay=maxdelay,
-                  allow_shutdown=allow_shutdown)
-    s.setServiceParent(application)
-    '';
-  default-packages = [ pkgs.git pkgs.bash ];
-  cfg = config.krebs.buildbot.slave;
-
-  api = {
-    enable = mkEnableOption "Buildbot Slave";
-
-    workDir = mkOption {
-      default = "/var/lib/buildbot/slave";
-      type = types.str;
-      description = ''
-        Path to build bot slave directory.
-        Will be created on startup.
-      '';
-    };
-
-    masterhost = mkOption {
-      default = "localhost";
-      type = types.str;
-      description = ''
-        Hostname/IP of the buildbot master
-      '';
-    };
-
-    username = mkOption {
-      type = types.str;
-      description = ''
-        slavename used to authenticate with master
-      '';
-    };
-
-    password = mkOption {
-      type = types.str;
-      description = ''
-        slave password used to authenticate with master
-      '';
-    };
-
-    contact = mkOption {
-      default = "nix slave <buildslave@${config.networking.hostName}>";
-      type = types.str;
-      description = ''
-        contact to be announced by buildslave
-      '';
-    };
-
-    description = mkOption {
-      default = "Nix Generated BuildSlave";
-      type = types.str;
-      description = ''
-        description for hostto be announced by buildslave
-      '';
-    };
-
-    packages = mkOption {
-      default = [ pkgs.git ];
-      type = with types; listOf package;
-      description = ''
-        packages which should be in path for buildslave
-      '';
-    };
-
-    extraEnviron = mkOption {
-      default = {};
-      example = {
-        NIX_PATH = "nixpkgs=/path/to/my/nixpkgs";
-      };
-      type = types.attrsOf types.str;
-      description = ''
-        extra environment variables to be provided to the buildslave service
-        if you need nixpkgs, e.g. for running nix-shell you can set NIX_PATH here.
-      '';
-    };
-
-    extraConfig = mkOption {
-      default = "";
-      type = types.lines;
-      example = ''
-        port = 443
-        keepalive = 600
-      '';
-      description = ''
-        extra config evaluated before calling BuildSlave init in .tac file
-      '';
-    };
-  };
-
-  imp = {
-
-    users.extraUsers.buildbotSlave = {
-      uid = genid "buildbotSlave";
-      group = "buildbotSlave";
-      description = "Buildbot Slave";
-      home = cfg.workDir;
-      createHome = false;
-      isSystemUser = true;
-    };
-
-    users.extraGroups.buildbotSlave = {
-      gid = 1408105834;
-    };
-
-    systemd.services."buildbotSlave-${cfg.username}-${cfg.masterhost}" = {
-      description = "Buildbot Slave for ${cfg.username}@${cfg.masterhost}";
-      after = [ "network.target" ];
-      wantedBy = [ "multi-user.target" ];
-      path = default-packages ++ cfg.packages;
-
-      environment = {
-          SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
-          NIX_REMOTE="daemon";
-      } // cfg.extraEnviron;
-
-      serviceConfig = let
-        workdir = shell.escape cfg.workDir;
-        contact = shell.escape cfg.contact;
-        description = shell.escape cfg.description;
-        # TODO:make this
-      in {
-        PermissionsStartOnly = true;
-        Type = "forking";
-        PIDFile = "${workdir}/twistd.pid";
-        # TODO: maybe also prepare buildbot.tac?
-        ExecStartPre = pkgs.writeDash "buildbot-master-init" ''
-          set -efux
-          mkdir -p ${workdir}/info
-          cp ${buildbot-slave-init} ${workdir}/buildbot.tac
-          echo ${contact} > ${workdir}/info/admin
-          echo ${description} > ${workdir}/info/host
-
-          chown buildbotSlave:buildbotSlave -R ${workdir}
-          chmod 700 ${workdir}
-        '';
-        ExecStart = "${pkgs.buildbot-classic-slave}/bin/buildslave start ${workdir}";
-        ExecStop = "${pkgs.buildbot-classic-slave}/bin/buildslave stop ${workdir}";
-        PrivateTmp = "true";
-        User = "buildbotSlave";
-        Restart = "always";
-        RestartSec = "10";
-      };
-    };
-  };
-in
-{
-  options.krebs.buildbot.slave = api;
-  config = lib.mkIf cfg.enable imp;
-}
diff --git a/krebs/3modules/ci.nix b/krebs/3modules/ci.nix
index 50db0b971..822dbab61 100644
--- a/krebs/3modules/ci.nix
+++ b/krebs/3modules/ci.nix
@@ -39,148 +39,144 @@ let
 
   profileRoot = "/nix/var/nix/profiles/ci";
 
-  imp = {
-    krebs.buildbot.master = {
-      slaves = {
-        testslave = "lasspass";
-      };
+  bcfg = config.services.buildbot-master;
 
-      change_source = mapAttrs' (name: repo:
-        nameValuePair name (concatMapStrings (url: ''
-          cs.append(
-              changes.GitPoller(
-                  "${url}",
-                  workdir='${name}-${elemAt(splitString "." url) 1}', branches=True,
-                  project='${name}',
-                  pollinterval=100
-              )
+  imp = {
+    services.buildbot-master = {
+      workers = [ "worker.Worker('testworker', 'pass')" ];
+
+      changeSource = mapAttrsToList (name: repo:
+        concatMapStringsSep "," (url: ''
+          changes.GitPoller(
+              "${url}",
+              workdir='${name}-${elemAt(splitString "." url) 1}', branches=True,
+              project='${name}',
+              pollinterval=100
           )
-        '') repo.urls)
+        '') repo.urls
       ) cfg.repos;
 
-      scheduler = mapAttrs' (name: repo:
-        nameValuePair name ''
-          sched.append(
-              schedulers.SingleBranchScheduler(
-                  change_filter=util.ChangeFilter(
-                      branch_re=".*",
-                      project='${name}',
-                  ),
-                  treeStableTimer=60,
-                  name="${name}-all-branches",
-                  builderNames=[
-                      "${name}",
-                  ]
-              )
-          )
-          sched.append(
-              schedulers.ForceScheduler(
-                  name="${name}",
-                  builderNames=[
-                      "${name}",
-                  ]
-              )
-          )
-        ''
-      ) cfg.repos;
-      builder_pre = ''
-        from buildbot import interfaces
-        from buildbot.steps.shell import ShellCommand
+      schedulers = mapAttrsToList (name: repo: ''
+        schedulers.SingleBranchScheduler(
+            change_filter=util.ChangeFilter(
+                branch_re=".*",
+                project='${name}',
+            ),
+            treeStableTimer=60,
+            name="${name}-all-branches",
+            builderNames=[
+                "${name}",
+            ]
+        ),
+        schedulers.ForceScheduler(
+            name="${name}",
+            builderNames=[
+                "${name}",
+            ]
+        )
+      '') cfg.repos;
+
+      builders = [];
 
-        class StepToStartMoreSteps(ShellCommand):
+      extraConfig = ''
+        # https://docs.buildbot.net/latest/manual/configuration/buildfactories.html
+        from buildbot.plugins import util, steps
+        from buildbot.process import buildstep, logobserver
+        from twisted.internet import defer
+        import json
+
+        class GenerateStagesCommand(buildstep.ShellMixin, steps.BuildStep):
             def __init__(self, **kwargs):
-                ShellCommand.__init__(self, **kwargs)
-
-            def addBuildSteps(self, steps_factories):
-                for sf in steps_factories:
-                    step = interfaces.IBuildStepFactory(sf).buildStep()
-                    step.setBuild(self.build)
-                    step.setBuildSlave(self.build.slavebuilder.slave)
-                    step_status = self.build.build_status.addStepWithName(step.name)
-                    step.setStepStatus(step_status)
-                    self.build.steps.append(step)
-
-            def start(self):
-                props = self.build.getProperties()
-                new_steps = json.loads(props.getProperty('steps_json'))
-                for new_step in new_steps:
-                    self.addBuildSteps([steps.ShellCommand(
-                        name=str(new_step),
-                        command=[
-                          "${pkgs.writeDash "build-stepper.sh" ''
+                kwargs = self.setupShellMixin(kwargs)
+                super().__init__(**kwargs)
+                self.observer = logobserver.BufferLogObserver()
+                self.addLogObserver('stdio', self.observer)
+
+            def extract_stages(self, stdout):
+                stages = json.loads(stdout)
+                return stages
+
+            @defer.inlineCallbacks
+            def run(self):
+                # run nix-instanstiate to generate the dict of stages
+                cmd = yield self.makeRemoteShellCommand()
+                yield self.runCommand(cmd)
+
+                # if the command passes extract the list of stages
+                result = cmd.results()
+                if result == util.SUCCESS:
+                    # create a ShellCommand for each stage and add them to the build
+                    stages = self.extract_stages(self.observer.getStdout())
+                    self.build.addStepsAfterCurrentStep([
+                        steps.ShellCommand(
+                          name=stage,
+                          env=dict(
+                            build_name = stage,
+                            build_script = stages[stage],
+                          ),
+                          command="${pkgs.writeDash "build.sh" ''
                             set -xefu
                             profile=${shell.escape profileRoot}/$build_name
                             result=$("$build_script")
                             if [ -n "$result" ]; then
                               ${pkgs.nix}/bin/nix-env -p "$profile" --set "$result"
                             fi
-                          ''}"
-                        ],
-                        env={
-                          "build_name": new_step,
-                          "build_script": new_steps[new_step],
-                          "NIX_REMOTE": "daemon",
-                          "NIX_PATH": "secrets=/var/src/stockholm/null:/var/src",
-                        },
-                        timeout=90001,
-                        workdir='build', # TODO figure out why we need this?
-                    )])
-
-                ShellCommand.start(self)
+                          ''}",
+                        ) for stage in stages
+                    ])
 
-      '';
+                return result
 
-      builder = mapAttrs' (name: repo:
-        nameValuePair name ''
-          f_${name} = util.BuildFactory()
-          f_${name}.addStep(steps.Git(
+
+        ${concatStringsSep "\n" (mapAttrsToList (name: repo: ''
+          factory_${name} = util.BuildFactory()
+          factory_${name}.addStep(steps.Git(
               repourl=util.Property('repository', '${head repo.urls}'),
               method='clobber',
               mode='full',
               submodules=True,
           ))
 
-          f_${name}.addStep(steps.SetPropertyFromCommand(
+          factory_${name}.addStep(GenerateStagesCommand(
               env={
-                "NIX_REMOTE": "daemon",
-                "NIX_PATH": "secrets=/var/src/stockholm/null:/var/src",
+                  "NIX_REMOTE": "daemon",
+                  "NIX_PATH": "secrets=/var/src/stockholm/null:/var/src",
               },
-              name="get_steps",
-              command=["${getJobs}"],
-              extract_fn=lambda rc, stdout, stderr: { 'steps_json': stdout },
+              name="Generate build stages",
+              command=[
+                  "${getJobs}"
+              ],
+              haltOnFailure=True,
           ))
-          f_${name}.addStep(StepToStartMoreSteps(command=["echo"])) # TODO remove dummy command from here
 
-          bu.append(
+          c['builders'].append(
               util.BuilderConfig(
                   name="${name}",
-                  slavenames=slavenames,
-                  factory=f_${name}
+                  workernames=['testworker'],
+                  factory=factory_${name}
               )
           )
-        ''
-      ) cfg.repos;
+        '') cfg.repos)}
+      '';
 
       enable = true;
-      web.enable = true;
-      irc = {
-        enable = true;
-        nick = "build|${hostname}";
-        server = "irc.r";
-        channels = [ "xxx" "noise" ];
-        allowForce = true;
-      };
-      extraConfig = ''
-        c['buildbotURL'] = "http://build.${hostname}.r/"
-      '';
+      reporters = [''
+        reporters.IRC(
+          host = "irc.r",
+          nick = "buildbot|${hostname}",
+          notify_events = [ 'started', 'finished', 'failure', 'success', 'exception', 'problem' ],
+          channels = [{"channel": "#xxx"}],
+        )
+      ''];
+
+      buildbotUrl = "http://build.${hostname}.r/";
     };
 
-    krebs.buildbot.slave = {
+    services.buildbot-worker = {
       enable = true;
-      masterhost = "localhost";
-      username = "testslave";
-      password = "lasspass";
-      packages = with pkgs; [ gnumake jq nix populate gnutar lzma gzip ];
+      workerUser = "testworker";
+      workerPass = "pass";
+      packages = with pkgs; [ git gnutar gzip jq nix populate ];
     };
 
     system.activationScripts.buildbots-nix-profile = ''
@@ -192,11 +188,10 @@ let
     users = {
       groups.buildbots.gid = genid "buildbots";
       users = {
-        buildbotMaster.extraGroups = [ "buildbots" ];
-        buildbotSlave.extraGroups = [ "buildbots" ];
+        buildbot.extraGroups = [ "buildbots" ];
+        bbworker.extraGroups = [ "buildbots" ];
       };
     };
   };
 
 in out
-
diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix
index 7b6639212..2772d8d37 100644
--- a/krebs/3modules/default.nix
+++ b/krebs/3modules/default.nix
@@ -13,8 +13,6 @@ let
       ./bepasty-server.nix
       ./bindfs.nix
       ./brockman.nix
-      ./buildbot/master.nix
-      ./buildbot/slave.nix
       ./build.nix
       ./cachecache.nix
       ./ci.nix
@@ -50,6 +48,7 @@ let
       ./secret.nix
       ./setuid.nix
       ./shadow.nix
+      ./ssl.nix
       ./sync-containers.nix
       ./tinc.nix
       ./tinc_graphs.nix
diff --git a/krebs/3modules/external/default.nix b/krebs/3modules/external/default.nix
index fe572f2ae..91ce66742 100644
--- a/krebs/3modules/external/default.nix
+++ b/krebs/3modules/external/default.nix
@@ -754,7 +754,9 @@ in {
       mail = "xq@shackspace.de";
       pubkey = ssh-for "xq";
     };
-    xkey = {};
+    xkey = {
+      pubkey = ssh-for "xkey";
+    };
     miaoski = {
     };
     filly = {
diff --git a/krebs/3modules/external/mic92.nix b/krebs/3modules/external/mic92.nix
index 0e6812a35..3c76a372c 100644
--- a/krebs/3modules/external/mic92.nix
+++ b/krebs/3modules/external/mic92.nix
@@ -19,10 +19,6 @@ in {
           aliases = [ "amy.i" ];
         };
         retiolum = {
-          addrs = [
-            config.krebs.hosts.amy.nets.retiolum.ip4.addr
-            config.krebs.hosts.amy.nets.retiolum.ip6.addr
-          ];
           ip4.addr = "10.243.29.181";
           aliases = [ "amy.r" ];
           tinc.pubkey = ''
@@ -53,10 +49,6 @@ in {
           aliases = [ "clara.i" ];
         };
         retiolum = {
-          addrs = [
-            config.krebs.hosts.clara.nets.retiolum.ip4.addr
-            config.krebs.hosts.clara.nets.retiolum.ip6.addr
-          ];
           ip4.addr = "10.243.29.182";
           aliases = [ "clara.r" ];
           tinc.pubkey = ''
@@ -130,10 +122,6 @@ in {
         };
         retiolum = {
           via = internet;
-          addrs = [
-            config.krebs.hosts.donna.nets.retiolum.ip4.addr
-            config.krebs.hosts.donna.nets.retiolum.ip6.addr
-          ];
           ip4.addr = "10.243.29.180";
           aliases = [ "donna.r" ];
           tinc.pubkey = ''
@@ -159,10 +147,6 @@ in {
       owner = config.krebs.users.mic92;
       nets = rec {
         retiolum = {
-          addrs = [
-            config.krebs.hosts.herbert.nets.retiolum.ip4.addr
-            config.krebs.hosts.herbert.nets.retiolum.ip6.addr
-          ];
           ip4.addr = "10.243.29.177";
           aliases = [ "herbert.r" ];
           tinc.pubkey = ''
@@ -178,40 +162,6 @@ in {
         };
       };
     };
-    eddie = {
-      owner = config.krebs.users.mic92;
-      nets = rec {
-        internet = {
-          # eddie.thalheim.io
-          ip4.addr = "129.215.197.11";
-          aliases = [ "eddie.i" ];
-        };
-        retiolum = {
-          via = internet;
-          addrs = [
-            config.krebs.hosts.eddie.nets.retiolum.ip4.addr
-            config.krebs.hosts.eddie.nets.retiolum.ip6.addr
-          ];
-          ip4.addr = "10.243.29.170";
-          aliases = [ "eddie.r" ];
-          tinc.pubkey = ''
-            -----BEGIN RSA PUBLIC KEY-----
-            MIICCgKCAgEAuRQphRlSIC/aqRTfvStPdJOJCx1ACeFIDEjRdgoxuu32qoBl7i6d
-            j7Voh+Msditf2a5+f0fVsNDaPnjPGfk0NkZBjmn+RZQDRXk0krpTNj2Vb6W5quTm
-            3yrjJMFJR9CU5khfppc47X+ir8bjn7RusWTFNEuDvUswHmRmnJHLS3Y+utOaRbCF
-            2hxpyxCn423gpsaBfORPEK8X90nPbuNpFDugWPnC+R45TpNmIf4qyKvfhd9OKrua
-            KNanGHG30xhBW/DclUwwWi8D44d94xFnIRVcG1O+Uto93WoUWZn90lI1qywSj5Aq
-            iWstBK4tc7VwvAj0UzPlaRYYPfFjOEkPQzj8xC6l/leJcgxkup252uo6m1njMx3t
-            6QWMgevjqosY22OZReZfIwb14aDWFKLTWs30J+zmWK4TjlRITdsOEKxlpODMbJAD
-            kfSoPwuwkWIzFhNOrFiD/NtKaRYmV8bTBCT3a9cvvObshJx13BP+IUFzBS1N1n/u
-            hJWYH5WFsQZn/8rHDwZGkS1zKPEaNoydjqCZNyJpJ5nhggyl6gpuD7wpXM/8tFay
-            pAjRP40+qRQLUWXmswV0hsZTOX1tvZs4f68y3WJ+GwCWw9HvvwmzYes5ayJrPsbJ
-            lyK301Jb42wGEsVWxu3Eo/PLtp8OdD+Wdh6o/ELcc0k/YCUGFMujUM8CAwEAAQ==
-            -----END RSA PUBLIC KEY-----
-          '';
-        };
-      };
-    };
     eve = {
       owner = config.krebs.users.mic92;
       nets = rec {
@@ -223,9 +173,6 @@ in {
         };
         retiolum = {
           via = internet;
-          addrs = [
-            config.krebs.hosts.eve.nets.retiolum.ip6.addr
-          ];
           aliases = [ "eve.r" ];
           tinc.pubkey = ''
             -----BEGIN RSA PUBLIC KEY-----
@@ -254,10 +201,6 @@ in {
     okelmann = {
       owner = config.krebs.users.mic92;
       nets.retiolum = {
-        addrs = [
-          config.krebs.hosts.okelmann.nets.retiolum.ip4.addr
-          config.krebs.hosts.okelmann.nets.retiolum.ip6.addr
-        ];
         ip4.addr = "10.243.29.190";
           aliases = [
             "okelmann.r"
@@ -277,10 +220,6 @@ in {
     aendernix = {
       owner = config.krebs.users.mic92;
       nets.retiolum = {
-        addrs = [
-          config.krebs.hosts.aendernix.nets.retiolum.ip4.addr
-          config.krebs.hosts.aendernix.nets.retiolum.ip6.addr
-        ];
         ip4.addr = "10.243.29.172";
         aliases = [
           "aendernix.r"
@@ -305,10 +244,6 @@ in {
     aenderpad = {
       owner = config.krebs.users.mic92;
       nets.retiolum = {
-        addrs = [
-          config.krebs.hosts.aenderpad.nets.retiolum.ip4.addr
-          config.krebs.hosts.aenderpad.nets.retiolum.ip6.addr
-        ];
         ip4.addr = "10.243.29.201";
         aliases = [
           "aendernix.r"
@@ -394,10 +329,6 @@ in {
         };
         retiolum = {
           via = internet;
-          addrs = [
-            config.krebs.hosts.martha.nets.retiolum.ip4.addr
-            config.krebs.hosts.martha.nets.retiolum.ip6.addr
-          ];
           ip4.addr = "10.243.29.179";
           aliases = [ "martha.r" ];
           tinc.pubkey = ''
@@ -454,10 +385,6 @@ in {
         };
         retiolum = {
           via = internet;
-          addrs = [
-            config.krebs.hosts.sauron.nets.retiolum.ip4.addr
-            config.krebs.hosts.sauron.nets.retiolum.ip6.addr
-          ];
           ip4.addr = "10.243.29.194";
           aliases = [ "sauron.r" ];
           tinc.pubkey = ''
@@ -484,10 +411,6 @@ in {
         };
         retiolum = {
           via = internet;
-          addrs = [
-            config.krebs.hosts.bill.nets.retiolum.ip4.addr
-            config.krebs.hosts.bill.nets.retiolum.ip6.addr
-          ];
           ip4.addr = "10.243.29.195";
           aliases = [ "bill.r" ];
           tinc.pubkey = ''
@@ -519,10 +442,6 @@ in {
         };
         retiolum = {
           via = internet;
-          addrs = [
-            config.krebs.hosts.nardole.nets.retiolum.ip4.addr
-            config.krebs.hosts.nardole.nets.retiolum.ip6.addr
-          ];
           ip4.addr = "10.243.29.173";
           aliases = [ "nardole.r" ];
           tinc.pubkey = ''
@@ -581,10 +500,6 @@ in {
         };
         retiolum = {
           via = internet;
-          addrs = [
-            config.krebs.hosts.rose.nets.retiolum.ip4.addr
-            config.krebs.hosts.rose.nets.retiolum.ip6.addr
-          ];
           ip4.addr = "10.243.29.178";
           aliases = [ "rose.r" ];
           tinc.pubkey = ''
@@ -610,7 +525,6 @@ in {
       owner = config.krebs.users.mic92;
       nets = {
         retiolum = {
-          ip4.addr = "10.243.29.168";
           aliases = [
             "turingmachine.r"
           ];
@@ -712,7 +626,6 @@ in {
         };
         retiolum = {
           via = internet;
-          ip4.addr = "10.243.29.185";
           aliases = [
             "eva.r"
             "loki.r"
@@ -743,10 +656,6 @@ in {
       owner = config.krebs.users.mic92;
       nets = rec {
         retiolum = {
-          addrs = [
-            config.krebs.hosts.doctor.nets.retiolum.ip4.addr
-            config.krebs.hosts.doctor.nets.retiolum.ip6.addr
-          ];
           ip4.addr = "10.243.29.186";
           aliases = [ "doctor.r" ];
           tinc.pubkey = ''
@@ -767,10 +676,6 @@ in {
       owner = config.krebs.users.mic92;
       nets = rec {
         retiolum = {
-          addrs = [
-            config.krebs.hosts.bernie.nets.retiolum.ip4.addr
-            config.krebs.hosts.bernie.nets.retiolum.ip6.addr
-          ];
           ip4.addr = "10.243.29.169";
           aliases = [ "bernie.r" ];
           tinc.pubkey = ''
@@ -805,10 +710,6 @@ in {
         };
         retiolum = {
           via = internet;
-          addrs = [
-            config.krebs.hosts.ryan.nets.retiolum.ip4.addr
-            config.krebs.hosts.ryan.nets.retiolum.ip6.addr
-          ];
           ip4.addr = "10.243.29.198";
           aliases = [ "ryan.r" ];
           tinc.pubkey = ''
@@ -837,10 +738,6 @@ in {
         };
         retiolum = {
           via = internet;
-          addrs = [
-            config.krebs.hosts.graham.nets.retiolum.ip4.addr
-            config.krebs.hosts.graham.nets.retiolum.ip6.addr
-          ];
           ip4.addr = "10.243.29.199";
           aliases = [ "graham.r" ];
           tinc.pubkey = ''
@@ -862,10 +759,6 @@ in {
       owner = config.krebs.users.mic92;
       nets = rec {
         retiolum = {
-          addrs = [
-            config.krebs.hosts.maurice.nets.retiolum.ip4.addr
-            config.krebs.hosts.maurice.nets.retiolum.ip6.addr
-          ];
           ip4.addr = "10.243.29.200";
           aliases = [ "maurice.r" ];
           tinc.pubkey = ''
@@ -882,5 +775,30 @@ in {
         };
       };
     };
+
+    mickey = {
+      owner = config.krebs.users.mic92;
+      nets = rec {
+        retiolum = {
+          aliases = [ "mickey.r" ];
+          tinc.pubkey = ''
+            Ed25519PublicKey = cE450gYxzp9kAzV5ytU9N7aV+WdnD7wQMjkPWV7r/bC
+            -----BEGIN RSA PUBLIC KEY-----
+            MIICCgKCAgEA7TwI3/tyl3z46Enr6p/0bpl5CpG6DZLxjAhsMcWBM+4xTL9s18IZ
+            2FGbyS3EyOBprMBQULrik1u0rfZ0AL8XdO6h+r1BD6XmlZtUu3FJaVeXrLBPGtC0
+            qqC0mZOj1ezTl3kC9/O7slU1/vuIRWiiRuvmvLnc6uWo+ShTl8fs0a3rY7/FsFVY
+            ZClf2M/5cJmeZpwy+PvgGmhSvjflO5+v+C+LvvhfVzoLw2zf8Gbi23ifS0uhhJt2
+            9ztGnmQg+n4+EWEN3XFS1XXHO2P2jyy1ss5NrN0JrO/1J519owHXxbo096MV12xr
+            azD6of8k0xHbfW4PW0/U1qzs9Ra1T54D+xtnyemLOyeCApwUy+bSg+XuqMz1Wy55
+            dci7cBguTIn+pnJqcf8lGSfWDSxlBiwrbXSPszlRQ6vO8MA2uciSmOKodKtNj4bQ
+            5IfdHHOHGAuuE+ZNt6owc/8QzQ3dVT+fVmTeN1PB4FmPmF5E2kOpe4NebZ0DhD+g
+            +g/bNO5FFlIy2M+LKauIXugAHlrVrxl4blfjVkb9xrfsSJHQl8/G/F9zMUAzUBv3
+            W8cVFn9mAw0FFaQljs9iha92we6Vs93v+ZvsmSG2MVOYBVwka4FJ7kjaABLFXcjN
+            RA8gQM/P3j1EmDvemlskWOoCLVELR40BtKdM9MFiGqxGMoNh3DvGWTECAwEAAQ==
+            -----END RSA PUBLIC KEY-----
+          '';
+        };
+      };
+    };
   };
 }
diff --git a/krebs/3modules/external/ssh/xkey.pub b/krebs/3modules/external/ssh/xkey.pub
new file mode 100644
index 000000000..cd09f06bb
--- /dev/null
+++ b/krebs/3modules/external/ssh/xkey.pub
@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIZFKgFcAEGXcsssJxDeUVvOTKD0U4LlT2Yw85+WmMTj
diff --git a/krebs/3modules/iptables.nix b/krebs/3modules/iptables.nix
index 6298a05a5..3bab13b0e 100644
--- a/krebs/3modules/iptables.nix
+++ b/krebs/3modules/iptables.nix
@@ -73,7 +73,7 @@ let
     };
   };
 
-  imp = {
+  imp = mkMerge ([{
     networking.firewall.enable = false;
 
     systemd.services.krebs-iptables = {
@@ -97,7 +97,41 @@ let
 
       unitConfig.DefaultDependencies = false;
     };
-  };
+  }] ++ compat);
+
+  compat = [
+    ({
+      krebs.iptables.tables.filter.INPUT.rules = map
+        (port: { predicate = "-p tcp --dport ${toString port}"; target = "ACCEPT"; })
+        config.networking.firewall.allowedTCPPorts;
+    })
+    ({
+      krebs.iptables.tables.filter.INPUT.rules = map
+        (port: { predicate = "-p udp --dport ${toString port}"; target = "ACCEPT"; })
+        config.networking.firewall.allowedUDPPorts;
+    })
+    ({
+      krebs.iptables.tables.filter.INPUT.rules = map
+        (portRange: { predicate = "-p tcp --dport ${toString port.from}:${toString port.to}"; target = "ACCEPT"; })
+        config.networking.firewall.allowedTCPPortRanges;
+    })
+    ({
+      krebs.iptables.tables.filter.INPUT.rules = map
+        (portRange: { predicate = "-p udp --dport ${toString port.from}:${toString port.to}"; target = "ACCEPT"; })
+        config.networking.firewall.allowedUDPPortRanges;
+    })
+    ({
+      krebs.iptables.tables.filter.INPUT.rules = flatten (mapAttrsToList
+        (interface: interfaceConfig: [
+          (map (port: { predicate = "-i ${interface} -p tcp --dport ${toString port}"; target = "ACCEPT"; }) interfaceConfig.allowedTCPPorts)
+          (map (port: { predicate = "-i ${interface} -p udp --dport ${toString port}"; target = "ACCEPT"; }) interfaceConfig.allowedUDPPorts)
+          (map (portRange: { predicate = "-i ${interface} -p tcp --dport ${toString port.from}:${toString port.to}"; target = "ACCEPT"; }) interfaceConfig.allowedTCPPortRanges)
+          (map (portRange: { predicate = "-i ${interface} -p udp --dport ${toString port.from}:${toString port.to}"; target = "ACCEPT"; }) interfaceConfig.allowedUDPPortRanges)
+        ])
+        config.networking.firewall.interfaces
+      );
+    })
+  ];
 
   #buildTable :: iptablesVersion -> iptablesAttrSet` -> str
   #todo: differentiate by iptables-version
diff --git a/krebs/3modules/krebs/default.nix b/krebs/3modules/krebs/default.nix
index f796f0323..1b5d903cb 100644
--- a/krebs/3modules/krebs/default.nix
+++ b/krebs/3modules/krebs/default.nix
@@ -76,10 +76,10 @@ in {
             "agenda.r"
             "build.r"
             "build.hotdog.r"
+            "ca.r"
             "cgit.hotdog.r"
             "irc.r"
             "wiki.r"
-            "wiki.hotdog.r"
           ];
           tinc.port = 0;
           tinc.pubkey = ''
diff --git a/krebs/3modules/realwallpaper.nix b/krebs/3modules/realwallpaper.nix
index 167afed2c..f2b34e8c4 100644
--- a/krebs/3modules/realwallpaper.nix
+++ b/krebs/3modules/realwallpaper.nix
@@ -19,7 +19,7 @@ let
 
     marker = mkOption {
       type = types.str;
-      default = "http://graph.r/marker.json";
+      default = "http://graph.r/network.json";
     };
 
     timerConfig = mkOption {
diff --git a/krebs/3modules/ssl.nix b/krebs/3modules/ssl.nix
new file mode 100644
index 000000000..3a9b5d329
--- /dev/null
+++ b/krebs/3modules/ssl.nix
@@ -0,0 +1,64 @@
+{ config, lib, pkgs, ... }: let
+  cfg = config.krebs.ssl;
+in {
+  options.krebs.ssl = {
+    rootCA = lib.mkOption {
+      type = lib.types.str;
+      readOnly = true;
+      default = ''
+        -----BEGIN CERTIFICATE-----
+        MIIC0jCCAjugAwIBAgIJAKeARo6lDD0YMA0GCSqGSIb3DQEBBQUAMIGBMQswCQYD
+        VQQGEwJaWjESMBAGA1UECAwJc3RhdGVsZXNzMRAwDgYDVQQKDAdLcmVic2NvMQsw
+        CQYDVQQLDAJLTTEWMBQGA1UEAwwNS3JlYnMgUm9vdCBDQTEnMCUGCSqGSIb3DQEJ
+        ARYYcm9vdC1jYUBzeW50YXgtZmVobGVyLmRlMB4XDTE0MDYxMTA4NTMwNloXDTM5
+        MDIwMTA4NTMwNlowgYExCzAJBgNVBAYTAlpaMRIwEAYDVQQIDAlzdGF0ZWxlc3Mx
+        EDAOBgNVBAoMB0tyZWJzY28xCzAJBgNVBAsMAktNMRYwFAYDVQQDDA1LcmVicyBS
+        b290IENBMScwJQYJKoZIhvcNAQkBFhhyb290LWNhQHN5bnRheC1mZWhsZXIuZGUw
+        gZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMs/WNyeQziccllLqom7bfCjlh6/
+        /qx9p6UOqpw96YOOT3sh/mNSBLyNxIUJbWsU7dN5hT7HkR7GwzpfKDtudd9qiZeU
+        QNYQ+OL0HdOnApjdPqdspZfKxKTXyC1T1vJlaODsM1RBrjLK9RUcQZeNhgg3iM9B
+        HptOCrMI2fjCdZuVAgMBAAGjUDBOMB0GA1UdDgQWBBSKeq01+rAwp7yAXwzlwZBo
+        3EGVLzAfBgNVHSMEGDAWgBSKeq01+rAwp7yAXwzlwZBo3EGVLzAMBgNVHRMEBTAD
+        AQH/MA0GCSqGSIb3DQEBBQUAA4GBAIWIffZuQ43ddY2/ZnjAxPCRpM3AjoKIwEj9
+        GZuLJJ1sB9+/PAPmRrpmUniRkPLD4gtmolDVuoLDNAT9os7/v90yg5dOuga33Ese
+        725musUbhEoQE1A1oVHrexBs2sQOplxHKsVXoYJp2/trQdqvaNaEKc3EeVnzFC63
+        80WiO952
+        -----END CERTIFICATE-----
+      '';
+    };
+    intermediateCA = lib.mkOption {
+      type = lib.types.str;
+      readOnly = true;
+      default = builtins.readFile ../6assets/krebsAcmeCA.crt;
+    };
+    acmeURL = lib.mkOption {
+      type = lib.types.str;
+      readOnly = true;
+      default = "https://ca.r/acme/acme/directory";
+    };
+    trustRoot = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = ''
+        whether to trust the krebs root CA.
+        This implies that krebs can forge a certficate for every domain
+      '';
+    };
+    trustIntermediate = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = ''
+        whether to trust the krebs ACME CA.
+        this only trusts the intermediate cert for .w and .r domains
+      '';
+    };
+  };
+  config = lib.mkMerge [
+    (lib.mkIf cfg.trustRoot {
+      security.pki.certificates = [ cfg.rootCA ];
+    })
+    (lib.mkIf cfg.trustIntermediate {
+      security.pki.certificates = [ cfg.intermediateCA ];
+    })
+  ];
+}
diff --git a/krebs/3modules/tinc.nix b/krebs/3modules/tinc.nix
index 898b5e8c3..3d0cc8fb4 100644
--- a/krebs/3modules/tinc.nix
+++ b/krebs/3modules/tinc.nix
@@ -17,7 +17,6 @@ let
       in {
 
         enable = mkEnableOption "krebs.tinc.${netname}" // { default = true; };
-        enableLegacy = mkEnableOption "/etc/tinc/${netname}";
 
         confDir = mkOption {
           type = types.package;
@@ -102,7 +101,7 @@ let
 
         tincPackage = mkOption {
           type = types.package;
-          default = pkgs.tinc;
+          default = pkgs.tinc_pre;
           description = "Tincd package to use.";
         };
 
@@ -263,9 +262,9 @@ let
     ) config.krebs.tinc;
 
     environment.etc = mapAttrs' (netname: cfg:
-      nameValuePair "tinc/${netname}" (mkIf cfg.enableLegacy {
+      nameValuePair "tinc/${netname}" {
         source = cfg.confDir;
-      })
+      }
     ) config.krebs.tinc;
 
     systemd.services = mapAttrs (netname: cfg:
@@ -287,9 +286,12 @@ let
         ];
         wantedBy = [ "multi-user.target" ];
         path = [ tinc iproute ];
+        reloadIfChanged = true;
+        restartTriggers = [ cfg.confDir ];
         serviceConfig = rec {
           Restart = "always";
-          ExecStart = "${tinc}/sbin/tincd -c ${cfg.confDir} -d 0 -U ${cfg.user.name} -D --pidfile=/var/run/tinc.${SyslogIdentifier}.pid";
+          ExecStart = "${tinc}/sbin/tincd -c /etc/tinc/${netname} -d 0 -U ${cfg.user.name} -D --pidfile=/var/run/tinc.${SyslogIdentifier}.pid";
+          ExecReload = "${tinc}/sbin/tinc -n ${netname} reload";
           SyslogIdentifier = netname;
         };
       }
diff --git a/krebs/5pkgs/haskell/brockman/default.nix b/krebs/5pkgs/haskell/brockman/default.nix
index de41d1886..01b7a0570 100644
--- a/krebs/5pkgs/haskell/brockman/default.nix
+++ b/krebs/5pkgs/haskell/brockman/default.nix
@@ -7,12 +7,12 @@
 }:
 mkDerivation rec {
   pname = "brockman";
-  version = "4.0.1";
+  version = "4.0.2";
   src = fetchFromGitHub {
     owner = "kmein";
     repo = "brockman";
     rev = version;
-    sha256 = "0hppgban8hfyhn4c8qgm8j7ml6jaa35pjgrv3k3q27ln71wnr8rz";
+    sha256 = "sha256-EKXKhGdIJRbRklD5zxJNGhOxqPzog4f9NMXo/c8iBGc=";
   };
   isLibrary = false;
   isExecutable = true;
diff --git a/krebs/5pkgs/simple/buildbot-classic/default.nix b/krebs/5pkgs/simple/buildbot-classic/default.nix
deleted file mode 100644
index 5e075f1a1..000000000
--- a/krebs/5pkgs/simple/buildbot-classic/default.nix
+++ /dev/null
@@ -1,34 +0,0 @@
-{ pkgs, fetchFromGitHub, python2Packages, git, ... }: let
-
-  # we need the old sqlparse since the new one is python2 incompatible
-  sqlparse = python2Packages.callPackage ./sqlparse.nix {};
-
-in python2Packages.buildPythonApplication rec {
-  name = "buildbot-classic-${version}";
-  version = "0.8.18";
-  namePrefix = "";
-  patches = [];
-
-  src = fetchFromGitHub {
-    owner = "krebs";
-    repo = "buildbot-classic";
-    rev = version;
-    sha256 = "0b4y3n9zd2gdy8xwk1vpvs4n9fbg72vi8mx4ydgijwngcmdqkjmq";
-  };
-  postUnpack = "sourceRoot=\${sourceRoot}/master";
-
-  propagatedBuildInputs = [
-    python2Packages.jinja2
-    python2Packages.twisted
-    python2Packages.dateutil
-    (python2Packages.sqlalchemy_migrate.override { sqlparse = sqlparse; })
-    python2Packages.pysqlite
-    pkgs.coreutils
-  ];
-  doCheck = false;
-  postInstall = ''
-    mkdir -p "$out/share/man/man1"
-    cp docs/buildbot.1 "$out/share/man/man1"
-  '';
-}
-
diff --git a/krebs/5pkgs/simple/buildbot-classic/sqlparse.nix b/krebs/5pkgs/simple/buildbot-classic/sqlparse.nix
deleted file mode 100644
index 2cbb51845..000000000
--- a/krebs/5pkgs/simple/buildbot-classic/sqlparse.nix
+++ /dev/null
@@ -1,34 +0,0 @@
-{ lib
-, buildPythonPackage
-, fetchPypi
-, pytest
-, isPy3k
-}:
-
-buildPythonPackage rec {
-  pname = "sqlparse";
-  version = "0.3.1";
-
-  src = fetchPypi {
-    inherit pname version;
-    sha256 = "e162203737712307dfe78860cc56c8da8a852ab2ee33750e33aeadf38d12c548";
-  };
-
-  checkInputs = [ pytest ];
-  checkPhase = ''
-    py.test
-  '';
-
-  # Package supports 3.x, but tests are clearly 2.x only.
-  doCheck = !isPy3k;
-
-  meta = with lib; {
-    description = "Non-validating SQL parser for Python";
-    longDescription = ''
-      Provides support for parsing, splitting and formatting SQL statements.
-    '';
-    homepage = "https://github.com/andialbrecht/sqlparse";
-    license = licenses.bsd3;
-  };
-
-}
diff --git a/krebs/5pkgs/simple/generate-krebs-intermediate-ca/default.nix b/krebs/5pkgs/simple/generate-krebs-intermediate-ca/default.nix
new file mode 100644
index 000000000..5055a78aa
--- /dev/null
+++ b/krebs/5pkgs/simple/generate-krebs-intermediate-ca/default.nix
@@ -0,0 +1,30 @@
+{ pkgs }:
+pkgs.writers.writeDashBin "generate-intermediate-ca" ''
+  TMPDIR=$(mktemp -d)
+  trap "rm -rf $TMPDIR;" INT TERM EXIT
+  mkdir -p "$TMPDIR/krebs"
+  brain show ca/ca.key > "$TMPDIR/krebs/ca.key"
+  brain show ca/ca.crt > "$TMPDIR/krebs/ca.crt"
+  export STEPPATH="$TMPDIR/step"
+  cat << EOF > "$TMPDIR/intermediate.tpl"
+  {
+      "subject": {{ toJson .Subject }},
+      "keyUsage": ["certSign", "crlSign"],
+      "basicConstraints": {
+          "isCA": true,
+          "maxPathLen": 0
+      },
+      "nameConstraints": {
+          "critical": true,
+          "permittedDNSDomains": ["r" ,"w"]
+      }
+  }
+  EOF
+
+  ${pkgs.step-cli}/bin/step certificate create "Krebs ACME CA" intermediate_ca.crt intermediate_ca.key \
+    --template "$TMPDIR/intermediate.tpl" \
+    --not-after 8760h \
+    --ca "$TMPDIR/krebs/ca.crt" \
+    --ca-key "$TMPDIR/krebs/ca.key" \
+    --no-password --insecure
+''
diff --git a/krebs/5pkgs/simple/rss-bridge/default.nix b/krebs/5pkgs/simple/rss-bridge/default.nix
index e0a927a1a..2ad322d48 100644
--- a/krebs/5pkgs/simple/rss-bridge/default.nix
+++ b/krebs/5pkgs/simple/rss-bridge/default.nix
@@ -2,13 +2,13 @@
 
 stdenv.mkDerivation rec {
   pname = "rss-bridge";
-  version = "unstable-2021-04-20";
+  version = "unstable-2021-12-02";
 
   src = fetchFromGitHub {
     owner = "RSS-Bridge";
     repo = "rss-bridge";
-    rev = "716f5ddc0e20c10cb77ded46380cc376913a92fd";
-    sha256 = "17aqmj7rz0ysk8nj4kbjvnsjdm47d0xsypfygzzk2vagxfz5w3p8";
+    rev = "f469489b569d22fb5edbd13c6e5f5abf2a4ee186";
+    sha256 = "sha256-LyxcycXbOFZR0mMDMUqAOjWrHIE2ftxkAYUGBbcQF5k==";
   };
 
   patchPhase = ''
diff --git a/krebs/6assets/krebsAcmeCA.crt b/krebs/6assets/krebsAcmeCA.crt
new file mode 100644
index 000000000..1cd5aed0b
--- /dev/null
+++ b/krebs/6assets/krebsAcmeCA.crt
@@ -0,0 +1,15 @@
+-----BEGIN CERTIFICATE-----
+MIICWTCCAcKgAwIBAgIQbAfVX2J0VIzhEYSPVAB4SzANBgkqhkiG9w0BAQsFADCB
+gTELMAkGA1UEBhMCWloxEjAQBgNVBAgMCXN0YXRlbGVzczEQMA4GA1UECgwHS3Jl
+YnNjbzELMAkGA1UECwwCS00xFjAUBgNVBAMMDUtyZWJzIFJvb3QgQ0ExJzAlBgkq
+hkiG9w0BCQEWGHJvb3QtY2FAc3ludGF4LWZlaGxlci5kZTAeFw0yMTEyMTAwODQ5
+MDZaFw0yMjEyMTAwODQ5MDZaMBgxFjAUBgNVBAMTDUtyZWJzIEFDTUUgQ0EwWTAT
+BgcqhkjOPQIBBggqhkjOPQMBBwNCAATL8dNO7ajNe60Km7wHrG06tCUj5kQKWsrQ
+Ay7KX8zO+RwQpYhd/i4bqpeGkGWh8uHLZ+164FlZaLgHO10DRja5o4GAMH4wDgYD
+VR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0OBBYEFMt9yJED
+mPRhXsrNZ0x+GtzjdnTLMB8GA1UdIwQYMBaAFIp6rTX6sDCnvIBfDOXBkGjcQZUv
+MBgGA1UdHgEB/wQOMAygCjADggFyMAOCAXcwDQYJKoZIhvcNAQELBQADgYEANo/2
+teIuEsniwxVdqu+ukjqOXHIkBK7F91+G7BuDjBlx2U96v1MwsmT4D9upajERnOOD
+tLx990Sj4t3avRTpytt+qLeIMIxt62YksUXVjDWndqaDcEUat5ZVEQsZ0ZmjOHrA
+BaB65eU0xhJWKAZdk55GqHEFz3Ym4rx7WUaomzk=
+-----END CERTIFICATE-----
diff --git a/krebs/nixpkgs-unstable.json b/krebs/nixpkgs-unstable.json
index 2e6bc6fe7..0e234d26a 100644
--- a/krebs/nixpkgs-unstable.json
+++ b/krebs/nixpkgs-unstable.json
@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/NixOS/nixpkgs",
-  "rev": "bc5d68306b40b8522ffb69ba6cff91898c2fbbff",
-  "date": "2021-12-07T01:07:01+09:00",
-  "path": "/nix/store/3z2f4r7kfkma94zwf083x7cvq8nypw42-nixpkgs",
-  "sha256": "0c5qjrmh1k2zr15x2i9cp6n1r2pvrlk7hdyfvrwzpk963gc9ssmz",
+  "rev": "b0bf5f888d377dd2f36d90340df6dc9f035aaada",
+  "date": "2021-12-12T23:14:25+01:00",
+  "path": "/nix/store/cdc72gqrly2dn2x1nb8xhnmw1q68f8zr-nixpkgs",
+  "sha256": "0l123s468r9h706grqkzf0x077r4hy6zcz529xxfiqxsh1ddx5xb",
   "fetchLFS": false,
   "fetchSubmodules": false,
   "deepClone": false,
diff --git a/krebs/nixpkgs.json b/krebs/nixpkgs.json
index 603544123..f90e6b08c 100644
--- a/krebs/nixpkgs.json
+++ b/krebs/nixpkgs.json
@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/NixOS/nixpkgs",
-  "rev": "1bd4bbd49bef217a3d1adea43498270d6e779d65",
-  "date": "2021-12-07T15:25:15+01:00",
-  "path": "/nix/store/4gssny5dsr4w6p5v7mni9xl7xs5fkv3c-nixpkgs",
-  "sha256": "1fx6nqz8x9biwlwsnh67z5qz0fmrdgr01yvmdw2cw9xjx8hyss3s",
+  "rev": "573095944e7c1d58d30fc679c81af63668b54056",
+  "date": "2021-12-10T10:33:46-08:00",
+  "path": "/nix/store/c0bvhzf1xsjrmzrda8jasa1da76x0zyk-nixpkgs",
+  "sha256": "07s5cwhskqvy82b4rld9b14ljc0013pig23i3jx3l3f957rk95pg",
   "fetchLFS": false,
   "fetchSubmodules": false,
   "deepClone": false,
diff --git a/lass/2configs/retiolum.nix b/lass/2configs/retiolum.nix
index 9932f8172..f900bc28e 100644
--- a/lass/2configs/retiolum.nix
+++ b/lass/2configs/retiolum.nix
@@ -14,7 +14,6 @@
   };
 
   krebs.tinc.retiolum = {
-    enableLegacy = true;
     enable = true;
     connectTo = [
       "prism"
diff --git a/lass/2configs/websites/util.nix b/lass/2configs/websites/util.nix
index bffa1036b..b6765037c 100644
--- a/lass/2configs/websites/util.nix
+++ b/lass/2configs/websites/util.nix
@@ -227,6 +227,7 @@ rec {
       services.phpfpm.pools."${domain}" = {
         user = "nginx";
         group = "nginx";
+        phpPackage = pkgs.php74;
         extraConfig = ''
           listen = /srv/http/${domain}/phpfpm.pool
           pm = dynamic
diff --git a/lass/krops.nix b/lass/krops.nix
index 769112706..4abd010e1 100644
--- a/lass/krops.nix
+++ b/lass/krops.nix
@@ -46,7 +46,9 @@ in {
         -f '<nixpkgs/nixos>' config.system.build.toplevel \
         -o "$outDir/out"
 
-      $outDir/out/bin/switch-to-configuration switch
+      nix-env -p /nix/var/nix/profiles/system --set "$outDir/out"
+
+      "$outDir/out/bin/switch-to-configuration" switch
     '';
     source = source { test = false; };
     allocateTTY = true;
diff --git a/makefu/1systems/omo/config.nix b/makefu/1systems/omo/config.nix
index 0b4aaacb3..3a216ea76 100644
--- a/makefu/1systems/omo/config.nix
+++ b/makefu/1systems/omo/config.nix
@@ -94,7 +94,7 @@ in {
       <stockholm/makefu/2configs/virtualisation/docker.nix>
       <stockholm/makefu/2configs/bluetooth-mpd.nix>
 
-      <stockholm/makefu/2configs/home/airsonic.nix>
+      <stockholm/makefu/2configs/home/music.nix>
       <stockholm/makefu/2configs/home/photoprism.nix>
       # <stockholm/makefu/2configs/home/metube.nix>
       <stockholm/makefu/2configs/home/ham>
diff --git a/makefu/2configs/bgt/social-to-irc.nix b/makefu/2configs/bgt/social-to-irc.nix
index e0898193b..9d9640a9f 100644
--- a/makefu/2configs/bgt/social-to-irc.nix
+++ b/makefu/2configs/bgt/social-to-irc.nix
@@ -1,12 +1,6 @@
 { pkgs, ... }:
 {
   systemd.services.brockman.environment."BROCKMAN_LOG_LEVEL" = "DEBUG";
-  systemd.services.restart-brockman = {
-    after = [ "brockman.service" ];
-    wantedBy = [ "multi-user.target" ];
-    startAt = "daily";
-    script = "${pkgs.systemd}/bin/systemctl try-restart brockman.service";
-  };
   krebs.brockman = {
     enable = true;
     config = {
@@ -34,7 +28,7 @@
         bgt-twitter = {
           feed = "http://rss.makefu.r/?action=display&bridge=Twitter&context=By+username&u=binaergewitter&format=Atom";
           #extraChannels = [ "#binaergewitter" ];
-          delay = 180;
+          delay = 280;
         };
       };
     };
diff --git a/makefu/2configs/binary-cache/server.nix b/makefu/2configs/binary-cache/server.nix
index 2e05fd52e..c1ae16e29 100644
--- a/makefu/2configs/binary-cache/server.nix
+++ b/makefu/2configs/binary-cache/server.nix
@@ -6,22 +6,9 @@
   services.nix-serve = {
     enable = true;
     port = 5001;
-    secretKeyFile = config.krebs.secret.files.nix-serve-key.path;
+    secretKeyFile = toString <secrets> + "/nix-serve.key";
   };
 
-  systemd.services.nix-serve = {
-    after = [
-      config.krebs.secret.files.nix-serve-key.service
-    ];
-    partOf = [
-      config.krebs.secret.files.nix-serve-key.service
-    ];
-  };
-  krebs.secret.files.nix-serve-key = {
-    path = "/run/secret/nix-serve.key";
-    owner.name = "nix-serve";
-    source-path = toString <secrets> + "/nix-serve.key";
-  };
   services.nginx = {
     enable = true;
     virtualHosts."cache.euer.krebsco.de" = {
diff --git a/makefu/2configs/bitwarden.nix b/makefu/2configs/bitwarden.nix
index 7e317e596..92c1c4e0e 100644
--- a/makefu/2configs/bitwarden.nix
+++ b/makefu/2configs/bitwarden.nix
@@ -2,7 +2,7 @@
 let
   port = 8812;
 in {
-  services.bitwarden_rs = {
+  services.vaultwarden = {
     enable = true;
     dbBackend = "postgresql";
     config.signups_allowed = false;
@@ -13,17 +13,15 @@ in {
     config.websocket_enabled = true;
   };
 
-  systemd.services.bitwarden_rs.after = [ "postgresql.service" ];
+  systemd.services.vaultwarden.after = [ "postgresql.service" ];
 
   services.postgresql = {
     enable = true;
     ensureDatabases = [ "bitwarden" ];
-    ensureUsers = [ { name = "bitwarden_rs"; ensurePermissions."DATABASE bitwarden" = "ALL PRIVILEGES"; } ];
-    #initialScript = pkgs.writeText "postgresql-init.sql" ''
-    #  CREATE DATABASE bitwarden;
-    #  CREATE USER bitwardenuser WITH PASSWORD '${dbPassword}';
-    #  GRANT ALL PRIVILEGES ON DATABASE bitwarden TO bitwardenuser;
-    #'';
+    ensureUsers = [
+      { name = "bitwarden_rs"; ensurePermissions."DATABASE bitwarden" = "ALL PRIVILEGES"; } 
+      { name = "vaultwarden"; ensurePermissions."DATABASE bitwarden" = "ALL PRIVILEGES"; } 
+    ];
   };
 
   services.nginx.virtualHosts."bw.euer.krebsco.de" ={
diff --git a/makefu/2configs/dcpp/hub.nix b/makefu/2configs/dcpp/hub.nix
index b8ca49b74..5a88f5ef8 100644
--- a/makefu/2configs/dcpp/hub.nix
+++ b/makefu/2configs/dcpp/hub.nix
@@ -63,8 +63,11 @@ in {
   networking.firewall.extraCommands = ''
     iptables -A PREROUTING -t nat -i ${ext-if} -p tcp --dport 411 -j REDIRECT --to-port 1511
   '';
-  systemd.services.uhub.serviceConfig = {
+  systemd.services.uhub-home.serviceConfig = {
     PrivateTmp = true;
+    DynamicUser = lib.mkForce false;
+    User = "uhub";
+    WorkingDirectory = uhubDir;
     PermissionsStartOnly = true;
     ExecStartPre = pkgs.writeDash "uhub-pre" ''
       cp -f ${toString <secrets/wildcard.krebsco.de.crt>} ${uhubDir}/uhub.crt
@@ -86,6 +89,7 @@ in {
     group = "uhub";
   };
   users.groups.uhub = {};
+
   services.uhub.home = {
     enable = true;
     enableTLS = true;
@@ -103,13 +107,12 @@ in {
       }
       {
         plugin = "${pkgs.uhub}/plugins/mod_welcome.so";
-        settings.motd = "shareit";
-        settings.rules = "1. Don't be an asshole";
+        settings.motd = toString (pkgs.writeText "motd" "shareit");
+        settings.rules = toString (pkgs.writeText "rules" "1. Don't be an asshole");
       }
       {
-        plugin = "${pkgs.uhub}/plugins/mod_history.so";
-        settings.motd = "shareit";
-        settings.rules = "1. Don't be an asshole";
+        plugin = "${pkgs.uhub}/plugins/mod_chat_history.so";
+        settings = {};
       }
     ];
   };
diff --git a/makefu/2configs/deployment/owncloud.nix b/makefu/2configs/deployment/owncloud.nix
index 0593cf7fc..610ba75fe 100644
--- a/makefu/2configs/deployment/owncloud.nix
+++ b/makefu/2configs/deployment/owncloud.nix
@@ -49,7 +49,7 @@ in {
 
   services.nextcloud = {
     enable = true;
-    package = pkgs.nextcloud21;
+    package = pkgs.nextcloud22;
     hostName = "o.euer.krebsco.de";
     # Use HTTPS for links
     https = true;
diff --git a/makefu/2configs/home/ham/mqtt.nix b/makefu/2configs/home/ham/mqtt.nix
index c90afff4a..5e668e7a0 100644
--- a/makefu/2configs/home/ham/mqtt.nix
+++ b/makefu/2configs/home/ham/mqtt.nix
@@ -12,15 +12,15 @@
         omitPasswordAuth = false;
         users.sensor = {
           hashedPassword = "$6$2DXU7W1bvqXPqxkF$vtdz5KTd/T09hmoc9LjgEGFjvpwQbQth6vlVcr5hJNLgcBHv4U03YCKC8TKXbmQAa8xiJ76xJIg25kcL+KI3tg==";
-          acl = [ "topic readwrite #" ];
+          acl = [ "readwrite #" ];
         };
         users.hass = {
           hashedPassword = "$6$SHuYGrE5kPSUc/hu$EomZ0KBy+vkxLt/6eJkrSBjYblCCeMjhDfUd2mwqXYJ4XsP8hGmZ59mMlmBCd3AvlFYQxb4DT/j3TYlrqo7cDA==";
-          acl = [ "topic readwrite #" ];
+          acl = [ "readwrite #" ];
         };
         users.stats = {
           hashedPassword = "$6$j4H7KXD/YZgvgNmL$8e9sUKRXowDqJLOVgzCdDrvDE3+4dGgU6AngfAeN/rleGOgaMhee2Mbg2KS5TC1TOW3tYbk9NhjLYtjBgfRkoA==";
-          acl = [ "topic read #" ];
+          acl = [ "read #" ];
         };
         settings = {
           allow_anonymous = false;
diff --git a/makefu/2configs/home/airsonic.nix b/makefu/2configs/home/music.nix
index c6112be26..59f6d9170 100644
--- a/makefu/2configs/home/airsonic.nix
+++ b/makefu/2configs/home/music.nix
@@ -1,19 +1,21 @@
 { config, ... }:
 let
   internal-ip = "192.168.111.11";
-  port = 4040;
+  port = 4533;
 in
 {
-  # networking.firewall.allowedTCPPorts = [ 4040 ];
-  services.airsonic = {
-    enable = true;
-    listenAddress = "0.0.0.0";
-    inherit port;
+  services.navidrome.enable = true;
+  services.navidrome.settings = {
+    MusicFolder = "/media/cryptX/music";
+    Address = "0.0.0.0";
   };
-  state = [ config.services.airsonic.home ];
-  services.nginx.virtualHosts."airsonic" = {
+
+  state = [ "/var/lib/navidrome" ];
+  # networking.firewall.allowedTCPPorts = [ 4040 ];
+  # state = [ config.services.airsonic.home ];
+  services.nginx.virtualHosts."navidrome" = {
     serverAliases = [
-              "airsonic.lan"
+              "navidrome.lan"
       "music"  "music.lan"
       "musik" "musik.lan"
     ];
diff --git a/makefu/2configs/systemdultras/ircbot.nix b/makefu/2configs/systemdultras/ircbot.nix
index 65583b9ac..df9741d9c 100644
--- a/makefu/2configs/systemdultras/ircbot.nix
+++ b/makefu/2configs/systemdultras/ircbot.nix
@@ -7,13 +7,6 @@
     virtualHost = "rss.makefu.r";
   };
 
-  systemd.services.restart-brockman = {
-    after = [ "brockman.service" ];
-    wantedBy = [ "multi-user.target" ];
-    startAt = "daily";
-    script = "${pkgs.systemd}/bin/systemctl try-restart brockman.service";
-  };
-
   krebs.brockman = {
     enable = true;
     config = {
@@ -27,11 +20,11 @@
       bots = {
         r-systemdultras-rss = {
           feed = "https://www.reddit.com/r/systemdultras/.rss";
-          delay = 136;
+          delay = 236;
         };
         r-systemd-rss = {
           feed = "https://www.reddit.com/r/systemd/.rss";
-          delay = 172;
+          delay = 272;
         };
         r-pid_eins-twitter = {
           feed = "http://rss.makefu.r/?action=display&bridge=Twitter&context=By+username&u=pid_eins&format=Atom";