Merge remote-tracking branch 'cd/master'

.rsync-filter

@@ -0,0 +1,2 @@
+- /.git
+- /.graveyard

Makefile

@@ -24,7 +24,27 @@ else ifdef system
 deploy infest:;@
 	export get=krebs.$@
 	export filter=json
-	make -s eval | sh
+	script=$$(make -s eval)
+	echo "$$script" | sh
+
+.PHONY: deploy2
+ifdef target
+deploy2: export target-host = $(target)
+else
+deploy2: export target-host = $(system)
+endif
+deploy2:;@
+	target=$${target-$$system}
+	result=$$(nix-instantiate \
+			--json \
+			--eval \
+			krebs/populate.nix \
+			--arg source 'with (import ~/stockholm {}).users.$(LOGNAME).$(system).config.krebs.build; assert source-version == 2; source' \
+			--argstr target-host "$$target" \
+			--argstr target-path /var/src)
+	script=$$(echo "$$result" | jq -r .)
+	echo "$$script" | sh
+	ssh root@$$target nixos-rebuild switch -I /var/src
 
 .PHONY: eval
 eval:

krebs/3modules/build.nix

@@ -28,48 +28,83 @@ let
       type = types.user;
     };
 
-    options.krebs.build.source.dir = mkOption {
-      type = let
-        default-host = config.krebs.current.host;
-      in types.attrsOf (types.submodule ({ config, ... }: {
-        options = {
-          host = mkOption {
-            type = types.host;
-            default = default-host;
-          };
-          path = mkOption {
-            type = types.str;
-          };
-          target-path = mkOption {
-            type = types.str;
-            default = "/root/${config._module.args.name}";
-          };
-          url = mkOption {
-            type = types.str;
-            default = "file://${config.host.name}${config.path}";
-          };
-        };
-      }));
-      default = {};
+    options.krebs.build.source-version = mkOption {
+      type = types.enum [ 1 2 ];
+      default = 1;
     };
 
-    options.krebs.build.source.git = mkOption {
-      type = with types; attrsOf (submodule ({ config, ... }: {
-        options = {
-          url = mkOption {
-            type = types.str; # TODO must be shell safe
-          };
-          rev = mkOption {
-            type = types.str;
-          };
-          target-path = mkOption {
-            type = types.str;
-            default = "/root/${config._module.args.name}";
+    options.krebs.build.source = getAttr "v${toString config.krebs.build.source-version}" {
+      v1 = {
+        dir = mkOption {
+          type = let
+            default-host = config.krebs.current.host;
+          in types.attrsOf (types.submodule ({ config, ... }: {
+            options = {
+              host = mkOption {
+                type = types.host;
+                default = default-host;
+              };
+              path = mkOption {
+                type = types.str;
+              };
+              target-path = mkOption {
+                type = types.str;
+                default = "/root/${config._module.args.name}";
+              };
+              url = mkOption {
+                type = types.str;
+                default = "file://${config.host.name}${config.path}";
+              };
+            };
+          }));
+          default = {};
+        };
+
+        git = mkOption {
+          type = with types; attrsOf (submodule ({ config, ... }: {
+            options = {
+              url = mkOption {
+                type = types.str; # TODO must be shell safe
+              };
+              rev = mkOption {
+                type = types.str;
+              };
+              target-path = mkOption {
+                type = types.str;
+                default = "/root/${config._module.args.name}";
+              };
+            };
+          }));
+          default = {};
+        };
+      };
+
+      v2 = let
+        raw = types.either types.str types.path;
+        url = types.submodule {
+          options = {
+            url = mkOption {
+              type = types.str;
+            };
+            rev = mkOption {
+              type = types.str;
+            };
+            dev = mkOption {
+              type = types.str;
+            };
           };
         };
-      }));
-      default = {};
+      in mkOption {
+        type = types.attrsOf (types.either types.str url);
+        apply = let f = mapAttrs (_: value: {
+          string = value;
+          path = toString value;
+          set = f value;
+        }.${typeOf value}); in f;
+        default = {};
+      };
     };
+
   };
 
 in out

krebs/3modules/urlwatch.nix

@@ -54,6 +54,10 @@ let
       example = [
         https://nixos.org/channels/nixos-unstable/git-revision
       ];
+      apply = map (x: getAttr (typeOf x) {
+        set = x;
+        string.url = x;
+      });
     };
     verbose = mkOption {
       type = types.bool;
@@ -64,7 +68,40 @@ let
     };
   };
 
-  urlsFile = toFile "urls" (concatStringsSep "\n" cfg.urls);
+  urlsFile = toFile "urls" (concatMapStringsSep "\n---\n" toJSON cfg.urls);
+
+  configFile = toFile "urlwatch.yaml" (toJSON {
+    display = {
+      error = true;
+      new = true;
+      unchanged = false;
+    };
+    report = {
+      email = {
+        enabled = false;
+        from = "";
+        html = false;
+        smtp = {
+          host = "localhost";
+          keyring = true;
+          port = 25;
+          starttls = true;
+        };
+        subject = "{count} changes: {jobs}";
+        to = "";
+      };
+      html.diff = "unified";
+      stdout = {
+        color = true;
+        enabled = true;
+      };
+      text = {
+        details = true;
+        footer = true;
+        line_length = 75;
+      };
+    };
+  });
 
   imp = {
     systemd.timers.urlwatch = {
@@ -109,10 +146,15 @@ let
           from=${escapeShellArg cfg.from}
           mailto=${escapeShellArg cfg.mailto}
           urlsFile=${escapeShellArg urlsFile}
+          configFile=${escapeShellArg configFile}
 
           cd /tmp
 
-          urlwatch -e ${optionalString cfg.verbose "-v"} --urls="$urlsFile" > changes || :
+          urlwatch \
+              ${optionalString cfg.verbose "-v"} \
+              --urls="$urlsFile" \
+              --config="$configFile" \
+            > changes || :
 
           if test -s changes; then
             date=$(date -R)

krebs/5pkgs/lentil/default.nix

@@ -1,15 +0,0 @@
-{ pkgs, ... }:
-
-(pkgs.haskellngPackages.override {
-  overrides = self: super: {
-    lentil = super.lentil.override {
-      mkDerivation = (attrs: self.mkDerivation (attrs // {
-        version = "0.1.3.0";
-        sha256 = "0xa59avh0bvfg69xh9p5b8dppfhx29mvfq8v41sk9j7qbcnzjivg";
-        patches = [
-          ./syntaxes.patch
-        ];
-      }));
-    };
-  };
-}).lentil

krebs/5pkgs/lentil/syntaxes.patch

@@ -1,11 +0,0 @@
-diff -rN -u old-lentil/src/Lentil/Parse/Syntaxes.hs new-lentil/src/Lentil/Parse/Syntaxes.hs
---- old-lentil/src/Lentil/Parse/Syntaxes.hs	2015-07-20 23:15:38.600539779 +0200
-+++ new-lentil/src/Lentil/Parse/Syntaxes.hs	2015-07-20 23:15:38.600539779 +0200
-@@ -30,6 +30,7 @@
-               | ext `elem` [".pas", ".pp", ".inc"] = Just pascal
-               | ext `elem` [".py"]                 = Just python
-               | ext `elem` [".rb"]                 = Just ruby
-+              | ext `elem` [".nix"]                = Just perl -- Nix
-               | ext `elem` [".pl", ".pm", ".t"]    = Just perl
-               | ext `elem` [".sh"]                 = Just perl -- shell
-               | ext `elem` [".txt"]                = Just text

krebs/5pkgs/much/default.nix

@@ -1,6 +1,6 @@
 { pkgs, ... }:
 
-pkgs.haskellngPackages.callPackage (
+pkgs.haskellPackages.callPackage (
 { mkDerivation, aeson, attoparsec, base, base64-bytestring
 , blaze-builder, blessings, bytestring, case-insensitive, containers, deepseq
 , directory, docopt, email-header, fetchgit, filepath

krebs/5pkgs/urlwatch/default.nix

@@ -0,0 +1,39 @@
+{ stdenv, fetchurl, python3Packages }:
+
+python3Packages.buildPythonPackage rec {
+  name = "urlwatch-2.0";
+
+  src = fetchurl {
+    url = "https://thp.io/2008/urlwatch/${name}.tar.gz";
+    sha256 = "0j38qzw4jxw41vnnpi6j851hqpv8d6p1cbni6cv8r2vqf5307s3b";
+  };
+
+  propagatedBuildInputs = with python3Packages; [
+    pyyaml
+    keyring
+    (python3Packages.buildPythonPackage rec {
+      name = "minidb-2.0.1";
+      src = fetchurl {
+        url = "https://thp.io/2010/minidb/${name}.tar.gz";
+        sha256 = "1x958zr9jc26vaqij451qb9m2l7apcpz34ir9fwfjg4fwv24z2dy";
+      };
+      meta = {
+        description = "A simple SQLite3-based store for Python objects";
+        homepage = https://thp.io/2010/minidb/;
+        license = stdenv.lib.licenses.isc;
+        maintainers = [ stdenv.lib.maintainers.tv ];
+      };
+    })
+  ];
+
+  postFixup = ''
+    wrapProgram "$out/bin/urlwatch" --prefix "PYTHONPATH" : "$PYTHONPATH"
+  '';
+
+  meta = {
+    description = "A tool for monitoring webpages for updates";
+    homepage = https://thp.io/2008/urlwatch/;
+    license = stdenv.lib.licenses.bsd3;
+    maintainers = [ stdenv.lib.maintainers.tv ];
+  };
+}#

krebs/default.nix

@@ -171,9 +171,12 @@ let out = {
     ${b}
     '';
 
-  get-config = system:
-    stockholm.users.${current-user-name}.${system}.config
+  get-config = system: let
+    config = stockholm.users.${current-user-name}.${system}.config
       or (abort "unknown system: ${system}, user: ${current-user-name}");
+  in
+  assert config.krebs.build.source-version == 1;
+  config;
 
   nix-install =
     { system ? current-host-name
@@ -240,7 +243,7 @@ let out = {
               --exclude tmp \
               --rsync-path='mkdir -p ${target-path} && rsync' \
               --delete-excluded \
-              -vrLptgoD \
+              -vrlptD \
               ${config.path}/ \
               root@${target}:${target-path}
           '';

krebs/populate.nix

@@ -0,0 +1,116 @@
+{ source
+, target-user ? "root"
+, target-host
+, target-path ? "/var/src"
+}:
+with import <nixpkgs/lib>;
+with import ~/stockholm/krebs/4lib {
+  lib = import <nixpkgs/lib>;
+};
+with builtins;
+let
+  out = ''
+    #! /bin/sh
+    set -efu
+
+    echo ${shell.escape git-script} \
+      | ssh ${shell.escape "${target-user}@${target-host}"} -T
+
+    tmpdir=$(mktemp -dt stockholm.XXXXXXXX)
+    trap '
+      set +f
+      rm "$tmpdir"/*
+      rmdir "$tmpdir"
+      trap - EXIT INT QUIT
+    '        EXIT INT QUIT
+    chmod 0755 "$tmpdir"
+    ${concatStringsSep "\n"
+      (mapAttrsToList
+        (name: spec: let dst = removePrefix "symlink:" (get-url spec); in
+          "ln -s ${shell.escape dst} $tmpdir/${shell.escape name}")
+        symlink-specs)}
+
+    proot \
+        -b $tmpdir:${shell.escape target-path} \
+        ${concatStringsSep " \\\n    "
+          (mapAttrsToList
+            (name: spec:
+              "-b ${shell.escape "${get-url spec}:${target-path}/${name}"}")
+            file-specs)} \
+        rsync \
+            -f ${shell.escape "P /*"} \
+            ${concatMapStringsSep " \\\n        "
+              (name: "-f ${shell.escape "R /${name}"}")
+              (attrNames file-specs)} \
+            --delete \
+            -vFrlptD \
+            ${shell.escape target-path}/ \
+            ${shell.escape "${target-user}@${target-host}:${target-path}"}
+  '';
+
+  get-schema = uri:
+    if substring 0 1 uri == "/"
+      then "file"
+      else head (splitString ":" uri);
+
+  has-schema = schema: uri: get-schema uri == schema;
+
+  get-url = spec: {
+    string = spec;
+    path = toString spec;
+    set = get-url spec.url;
+  }.${typeOf spec};
+
+  git-specs =
+    filterAttrs (_: spec: has-schema "https" (get-url spec)) source //
+    filterAttrs (_: spec: has-schema "http" (get-url spec)) source //
+    filterAttrs (_: spec: has-schema "git" (get-url spec)) source;
+
+  file-specs =
+    filterAttrs (_: spec: has-schema "file" (get-url spec)) source;
+
+  symlink-specs =
+    filterAttrs (_: spec: has-schema "symlink" (get-url spec)) source;
+
+  git-script = ''
+    fetch_git() {(
+      dst_dir=$1
+      src_url=$2
+      src_ref=$3
+
+      if ! test -e "$dst_dir"; then
+        git clone "$src_url" "$dst_dir"
+      fi
+
+      cd "$dst_dir"
+
+      if ! url=$(git config remote.origin.url); then
+        git remote add origin "$src_url"
+      elif test "$url" != "$src_url"; then
+        git remote set-url origin "$src_url"
+      fi
+
+      # TODO resolve src_ref to commit hash
+      hash=$src_ref
+
+      if ! test "$(git log --format=%H -1)" = "$hash"; then
+        git fetch origin
+        git checkout "$hash" -- "$dst_dir"
+        git checkout "$hash"
+      fi
+
+      git clean -dxf
+    )}
+
+    ${concatStringsSep "\n"
+      (mapAttrsToList
+        (name: spec: toString (map shell.escape [
+          "fetch_git"
+          "${target-path}/${name}"
+          spec.url
+          spec.rev
+        ]))
+        git-specs)}
+  '';
+
+in out

nixpkgs/default.nix

@@ -0,0 +1 @@
+import <upstream-nixpkgs>

nixpkgs/nixos/default.nix

@@ -0,0 +1,65 @@
+{ configuration ? import <upstream-nixpkgs/nixos/lib/from-env.nix> "NIXOS_CONFIG" <nixos-config>
+, system ? builtins.currentSystem
+}:
+
+let
+  eval-config = modules: import <upstream-nixpkgs/nixos/lib/eval-config.nix> {
+    inherit system;
+    modules = modules ++ [({ config, lib, ... }: with lib; {
+      imports = filter dir.has-default-nix (concatLists [
+        (map (p: p + "/2configs") [ <stockholm-private> ])
+        (map (p: p + "/3modules") [ <stockholm-krebs> <stockholm-private> ])
+      ]);
+
+      krebs.current = {
+        enable = true;
+        host = config.krebs.hosts.${readFile /proc/sys/kernel/hostname};
+        user = config.krebs.users.${getEnv "LOGNAME"};
+      };
+
+      nixpkgs.config.packageOverrides = pkgs: let
+        kpkgs = import <stockholm-krebs/5pkgs> { inherit lib pkgs; };
+        upkgs = import <stockholm-private/5pkgs> { inherit lib; pkgs = pkgs // kpkgs; };
+      in kpkgs // upkgs;
+    })];
+    specialArgs = {
+      lib = let
+        nlib = import <upstream-nixpkgs/lib> // builtins;
+        klib = nlib // import <stockholm-krebs/4lib> { lib = nlib; };
+        ulib = klib // (with klib; let p = <stockholm-private> + "/4lib"; in
+          optionalAttrs (dir.has-default-nix p)
+                        (import p { lib = klib; }));
+      in ulib;
+    };
+  };
+
+  eval = eval-config [
+    configuration
+  ];
+
+  # This is for `nixos-rebuild build-vm'.
+  vm = eval-config [
+    configuration
+    <upstream-nixpkgs/nixos/modules/virtualisation/qemu-vm.nix>
+  ];
+
+  # This is for `nixos-rebuild build-vm-with-bootloader'.
+  vm-with-bootloader = eval-config [
+    configuration
+    <upstream-nixpkgs/nixos/modules/virtualisation/qemu-vm.nix>
+    { virtualisation.useBootLoader = true; }
+  ];
+in
+
+{
+  inherit (eval) config options;
+
+  system = eval.config.system.build.toplevel;
+
+  vm = vm.config.system.build.vm;
+
+  vmWithBootLoader = vm-with-bootloader.config.system.build.vm;
+
+  # The following are used by nixos-rebuild.
+  nixFallback = eval.pkgs.nixUnstable;
+}

nixpkgs/nixos/modules

@@ -0,0 +1 @@
+../../upstream-nixpkgs/nixos/modules

tv/1systems/cd.nix

@@ -6,6 +6,11 @@ with lib;
   krebs.build.host = config.krebs.hosts.cd;
   krebs.build.target = "root@cd.internet";
 
+  krebs.build.source.upstream-nixpkgs = {
+    url = https://github.com/NixOS/nixpkgs;
+    rev = "b7ff030";
+  };
+
   imports = [
     ../2configs/hw/CAC-Developer-2.nix
     ../2configs/fs/CAC-CentOS-7-64bit.nix

tv/1systems/wu.nix

@@ -18,7 +18,7 @@ with lib;
         # stockholm
         gnumake
         hashPassword
-        lentil
+        haskellPackages.lentil
         parallel
         (pkgs.writeScriptBin "im" ''
           #! ${pkgs.bash}/bin/bash

tv/1systems/xu.nix

@@ -21,7 +21,7 @@ with lib;
         # stockholm
         gnumake
         hashPassword
-        lentil
+        haskellPackages.lentil
         parallel
         (pkgs.writeScriptBin "im" ''
           #! ${pkgs.bash}/bin/bash

tv/2configs/default.nix

@@ -8,20 +8,23 @@ with lib;
   krebs.build = {
     user = config.krebs.users.tv;
     target = mkDefault "root@${config.krebs.build.host.name}";
-    source = {
-      git.nixpkgs = {
-        url = mkDefault https://github.com/NixOS/nixpkgs;
-        rev = mkDefault "b7ff0301d6f26bd8419e888fd0e129f3dc8bd328";
-        target-path = mkDefault "/var/src/nixpkgs";
+    source-version = 2;
+    source = mapAttrs (_: mkDefault) ({
+      nixos-config = "symlink:stockholm-private/1systems/${config.krebs.build.host.name}.nix";
+      nixpkgs = symlink:stockholm-nixpkgs;
+      secrets = "/home/tv/secrets/${config.krebs.build.host.name}";
+      secrets-common = "/home/tv/secrets/common";
+      stockholm-krebs = "/home/tv/stockholm/krebs";
+      stockholm-nixpkgs = "/home/tv/stockholm/nixpkgs";
+      stockholm-private = "/home/tv/stockholm/tv";
+      upstream-nixpkgs = {
+        url = https://github.com/NixOS/nixpkgs;
+        rev = "77f8f35d57618c1ba456d968524f2fb2c3448295";
+        dev = "/home/tv/nixpkgs";
       };
-      dir.secrets = {
-        path = mkDefault "/home/tv/secrets/${config.krebs.build.host.name}";
-      };
-      dir.stockholm = {
-        path = mkDefault "/home/tv/stockholm";
-        target-path = mkDefault "/var/src/stockholm";
-      };
-    };
+    } // optionalAttrs config.krebs.build.host.secure {
+      secrets-master = "/home/tv/secrets/master";
+    });
   };
 
   networking.hostName = config.krebs.build.host.name;
@@ -98,12 +101,7 @@ with lib;
       };
 
       environment.variables = {
-        NIX_PATH =
-          with config.krebs.build.source; with dir; with git;
-          mkForce (concatStringsSep ":" [
-            "nixpkgs=${nixpkgs.target-path}"
-            "secrets=${stockholm.target-path}/null"
-          ]);
+        NIX_PATH = mkForce "/var/src";
       };
 
       programs.bash = {
@@ -182,7 +180,8 @@ with lib;
     {
       systemd.tmpfiles.rules = let
         forUsers = flip map users;
-        isUser = { group, ... }: hasSuffix "users" group;
+        isUser = { name, group, ... }:
+          name == "root" || hasSuffix "users" group;
         users = filter isUser (mapAttrsToList (_: id) config.users.users);
       in forUsers (u: "d /run/xdg/${u.name} 0700 ${u.name} ${u.group} -");
       environment.variables.XDG_RUNTIME_DIR = "/run/xdg/$LOGNAME";

tv/5pkgs/default.nix

@@ -11,6 +11,9 @@
         --disk-cache-size=50000000 \
         "%@"
   '';
+  ejabberd = pkgs.callPackage ./ejabberd {
+    erlang = pkgs.erlangR16;
+  };
   ff = pkgs.callPackage ./ff {};
   viljetic-pages = pkgs.callPackage ./viljetic-pages {};
   xmonad-tv =

tv/5pkgs/ejabberd/default.nix

@@ -0,0 +1,28 @@
+{stdenv, fetchurl, expat, erlang, zlib, openssl, pam, lib}:
+
+stdenv.mkDerivation rec {
+  version = "2.1.13";
+  name = "ejabberd-${version}";
+  src = fetchurl {
+    url = "http://www.process-one.net/downloads/ejabberd/${version}/${name}.tgz";
+    sha256 = "0vf8mfrx7vr3c5h3nfp3qcgwf2kmzq20rjv1h9sk3nimwir1q3d8";
+  };
+  buildInputs = [ expat erlang zlib openssl pam ];
+  patchPhase = ''
+    sed -i \
+      -e "s|erl \\\|${erlang}/bin/erl \\\|" \
+      -e 's|EXEC_CMD=\"sh -c\"|EXEC_CMD=\"${stdenv.shell} -c\"|' \
+      src/ejabberdctl.template
+  '';
+  preConfigure = ''
+    cd src
+  '';
+  configureFlags = ["--enable-pam"];
+
+  meta = {
+    description = "Open-source XMPP application server written in Erlang";
+    license = stdenv.lib.licenses.gpl2;
+    homepage = http://www.ejabberd.im;
+    maintainers = [ lib.maintainers.sander ];
+  };
+}