replace krebs.build.populate by populate

10 files changed, 127 insertions, 193 deletions

diff --git a/Makefile b/Makefile
index aa5d5d8ca..be069ec3e 100644
--- a/Makefile
+++ b/Makefile
@@ -68,23 +68,27 @@ evaluate = \
 		-I stockholm=$(stockholm) \
 		-E "let eval = import <stockholm>; in with eval; $(1)"
 
-execute = \
-	result=$$($(call evaluate,config.krebs.build.$(1))) && \
-	script=$$(echo "$$result" | jq -r .) && \
-	echo "$$script" | PS5=% sh
-
 ifeq ($(MAKECMDGOALS),)
 $(error No goals specified)
 endif
 
 # usage: make deploy system=foo [target_host=bar]
 deploy: ssh ?= ssh
-deploy:
-	$(call execute,populate)
+deploy: populate
 	$(ssh) $(target_user)@$(target_host) -p $(target_port) \
 		env STOCKHOLM_VERSION="$$STOCKHOLM_VERSION" \
 			nixos-rebuild switch --show-trace -I $(target_path)
 
+# usage: make populate system=foo
+ifeq ($(debug),true)
+populate: populate-flags = --debug
+endif
+populate:
+	source=$$($(call evaluate,config.krebs.build.source) --json --strict) && \
+	echo $$source | populate \
+			$(target_user)@$(target_host):$(target_port)$(target_path) \
+			$(populate-flags)
+
 # usage: make build.pkgs.get
 build build.:;@$(call build,$${expr-eval})
 build.%:;@$(call build,$@)
@@ -99,7 +103,7 @@ install:
 	$(ssh) $(target_user)@$(target_host) -p $(target_port) \
 		env target_path=$(target_path) \
 			sh -s prepare < krebs/4lib/infest/prepare.sh
-	target_path=/mnt$(target_path) $(call execute,populate)
+	$(MAKE) populate target_path=/mnt$(target_path)
 	$(ssh) $(target_user)@$(target_host) -p $(target_port) \
 		env NIXOS_CONFIG=$(target_path)/nixos-config \
 				STOCKHOLM_VERSION="$$STOCKHOLM_VERSION" \
@@ -117,8 +121,7 @@ $(error bad method: $(method))
 endif
 endif
 test: ssh ?= ssh
-test:
-	$(call execute,populate)
+test: populate
 	$(ssh) $(target_user)@$(target_host) -p $(target_port) \
 		$(command) --show-trace -I $(target_path) \
 			-A config.system.build.toplevel $(target_path)/stockholm
diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix
index 9cd095622..5924d1033 100644
--- a/krebs/3modules/build.nix
+++ b/krebs/3modules/build.nix
@@ -21,145 +21,9 @@ let
     };
 
     options.krebs.build.source = mkOption {
-      type = with types; attrsOf (either str (submodule {
-        options = {
-          url = str;
-          rev = str;
-        };
-      }));
+      type = types.attrsOf types.source;
       default = {};
     };
-
-    options.krebs.build.populate = mkOption {
-      type = types.str;
-      default = let
-        target-user = maybeEnv "target_user" "root";
-        target-host = maybeEnv "target_host" config.krebs.build.host.name;
-        target-port = maybeEnv "target_port" "22";
-        target-path = maybeEnv "target_path" "/var/src";
-        out = ''
-          #! /bin/sh
-          set -eu
-
-          ssh=''${ssh-ssh}
-
-          verbose() {
-            printf '%s%s\n' "$PS5$(printf ' %q' "$@")" >&2
-            "$@"
-          }
-
-          { printf 'PS5=%q%q\n' @ "$PS5"
-            echo ${shell.escape git-script}
-          } | verbose $ssh -p ${shell.escape target-port} \
-                  ${shell.escape "${target-user}@${target-host}"} -T
-
-          unset tmpdir
-          trap '
-            rm -f "$tmpdir"/*
-            rmdir "$tmpdir"
-            trap - EXIT INT QUIT
-          '        EXIT INT QUIT
-          tmpdir=$(mktemp -dt stockholm.XXXXXXXX)
-          chmod 0755 "$tmpdir"
-
-          ${concatStringsSep "\n" (mapAttrsToList (name: symlink: ''
-            verbose ln -s ${shell.escape symlink.target} \
-                          "$tmpdir"/${shell.escape name}
-          '') source-by-method.symlink)}
-
-          verbose proot \
-              -b "$tmpdir":${shell.escape target-path} \
-              ${concatStringsSep " \\\n    " (mapAttrsToList (name: file:
-                "-b ${shell.escape "${file.path}:${target-path}/${name}"}"
-              ) source-by-method.file)} \
-              rsync \
-                  -f ${shell.escape "P /*"} \
-                  ${concatMapStringsSep " \\\n        " (name:
-                    "-f ${shell.escape "R /${name}"}"
-                  ) (attrNames source-by-method.file)} \
-                  --delete \
-                  -vFrlptD \
-                  -e "$ssh -p ${shell.escape target-port}" \
-                  ${shell.escape target-path}/ \
-                  ${shell.escape "${target-user}@${target-host}:${target-path}"}
-        '';
-
-        git-script = ''
-          #! /bin/sh
-          set -efu
-
-          export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
-
-          verbose() {
-            printf '%s%s\n' "$PS5$(printf ' %q' "$@")" >&2
-            "$@"
-          }
-
-          fetch_git() {(
-            dst_dir=$1
-            src_url=$2
-            src_ref=$3
-
-            if ! test -e "$dst_dir"; then
-              git clone "$src_url" "$dst_dir"
-            fi
-
-            cd "$dst_dir"
-
-            if ! url=$(git config remote.origin.url); then
-              git remote add origin "$src_url"
-            elif test "$url" != "$src_url"; then
-              git remote set-url origin "$src_url"
-            fi
-
-            # TODO resolve src_ref to commit hash
-            hash=$src_ref
-
-            if ! test "$(git log --format=%H -1)" = "$hash"; then
-              git fetch origin
-              git checkout "$hash" -- "$dst_dir"
-              git checkout -f "$hash"
-            fi
-
-            git clean -dxf
-          )}
-
-          ${concatStringsSep "\n" (mapAttrsToList (name: git: ''
-            verbose fetch_git ${concatMapStringsSep " " shell.escape [
-              "${target-path}/${name}"
-              git.url
-              git.rev
-            ]}
-          '') source-by-method.git)}
-        '';
-      in out;
-    };
-
-  };
-
-  source-by-method = let
-    known-methods = ["git" "file" "symlink"];
-  in genAttrs known-methods (const {}) // recursiveUpdate source-by-scheme {
-    git = source-by-scheme.http or {} //
-          source-by-scheme.https or {};
   };
 
-  source-by-scheme = foldl' (out: { k, v }: recursiveUpdate out {
-    ${v.scheme}.${k} = v;
-  }) {} (mapAttrsToList (k: v: { inherit k v; }) normalized-source);
-
-  normalized-source = mapAttrs (name: let f = x: getAttr (typeOf x) {
-    path = f (toString x);
-    string = f {
-      url = if substring 0 1 x == "/" then "file://${x}" else x;
-    };
-    set = let scheme = head (splitString ":" x.url); in recursiveUpdate x {
-      inherit scheme;
-    } // {
-      symlink.target = removePrefix "symlink:" x.url;
-      file.path = # TODO file://host/...
-                  assert hasPrefix "file:///" x.url;
-                  removePrefix "file://" x.url;
-    }.${scheme} or {};
-  }; in f) config.krebs.build.source;
 in out
diff --git a/krebs/4lib/types.nix b/krebs/4lib/types.nix
index aa7b7a9f5..8906eff4a 100644
--- a/krebs/4lib/types.nix
+++ b/krebs/4lib/types.nix
@@ -188,6 +188,75 @@ types // rec {
     };
   });
 
+
+  source = submodule ({ config, ... }: {
+    options = {
+      type = let
+        types = ["file" "git" "symlink"];
+      in mkOption {
+        type = enum types;
+        default = let
+          cands = filter (k: config.${k} != null) types;
+        in
+          if length cands == 1
+            then head cands
+            else throw "cannot determine type";
+      };
+      file = let
+        file-path = (file-source.getSubOptions "FIXME").path.type;
+      in mkOption {
+        type = nullOr (either file-source file-path);
+        default = null;
+        apply = x:
+          if file-path.check x
+            then { path = x; }
+            else x;
+      };
+      git = mkOption {
+        type = nullOr git-source;
+        default = null;
+      };
+      symlink = let
+        symlink-target = (symlink-source.getSubOptions "FIXME").target.type;
+      in mkOption {
+        type = nullOr (either symlink-source symlink-target);
+        default = null;
+        apply = x:
+          if symlink-target.check x
+            then { target = x; }
+            else x;
+      };
+    };
+  });
+
+  file-source = submodule {
+    options = {
+      path = mkOption {
+        type = absolute-pathname;
+      };
+    };
+  };
+
+  git-source = submodule {
+    options = {
+      ref = mkOption {
+        type = str; # TODO types.git.ref
+      };
+      url = mkOption {
+        type = str; # TODO types.git.url
+      };
+    };
+  };
+
+  symlink-source = submodule {
+    options = {
+      target = mkOption {
+        type = pathname; # TODO relative-pathname
+      };
+    };
+  };
+
+
   suffixed-str = suffs:
     mkOptionType {
       name = "string suffixed by ${concatStringsSep ", " suffs}";
diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes
index ab6bc557c..2a3ebd6fc 100755
--- a/krebs/5pkgs/test/infest-cac-centos7/notes
+++ b/krebs/5pkgs/test/infest-cac-centos7/notes
@@ -138,8 +138,8 @@ ip=$(cac-api getserver $id | jq -r .ip)
 cat > shared/2configs/temp/dirs.nix <<EOF
 _: {
   krebs.build.source = {
-    secrets = "$krebs_secrets";
-    stockholm = "$(pwd)";
+    secrets.file = "$krebs_secrets";
+    stockholm.file = "$(pwd)";
   };
   users.extraUsers.root.openssh.authorizedKeys.keys = [
     "$(cat ${krebs_ssh}.pub)"
diff --git a/lass/2configs/buildbot-standalone.nix b/lass/2configs/buildbot-standalone.nix
index 04bdcf9d8..4c63d857c 100644
--- a/lass/2configs/buildbot-standalone.nix
+++ b/lass/2configs/buildbot-standalone.nix
@@ -64,7 +64,7 @@ in {
 
       # prepare nix-shell
       # the dependencies which are used by the test script
-      deps = [ "gnumake", "jq", "nix", "rsync", "proot" ]
+      deps = [ "gnumake", "jq", "nix", "(import <stockholm>).pkgs.populate" ]
       # TODO: --pure , prepare ENV in nix-shell command:
       #                   SSL_CERT_FILE,LOGNAME,NIX_REMOTE
       nixshell = ["nix-shell",
@@ -112,8 +112,7 @@ in {
         for i in [ "prism", "mors", "echelon" ]:
           addShell(f,name="populate-{}".format(i),env=env_lass,
                   command=nixshell + \
-                            ["{}( make system={} eval.config.krebs.build.populate \
-                               | jq -er .)".format("!" if "failing" in i else "",i)])
+                            ["{}(make system={} populate debug=true)".format("!" if "failing" in i else "",i)])
 
         addShell(f,name="build-test-minimal",env=env_lass,
                   command=nixshell + \
@@ -146,7 +145,7 @@ in {
     masterhost = "localhost";
     username = "testslave";
     password = "lasspass";
-    packages = with pkgs;[ git nix gnumake jq rsync ];
+    packages = with pkgs; [ gnumake jq nix populate ];
     extraEnviron = {
       NIX_PATH="nixpkgs=/var/src/nixpkgs";
     };
diff --git a/lass/2configs/default.nix b/lass/2configs/default.nix
index e3065ba84..b8c50f1aa 100644
--- a/lass/2configs/default.nix
+++ b/lass/2configs/default.nix
@@ -53,16 +53,14 @@ with config.krebs.lib;
     search-domain = "retiolum";
     build = {
       user = config.krebs.users.lass;
-      source = mapAttrs (_: mkDefault) ({
-        nixos-config = "symlink:stockholm/lass/1systems/${config.krebs.build.host.name}.nix";
-        secrets = if getEnv "dummy_secrets" == "true"
-          then toString <stockholm/lass/2configs/tests/dummy-secrets>
-          else "/home/lass/secrets/${config.krebs.build.host.name}";
-        #secrets-common = "/home/lass/secrets/common";
-        stockholm = getEnv "PWD";
-      } // optionalAttrs config.krebs.build.host.secure {
-        #secrets-master = "/home/lass/secrets/master";
-      });
+      source = let inherit (config.krebs.build) host; in {
+        nixos-config.symlink = "stockholm/lass/1systems/${host.name}.nix";
+        secrets.file =
+          if getEnv "dummy_secrets" == "true"
+            then toString <stockholm/lass/2configs/tests/dummy-secrets>
+            else "/home/lass/secrets/${host.name}";
+        stockholm.file = getEnv "PWD";
+      };
     };
   };
 
diff --git a/makefu/2configs/default.nix b/makefu/2configs/default.nix
index 422927b28..58a537a2b 100644
--- a/makefu/2configs/default.nix
+++ b/makefu/2configs/default.nix
@@ -19,20 +19,22 @@ with config.krebs.lib;
 
     dns.providers.siem = "hosts";
     search-domain = "retiolum";
-    build =  {
+    build = {
       user = config.krebs.users.makefu;
-      source =  mapAttrs (_: mkDefault) {
-        nixpkgs = {
+      source = let inherit (config.krebs.build) host user; in {
+        nixpkgs.git = {
           url = https://github.com/nixos/nixpkgs;
-          rev = "63b9785"; # stable @ 2016-06-01
+          ref = "63b9785"; # stable @ 2016-06-01
         };
-        secrets = if getEnv "dummy_secrets" == "true"
-                  then toString <stockholm/makefu/6tests/data/secrets>
-                  else "/home/makefu/secrets/${config.krebs.build.host.name}";
-        stockholm = "/home/makefu/stockholm";
+        secrets.file =
+          if getEnv "dummy_secrets" == "true"
+            then toString <stockholm/makefu/6tests/data/secrets>
+            else "/home/makefu/secrets/${host.name}";
+        stockholm.file = "/home/makefu/stockholm";
 
         # Defaults for all stockholm users?
-        nixos-config = "symlink:stockholm/${config.krebs.build.user.name}/1systems/${config.krebs.build.host.name}.nix";
+        nixos-config.symlink =
+          "stockholm/${user.name}/1systems/${host.name}.nix";
       };
     };
   };
diff --git a/shared/2configs/base.nix b/shared/2configs/base.nix
index bbb089c2c..a92a0df35 100644
--- a/shared/2configs/base.nix
+++ b/shared/2configs/base.nix
@@ -7,15 +7,14 @@ with config.krebs.lib;
 
   # TODO rename shared user to "krebs"
   krebs.build.user = mkDefault config.krebs.users.shared;
-  krebs.build.source = {
-    nixpkgs = mkDefault {
+  krebs.build.source = let inherit (config.krebs.build) host user; in {
+    nixos-config.symlink = "stockholm/${user.name}/1systems/${host.name}.nix";
+    nixpkgs.git = {
       url = https://github.com/NixOS/nixpkgs;
-      rev = "63b9785"; # stable @ 2016-06-01
+      ref = "63b9785"; # stable @ 2016-06-01
     };
-    secrets =  mkDefault "${getEnv "HOME"}/secrets/krebs/${config.krebs.build.host.name}";
-    stockholm = mkDefault "${getEnv "HOME"}/stockholm";
-
-    nixos-config = "symlink:stockholm/${config.krebs.build.user.name}/1systems/${config.krebs.build.host.name}.nix";
+    secrets.file = "${getEnv "HOME"}/secrets/krebs/${host.name}";
+    stockholm.file = "${getEnv "HOME"}/stockholm";
   };
 
   networking.hostName = config.krebs.build.host.name;
diff --git a/shared/2configs/shared-buildbot.nix b/shared/2configs/shared-buildbot.nix
index 6c40d9966..688f8f9aa 100644
--- a/shared/2configs/shared-buildbot.nix
+++ b/shared/2configs/shared-buildbot.nix
@@ -75,7 +75,8 @@
 
   # prepare nix-shell
   # the dependencies which are used by the test script
-  deps = [ "gnumake", "jq","nix","rsync",
+  deps = [ "gnumake", "jq", "nix",
+            "(import <stockholm>).pkgs.populate",
             "(import <stockholm>).pkgs.test.infest-cac-centos7" ]
   # TODO: --pure , prepare ENV in nix-shell command:
   #                   SSL_CERT_FILE,LOGNAME,NIX_REMOTE
@@ -95,8 +96,7 @@
   for i in [ "test-centos7", "wolf", "test-failing" ]:
     addShell(f,name="populate-{}".format(i),env=env,
             command=nixshell + \
-                      ["{}( make system={} eval.config.krebs.build.populate \
-                         | jq -er .)".format("!" if "failing" in i else "",i)])
+                      ["{}(make system={} populate debug=true)".format("!" if "failing" in i else "",i)])
 
   # XXX we must prepare ./retiolum.rsa_key.priv for secrets to work
   addShell(f,name="instantiate-test-all-modules",env=env,
@@ -179,7 +179,7 @@
     masterhost = "localhost";
     username = "testslave";
     password = "krebspass";
-    packages = with pkgs;[ git nix gnumake jq rsync ];
+    packages = with pkgs; [ gnumake jq nix populate ];
     # all nix commands will need a working nixpkgs installation
     extraEnviron = {
       NIX_PATH="nixpkgs=/var/src/nixpkgs:nixos-config=./shared/1systems/wolf.nix"; };
diff --git a/tv/2configs/default.nix b/tv/2configs/default.nix
index a9ba1eadd..04009f54d 100644
--- a/tv/2configs/default.nix
+++ b/tv/2configs/default.nix
@@ -7,18 +7,18 @@ with config.krebs.lib;
 
   krebs.build = {
     user = config.krebs.users.tv;
-    source = mapAttrs (_: mkDefault) ({
-      nixos-config = "symlink:stockholm/tv/1systems/${config.krebs.build.host.name}.nix";
-      secrets = "/home/tv/secrets/${config.krebs.build.host.name}";
-      secrets-common = "/home/tv/secrets/common";
-      stockholm = "/home/tv/stockholm";
-      nixpkgs = {
+    source = let inherit (config.krebs.build) host; in {
+      nixos-config.symlink = "stockholm/tv/1systems/${host.name}.nix";
+      secrets.file = "/home/tv/secrets/${host.name}";
+      secrets-common.file = "/home/tv/secrets/common";
+      stockholm.file = "/home/tv/stockholm";
+      nixpkgs.git = {
         url = https://github.com/NixOS/nixpkgs;
-        rev = "8bf31d7d27cae435d7c1e9e0ccb0a320b424066f";
+        ref = "8bf31d7d27cae435d7c1e9e0ccb0a320b424066f";
       };
-    } // optionalAttrs config.krebs.build.host.secure {
-      secrets-master = "/home/tv/secrets/master";
-    });
+    } // optionalAttrs host.secure {
+      secrets-master.file = "/home/tv/secrets/master";
+    };
   };
 
   networking.hostName = config.krebs.build.host.name;