From 514daf3d4611c3d6f451964b5f7ebce22219e6d3 Mon Sep 17 00:00:00 2001
From: tv <tv@krebsco.de>
Date: Sat, 16 Jul 2016 21:43:38 +0200
Subject: [PATCH] replace krebs.build.populate by populate
---
Makefile | 23 ++--
krebs/3modules/build.nix | 138 +---------------------
krebs/4lib/types.nix | 69 +++++++++++
krebs/5pkgs/test/infest-cac-centos7/notes | 4 +-
lass/2configs/buildbot-standalone.nix | 7 +-
lass/2configs/default.nix | 18 ++-
makefu/2configs/default.nix | 20 ++--
shared/2configs/base.nix | 13 +-
shared/2configs/shared-buildbot.nix | 8 +-
tv/2configs/default.nix | 20 ++--
10 files changed, 127 insertions(+), 193 deletions(-)
diff --git a/Makefile b/Makefile
index aa5d5d8ca..be069ec3e 100644
--- a/Makefile
+++ b/Makefile
@@ -68,23 +68,27 @@ evaluate = \
-I stockholm=$(stockholm) \
-E "let eval = import <stockholm>; in with eval; $(1)"
-execute = \
- result=$$($(call evaluate,config.krebs.build.$(1))) && \
- script=$$(echo "$$result" | jq -r .) && \
- echo "$$script" | PS5=% sh
-
ifeq ($(MAKECMDGOALS),)
$(error No goals specified)
endif
# usage: make deploy system=foo [target_host=bar]
deploy: ssh ?= ssh
-deploy:
- $(call execute,populate)
+deploy: populate
$(ssh) $(target_user)@$(target_host) -p $(target_port) \
env STOCKHOLM_VERSION="$$STOCKHOLM_VERSION" \
nixos-rebuild switch --show-trace -I $(target_path)
+# usage: make populate system=foo
+ifeq ($(debug),true)
+populate: populate-flags = --debug
+endif
+populate:
+ source=$$($(call evaluate,config.krebs.build.source) --json --strict) && \
+ echo $$source | populate \
+ $(target_user)@$(target_host):$(target_port)$(target_path) \
+ $(populate-flags)
+
# usage: make build.pkgs.get
build build.:;@$(call build,$${expr-eval})
build.%:;@$(call build,$@)
@@ -99,7 +103,7 @@ install:
$(ssh) $(target_user)@$(target_host) -p $(target_port) \
env target_path=$(target_path) \
sh -s prepare < krebs/4lib/infest/prepare.sh
- target_path=/mnt$(target_path) $(call execute,populate)
+ $(MAKE) populate target_path=/mnt$(target_path)
$(ssh) $(target_user)@$(target_host) -p $(target_port) \
env NIXOS_CONFIG=$(target_path)/nixos-config \
STOCKHOLM_VERSION="$$STOCKHOLM_VERSION" \
@@ -117,8 +121,7 @@ $(error bad method: $(method))
endif
endif
test: ssh ?= ssh
-test:
- $(call execute,populate)
+test: populate
$(ssh) $(target_user)@$(target_host) -p $(target_port) \
$(command) --show-trace -I $(target_path) \
-A config.system.build.toplevel $(target_path)/stockholm
diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix
index 9cd095622..5924d1033 100644
--- a/krebs/3modules/build.nix
+++ b/krebs/3modules/build.nix
@@ -21,145 +21,9 @@ let
};
options.krebs.build.source = mkOption {
- type = with types; attrsOf (either str (submodule {
- options = {
- url = str;
- rev = str;
- };
- }));
+ type = types.attrsOf types.source;
default = {};
};
-
- options.krebs.build.populate = mkOption {
- type = types.str;
- default = let
- target-user = maybeEnv "target_user" "root";
- target-host = maybeEnv "target_host" config.krebs.build.host.name;
- target-port = maybeEnv "target_port" "22";
- target-path = maybeEnv "target_path" "/var/src";
- out = ''
- #! /bin/sh
- set -eu
-
- ssh=''${ssh-ssh}
-
- verbose() {
- printf '%s%s\n' "$PS5$(printf ' %q' "$@")" >&2
- "$@"
- }
-
- { printf 'PS5=%q%q\n' @ "$PS5"
- echo ${shell.escape git-script}
- } | verbose $ssh -p ${shell.escape target-port} \
- ${shell.escape "${target-user}@${target-host}"} -T
-
- unset tmpdir
- trap '
- rm -f "$tmpdir"/*
- rmdir "$tmpdir"
- trap - EXIT INT QUIT
- ' EXIT INT QUIT
- tmpdir=$(mktemp -dt stockholm.XXXXXXXX)
- chmod 0755 "$tmpdir"
-
- ${concatStringsSep "\n" (mapAttrsToList (name: symlink: ''
- verbose ln -s ${shell.escape symlink.target} \
- "$tmpdir"/${shell.escape name}
- '') source-by-method.symlink)}
-
- verbose proot \
- -b "$tmpdir":${shell.escape target-path} \
- ${concatStringsSep " \\\n " (mapAttrsToList (name: file:
- "-b ${shell.escape "${file.path}:${target-path}/${name}"}"
- ) source-by-method.file)} \
- rsync \
- -f ${shell.escape "P /*"} \
- ${concatMapStringsSep " \\\n " (name:
- "-f ${shell.escape "R /${name}"}"
- ) (attrNames source-by-method.file)} \
- --delete \
- -vFrlptD \
- -e "$ssh -p ${shell.escape target-port}" \
- ${shell.escape target-path}/ \
- ${shell.escape "${target-user}@${target-host}:${target-path}"}
- '';
-
- git-script = ''
- #! /bin/sh
- set -efu
-
- export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
-
- verbose() {
- printf '%s%s\n' "$PS5$(printf ' %q' "$@")" >&2
- "$@"
- }
-
- fetch_git() {(
- dst_dir=$1
- src_url=$2
- src_ref=$3
-
- if ! test -e "$dst_dir"; then
- git clone "$src_url" "$dst_dir"
- fi
-
- cd "$dst_dir"
-
- if ! url=$(git config remote.origin.url); then
- git remote add origin "$src_url"
- elif test "$url" != "$src_url"; then
- git remote set-url origin "$src_url"
- fi
-
- # TODO resolve src_ref to commit hash
- hash=$src_ref
-
- if ! test "$(git log --format=%H -1)" = "$hash"; then
- git fetch origin
- git checkout "$hash" -- "$dst_dir"
- git checkout -f "$hash"
- fi
-
- git clean -dxf
- )}
-
- ${concatStringsSep "\n" (mapAttrsToList (name: git: ''
- verbose fetch_git ${concatMapStringsSep " " shell.escape [
- "${target-path}/${name}"
- git.url
- git.rev
- ]}
- '') source-by-method.git)}
- '';
- in out;
- };
-
};
- source-by-method = let
- known-methods = ["git" "file" "symlink"];
- in genAttrs known-methods (const {}) // recursiveUpdate source-by-scheme {
- git = source-by-scheme.http or {} //
- source-by-scheme.https or {};
- };
-
- source-by-scheme = foldl' (out: { k, v }: recursiveUpdate out {
- ${v.scheme}.${k} = v;
- }) {} (mapAttrsToList (k: v: { inherit k v; }) normalized-source);
-
- normalized-source = mapAttrs (name: let f = x: getAttr (typeOf x) {
- path = f (toString x);
- string = f {
- url = if substring 0 1 x == "/" then "file://${x}" else x;
- };
- set = let scheme = head (splitString ":" x.url); in recursiveUpdate x {
- inherit scheme;
- } // {
- symlink.target = removePrefix "symlink:" x.url;
- file.path = # TODO file://host/...
- assert hasPrefix "file:///" x.url;
- removePrefix "file://" x.url;
- }.${scheme} or {};
- }; in f) config.krebs.build.source;
in out
diff --git a/krebs/4lib/types.nix b/krebs/4lib/types.nix
index aa7b7a9f5..8906eff4a 100644
--- a/krebs/4lib/types.nix
+++ b/krebs/4lib/types.nix
@@ -188,6 +188,75 @@ types // rec {
};
});
+
+ source = submodule ({ config, ... }: {
+ options = {
+ type = let
+ types = ["file" "git" "symlink"];
+ in mkOption {
+ type = enum types;
+ default = let
+ cands = filter (k: config.${k} != null) types;
+ in
+ if length cands == 1
+ then head cands
+ else throw "cannot determine type";
+ };
+ file = let
+ file-path = (file-source.getSubOptions "FIXME").path.type;
+ in mkOption {
+ type = nullOr (either file-source file-path);
+ default = null;
+ apply = x:
+ if file-path.check x
+ then { path = x; }
+ else x;
+ };
+ git = mkOption {
+ type = nullOr git-source;
+ default = null;
+ };
+ symlink = let
+ symlink-target = (symlink-source.getSubOptions "FIXME").target.type;
+ in mkOption {
+ type = nullOr (either symlink-source symlink-target);
+ default = null;
+ apply = x:
+ if symlink-target.check x
+ then { target = x; }
+ else x;
+ };
+ };
+ });
+
+ file-source = submodule {
+ options = {
+ path = mkOption {
+ type = absolute-pathname;
+ };
+ };
+ };
+
+ git-source = submodule {
+ options = {
+ ref = mkOption {
+ type = str; # TODO types.git.ref
+ };
+ url = mkOption {
+ type = str; # TODO types.git.url
+ };
+ };
+ };
+
+ symlink-source = submodule {
+ options = {
+ target = mkOption {
+ type = pathname; # TODO relative-pathname
+ };
+ };
+ };
+
+
suffixed-str = suffs:
mkOptionType {
name = "string suffixed by ${concatStringsSep ", " suffs}";
diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes
index ab6bc557c..2a3ebd6fc 100755
--- a/krebs/5pkgs/test/infest-cac-centos7/notes
+++ b/krebs/5pkgs/test/infest-cac-centos7/notes
@@ -138,8 +138,8 @@ ip=$(cac-api getserver $id | jq -r .ip)
cat > shared/2configs/temp/dirs.nix <<EOF
_: {
krebs.build.source = {
- secrets = "$krebs_secrets";
- stockholm = "$(pwd)";
+ secrets.file = "$krebs_secrets";
+ stockholm.file = "$(pwd)";
};
users.extraUsers.root.openssh.authorizedKeys.keys = [
"$(cat ${krebs_ssh}.pub)"
diff --git a/lass/2configs/buildbot-standalone.nix b/lass/2configs/buildbot-standalone.nix
index 04bdcf9d8..4c63d857c 100644
--- a/lass/2configs/buildbot-standalone.nix
+++ b/lass/2configs/buildbot-standalone.nix
@@ -64,7 +64,7 @@ in {
# prepare nix-shell
# the dependencies which are used by the test script
- deps = [ "gnumake", "jq", "nix", "rsync", "proot" ]
+ deps = [ "gnumake", "jq", "nix", "(import <stockholm>).pkgs.populate" ]
# TODO: --pure , prepare ENV in nix-shell command:
# SSL_CERT_FILE,LOGNAME,NIX_REMOTE
nixshell = ["nix-shell",
@@ -112,8 +112,7 @@ in {
for i in [ "prism", "mors", "echelon" ]:
addShell(f,name="populate-{}".format(i),env=env_lass,
command=nixshell + \
- ["{}( make system={} eval.config.krebs.build.populate \
- | jq -er .)".format("!" if "failing" in i else "",i)])
+ ["{}(make system={} populate debug=true)".format("!" if "failing" in i else "",i)])
addShell(f,name="build-test-minimal",env=env_lass,
command=nixshell + \
@@ -146,7 +145,7 @@ in {
masterhost = "localhost";
username = "testslave";
password = "lasspass";
- packages = with pkgs;[ git nix gnumake jq rsync ];
+ packages = with pkgs; [ gnumake jq nix populate ];
extraEnviron = {
NIX_PATH="nixpkgs=/var/src/nixpkgs";
};
diff --git a/lass/2configs/default.nix b/lass/2configs/default.nix
index e3065ba84..b8c50f1aa 100644
--- a/lass/2configs/default.nix
+++ b/lass/2configs/default.nix
@@ -53,16 +53,14 @@ with config.krebs.lib;
search-domain = "retiolum";
build = {
user = config.krebs.users.lass;
- source = mapAttrs (_: mkDefault) ({
- nixos-config = "symlink:stockholm/lass/1systems/${config.krebs.build.host.name}.nix";
- secrets = if getEnv "dummy_secrets" == "true"
- then toString <stockholm/lass/2configs/tests/dummy-secrets>
- else "/home/lass/secrets/${config.krebs.build.host.name}";
- #secrets-common = "/home/lass/secrets/common";
- stockholm = getEnv "PWD";
- } // optionalAttrs config.krebs.build.host.secure {
- #secrets-master = "/home/lass/secrets/master";
- });
+ source = let inherit (config.krebs.build) host; in {
+ nixos-config.symlink = "stockholm/lass/1systems/${host.name}.nix";
+ secrets.file =
+ if getEnv "dummy_secrets" == "true"
+ then toString <stockholm/lass/2configs/tests/dummy-secrets>
+ else "/home/lass/secrets/${host.name}";
+ stockholm.file = getEnv "PWD";
+ };
};
};
diff --git a/makefu/2configs/default.nix b/makefu/2configs/default.nix
index 422927b28..58a537a2b 100644
--- a/makefu/2configs/default.nix
+++ b/makefu/2configs/default.nix
@@ -19,20 +19,22 @@ with config.krebs.lib;
dns.providers.siem = "hosts";
search-domain = "retiolum";
- build = {
+ build = {
user = config.krebs.users.makefu;
- source = mapAttrs (_: mkDefault) {
- nixpkgs = {
+ source = let inherit (config.krebs.build) host user; in {
+ nixpkgs.git = {
url = https://github.com/nixos/nixpkgs;
- rev = "63b9785"; # stable @ 2016-06-01
+ ref = "63b9785"; # stable @ 2016-06-01
};
- secrets = if getEnv "dummy_secrets" == "true"
- then toString <stockholm/makefu/6tests/data/secrets>
- else "/home/makefu/secrets/${config.krebs.build.host.name}";
- stockholm = "/home/makefu/stockholm";
+ secrets.file =
+ if getEnv "dummy_secrets" == "true"
+ then toString <stockholm/makefu/6tests/data/secrets>
+ else "/home/makefu/secrets/${host.name}";
+ stockholm.file = "/home/makefu/stockholm";
# Defaults for all stockholm users?
- nixos-config = "symlink:stockholm/${config.krebs.build.user.name}/1systems/${config.krebs.build.host.name}.nix";
+ nixos-config.symlink =
+ "stockholm/${user.name}/1systems/${host.name}.nix";
};
};
};
diff --git a/shared/2configs/base.nix b/shared/2configs/base.nix
index bbb089c2c..a92a0df35 100644
--- a/shared/2configs/base.nix
+++ b/shared/2configs/base.nix
@@ -7,15 +7,14 @@ with config.krebs.lib;
# TODO rename shared user to "krebs"
krebs.build.user = mkDefault config.krebs.users.shared;
- krebs.build.source = {
- nixpkgs = mkDefault {
+ krebs.build.source = let inherit (config.krebs.build) host user; in {
+ nixos-config.symlink = "stockholm/${user.name}/1systems/${host.name}.nix";
+ nixpkgs.git = {
url = https://github.com/NixOS/nixpkgs;
- rev = "63b9785"; # stable @ 2016-06-01
+ ref = "63b9785"; # stable @ 2016-06-01
};
- secrets = mkDefault "${getEnv "HOME"}/secrets/krebs/${config.krebs.build.host.name}";
- stockholm = mkDefault "${getEnv "HOME"}/stockholm";
-
- nixos-config = "symlink:stockholm/${config.krebs.build.user.name}/1systems/${config.krebs.build.host.name}.nix";
+ secrets.file = "${getEnv "HOME"}/secrets/krebs/${host.name}";
+ stockholm.file = "${getEnv "HOME"}/stockholm";
};
networking.hostName = config.krebs.build.host.name;
diff --git a/shared/2configs/shared-buildbot.nix b/shared/2configs/shared-buildbot.nix
index 6c40d9966..688f8f9aa 100644
--- a/shared/2configs/shared-buildbot.nix
+++ b/shared/2configs/shared-buildbot.nix
@@ -75,7 +75,8 @@
# prepare nix-shell
# the dependencies which are used by the test script
- deps = [ "gnumake", "jq","nix","rsync",
+ deps = [ "gnumake", "jq", "nix",
+ "(import <stockholm>).pkgs.populate",
"(import <stockholm>).pkgs.test.infest-cac-centos7" ]
# TODO: --pure , prepare ENV in nix-shell command:
# SSL_CERT_FILE,LOGNAME,NIX_REMOTE
@@ -95,8 +96,7 @@
for i in [ "test-centos7", "wolf", "test-failing" ]:
addShell(f,name="populate-{}".format(i),env=env,
command=nixshell + \
- ["{}( make system={} eval.config.krebs.build.populate \
- | jq -er .)".format("!" if "failing" in i else "",i)])
+ ["{}(make system={} populate debug=true)".format("!" if "failing" in i else "",i)])
# XXX we must prepare ./retiolum.rsa_key.priv for secrets to work
addShell(f,name="instantiate-test-all-modules",env=env,
@@ -179,7 +179,7 @@
masterhost = "localhost";
username = "testslave";
password = "krebspass";
- packages = with pkgs;[ git nix gnumake jq rsync ];
+ packages = with pkgs; [ gnumake jq nix populate ];
# all nix commands will need a working nixpkgs installation
extraEnviron = {
NIX_PATH="nixpkgs=/var/src/nixpkgs:nixos-config=./shared/1systems/wolf.nix"; };
diff --git a/tv/2configs/default.nix b/tv/2configs/default.nix
index a9ba1eadd..04009f54d 100644
--- a/tv/2configs/default.nix
+++ b/tv/2configs/default.nix
@@ -7,18 +7,18 @@ with config.krebs.lib;
krebs.build = {
user = config.krebs.users.tv;
- source = mapAttrs (_: mkDefault) ({
- nixos-config = "symlink:stockholm/tv/1systems/${config.krebs.build.host.name}.nix";
- secrets = "/home/tv/secrets/${config.krebs.build.host.name}";
- secrets-common = "/home/tv/secrets/common";
- stockholm = "/home/tv/stockholm";
- nixpkgs = {
+ source = let inherit (config.krebs.build) host; in {
+ nixos-config.symlink = "stockholm/tv/1systems/${host.name}.nix";
+ secrets.file = "/home/tv/secrets/${host.name}";
+ secrets-common.file = "/home/tv/secrets/common";
+ stockholm.file = "/home/tv/stockholm";
+ nixpkgs.git = {
url = https://github.com/NixOS/nixpkgs;
- rev = "8bf31d7d27cae435d7c1e9e0ccb0a320b424066f";
+ ref = "8bf31d7d27cae435d7c1e9e0ccb0a320b424066f";
};
- } // optionalAttrs config.krebs.build.host.secure {
- secrets-master = "/home/tv/secrets/master";
- });
+ } // optionalAttrs host.secure {
+ secrets-master.file = "/home/tv/secrets/master";
+ };
};
networking.hostName = config.krebs.build.host.name;