makefu/stockholm
1
0
Fork 0
Code Issues Pull requests Actions1 Packages Projects Releases Wiki Activity

replace krebs.build.populate by populate

Browse source
This commit is contained in:
tv 2016-07-16 21:43:38 +02:00
parent e349c7467c
commit 514daf3d46
10 changed files with 127 additions and 193 deletions
Makefile
krebs
3modules
build.nix
4lib
types.nix
5pkgs/test/infest-cac-centos7
notes
lass/2configs
buildbot-standalone.nixdefault.nix
makefu/2configs
default.nix
shared/2configs
base.nixshared-buildbot.nix
tv/2configs
default.nix

23
Makefile
View file

@ -68,23 +68,27 @@ evaluate = \
-I stockholm=$(stockholm) \
-E "let eval = import <stockholm>; in with eval; $(1)"
execute = \
result=$$($(call evaluate,config.krebs.build.$(1))) && \
script=$$(echo "$$result" | jq -r .) && \
echo "$$script" | PS5=% sh
ifeq ($(MAKECMDGOALS),)
$(error No goals specified)
endif
# usage: make deploy system=foo [target_host=bar]
deploy: ssh ?= ssh
deploy:
$(call execute,populate)
deploy: populate
$(ssh) $(target_user)@$(target_host) -p $(target_port) \
env STOCKHOLM_VERSION="$$STOCKHOLM_VERSION" \
nixos-rebuild switch --show-trace -I $(target_path)
# usage: make populate system=foo
ifeq ($(debug),true)
populate: populate-flags = --debug
endif
populate:
source=$$($(call evaluate,config.krebs.build.source) --json --strict) && \
echo $$source | populate \
$(target_user)@$(target_host):$(target_port)$(target_path) \
$(populate-flags)
# usage: make build.pkgs.get
build build.:;@$(call build,$${expr-eval})
build.%:;@$(call build,$@)
@ -99,7 +103,7 @@ install:
$(ssh) $(target_user)@$(target_host) -p $(target_port) \
env target_path=$(target_path) \
sh -s prepare < krebs/4lib/infest/prepare.sh
target_path=/mnt$(target_path) $(call execute,populate)
$(MAKE) populate target_path=/mnt$(target_path)
$(ssh) $(target_user)@$(target_host) -p $(target_port) \
env NIXOS_CONFIG=$(target_path)/nixos-config \
STOCKHOLM_VERSION="$$STOCKHOLM_VERSION" \
@ -117,8 +121,7 @@ $(error bad method: $(method))
endif
endif
test: ssh ?= ssh
test:
$(call execute,populate)
test: populate
$(ssh) $(target_user)@$(target_host) -p $(target_port) \
$(command) --show-trace -I $(target_path) \
-A config.system.build.toplevel $(target_path)/stockholm

138
krebs/3modules/build.nix
View file

@ -21,145 +21,9 @@ let
};
options.krebs.build.source = mkOption {
type = with types; attrsOf (either str (submodule {
options = {
url = str;
rev = str;
};
}));
type = types.attrsOf types.source;
default = {};
};
options.krebs.build.populate = mkOption {
type = types.str;
default = let
target-user = maybeEnv "target_user" "root";
target-host = maybeEnv "target_host" config.krebs.build.host.name;
target-port = maybeEnv "target_port" "22";
target-path = maybeEnv "target_path" "/var/src";
out = ''
#! /bin/sh
set -eu
ssh=''${ssh-ssh}
verbose() {
printf '%s%s\n' "$PS5$(printf ' %q' "$@")" >&2
"$@"
}
{ printf 'PS5=%q%q\n' @ "$PS5"
echo ${shell.escape git-script}
} | verbose $ssh -p ${shell.escape target-port} \
${shell.escape "${target-user}@${target-host}"} -T
unset tmpdir
trap '
rm -f "$tmpdir"/*
rmdir "$tmpdir"
trap - EXIT INT QUIT
' EXIT INT QUIT
tmpdir=$(mktemp -dt stockholm.XXXXXXXX)
chmod 0755 "$tmpdir"
${concatStringsSep "\n" (mapAttrsToList (name: symlink: ''
verbose ln -s ${shell.escape symlink.target} \
"$tmpdir"/${shell.escape name}
'') source-by-method.symlink)}
verbose proot \
-b "$tmpdir":${shell.escape target-path} \
${concatStringsSep " \\\n " (mapAttrsToList (name: file:
"-b ${shell.escape "${file.path}:${target-path}/${name}"}"
) source-by-method.file)} \
rsync \
-f ${shell.escape "P /*"} \
${concatMapStringsSep " \\\n " (name:
"-f ${shell.escape "R /${name}"}"
) (attrNames source-by-method.file)} \
--delete \
-vFrlptD \
-e "$ssh -p ${shell.escape target-port}" \
${shell.escape target-path}/ \
${shell.escape "${target-user}@${target-host}:${target-path}"}
'';
git-script = ''
#! /bin/sh
set -efu
export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
verbose() {
printf '%s%s\n' "$PS5$(printf ' %q' "$@")" >&2
"$@"
}
fetch_git() {(
dst_dir=$1
src_url=$2
src_ref=$3
if ! test -e "$dst_dir"; then
git clone "$src_url" "$dst_dir"
fi
cd "$dst_dir"
if ! url=$(git config remote.origin.url); then
git remote add origin "$src_url"
elif test "$url" != "$src_url"; then
git remote set-url origin "$src_url"
fi
# TODO resolve src_ref to commit hash
hash=$src_ref
if ! test "$(git log --format=%H -1)" = "$hash"; then
git fetch origin
git checkout "$hash" -- "$dst_dir"
git checkout -f "$hash"
fi
git clean -dxf
)}
${concatStringsSep "\n" (mapAttrsToList (name: git: ''
verbose fetch_git ${concatMapStringsSep " " shell.escape [
"${target-path}/${name}"
git.url
git.rev
]}
'') source-by-method.git)}
'';
in out;
};
};
source-by-method = let
known-methods = ["git" "file" "symlink"];
in genAttrs known-methods (const {}) // recursiveUpdate source-by-scheme {
git = source-by-scheme.http or {} //
source-by-scheme.https or {};
};
source-by-scheme = foldl' (out: { k, v }: recursiveUpdate out {
${v.scheme}.${k} = v;
}) {} (mapAttrsToList (k: v: { inherit k v; }) normalized-source);
normalized-source = mapAttrs (name: let f = x: getAttr (typeOf x) {
path = f (toString x);
string = f {
url = if substring 0 1 x == "/" then "file://${x}" else x;
};
set = let scheme = head (splitString ":" x.url); in recursiveUpdate x {
inherit scheme;
} // {
symlink.target = removePrefix "symlink:" x.url;
file.path = # TODO file://host/...
assert hasPrefix "file:///" x.url;
removePrefix "file://" x.url;
}.${scheme} or {};
}; in f) config.krebs.build.source;
in out

69
krebs/4lib/types.nix
View file

@ -188,6 +188,75 @@ types // rec {
};
});
source = submodule ({ config, ... }: {
options = {
type = let
types = ["file" "git" "symlink"];
in mkOption {
type = enum types;
default = let
cands = filter (k: config.${k} != null) types;
in
if length cands == 1
then head cands
else throw "cannot determine type";
};
file = let
file-path = (file-source.getSubOptions "FIXME").path.type;
in mkOption {
type = nullOr (either file-source file-path);
default = null;
apply = x:
if file-path.check x
then { path = x; }
else x;
};
git = mkOption {
type = nullOr git-source;
default = null;
};
symlink = let
symlink-target = (symlink-source.getSubOptions "FIXME").target.type;
in mkOption {
type = nullOr (either symlink-source symlink-target);
default = null;
apply = x:
if symlink-target.check x
then { target = x; }
else x;
};
};
});
file-source = submodule {
options = {
path = mkOption {
type = absolute-pathname;
};
};
};
git-source = submodule {
options = {
ref = mkOption {
type = str; # TODO types.git.ref
};
url = mkOption {
type = str; # TODO types.git.url
};
};
};
symlink-source = submodule {
options = {
target = mkOption {
type = pathname; # TODO relative-pathname
};
};
};
suffixed-str = suffs:
mkOptionType {
name = "string suffixed by ${concatStringsSep ", " suffs}";

4
krebs/5pkgs/test/infest-cac-centos7/notes
View file

@ -138,8 +138,8 @@ ip=$(cac-api getserver $id | jq -r .ip)
cat > shared/2configs/temp/dirs.nix <<EOF
_: {
krebs.build.source = {
secrets = "$krebs_secrets";
stockholm = "$(pwd)";
secrets.file = "$krebs_secrets";
stockholm.file = "$(pwd)";
};
users.extraUsers.root.openssh.authorizedKeys.keys = [
"$(cat ${krebs_ssh}.pub)"

7
lass/2configs/buildbot-standalone.nix
View file

@ -64,7 +64,7 @@ in {
# prepare nix-shell
# the dependencies which are used by the test script
deps = [ "gnumake", "jq", "nix", "rsync", "proot" ]
deps = [ "gnumake", "jq", "nix", "(import <stockholm>).pkgs.populate" ]
# TODO: --pure , prepare ENV in nix-shell command:
# SSL_CERT_FILE,LOGNAME,NIX_REMOTE
nixshell = ["nix-shell",
@ -112,8 +112,7 @@ in {
for i in [ "prism", "mors", "echelon" ]:
addShell(f,name="populate-{}".format(i),env=env_lass,
command=nixshell + \
["{}( make system={} eval.config.krebs.build.populate \
| jq -er .)".format("!" if "failing" in i else "",i)])
["{}(make system={} populate debug=true)".format("!" if "failing" in i else "",i)])
addShell(f,name="build-test-minimal",env=env_lass,
command=nixshell + \
@ -146,7 +145,7 @@ in {
masterhost = "localhost";
username = "testslave";
password = "lasspass";
packages = with pkgs;[ git nix gnumake jq rsync ];
packages = with pkgs; [ gnumake jq nix populate ];
extraEnviron = {
NIX_PATH="nixpkgs=/var/src/nixpkgs";
};

18
lass/2configs/default.nix
View file

@ -53,16 +53,14 @@ with config.krebs.lib;
search-domain = "retiolum";
build = {
user = config.krebs.users.lass;
source = mapAttrs (_: mkDefault) ({
nixos-config = "symlink:stockholm/lass/1systems/${config.krebs.build.host.name}.nix";
secrets = if getEnv "dummy_secrets" == "true"
then toString <stockholm/lass/2configs/tests/dummy-secrets>
else "/home/lass/secrets/${config.krebs.build.host.name}";
#secrets-common = "/home/lass/secrets/common";
stockholm = getEnv "PWD";
} // optionalAttrs config.krebs.build.host.secure {
#secrets-master = "/home/lass/secrets/master";
});
source = let inherit (config.krebs.build) host; in {
nixos-config.symlink = "stockholm/lass/1systems/${host.name}.nix";
secrets.file =
if getEnv "dummy_secrets" == "true"
then toString <stockholm/lass/2configs/tests/dummy-secrets>
else "/home/lass/secrets/${host.name}";
stockholm.file = getEnv "PWD";
};
};
};

20
makefu/2configs/default.nix
View file

@ -19,20 +19,22 @@ with config.krebs.lib;
dns.providers.siem = "hosts";
search-domain = "retiolum";
build = {
build = {
user = config.krebs.users.makefu;
source = mapAttrs (_: mkDefault) {
nixpkgs = {
source = let inherit (config.krebs.build) host user; in {
nixpkgs.git = {
url = https://github.com/nixos/nixpkgs;
rev = "63b9785"; # stable @ 2016-06-01
ref = "63b9785"; # stable @ 2016-06-01
};
secrets = if getEnv "dummy_secrets" == "true"
then toString <stockholm/makefu/6tests/data/secrets>
else "/home/makefu/secrets/${config.krebs.build.host.name}";
stockholm = "/home/makefu/stockholm";
secrets.file =
if getEnv "dummy_secrets" == "true"
then toString <stockholm/makefu/6tests/data/secrets>
else "/home/makefu/secrets/${host.name}";
stockholm.file = "/home/makefu/stockholm";
# Defaults for all stockholm users?
nixos-config = "symlink:stockholm/${config.krebs.build.user.name}/1systems/${config.krebs.build.host.name}.nix";
nixos-config.symlink =
"stockholm/${user.name}/1systems/${host.name}.nix";
};
};
};

13
shared/2configs/base.nix
View file

@ -7,15 +7,14 @@ with config.krebs.lib;
# TODO rename shared user to "krebs"
krebs.build.user = mkDefault config.krebs.users.shared;
krebs.build.source = {
nixpkgs = mkDefault {
krebs.build.source = let inherit (config.krebs.build) host user; in {
nixos-config.symlink = "stockholm/${user.name}/1systems/${host.name}.nix";
nixpkgs.git = {
url = https://github.com/NixOS/nixpkgs;
rev = "63b9785"; # stable @ 2016-06-01
ref = "63b9785"; # stable @ 2016-06-01
};
secrets = mkDefault "${getEnv "HOME"}/secrets/krebs/${config.krebs.build.host.name}";
stockholm = mkDefault "${getEnv "HOME"}/stockholm";
nixos-config = "symlink:stockholm/${config.krebs.build.user.name}/1systems/${config.krebs.build.host.name}.nix";
secrets.file = "${getEnv "HOME"}/secrets/krebs/${host.name}";
stockholm.file = "${getEnv "HOME"}/stockholm";
};
networking.hostName = config.krebs.build.host.name;

8
shared/2configs/shared-buildbot.nix
View file

@ -75,7 +75,8 @@
# prepare nix-shell
# the dependencies which are used by the test script
deps = [ "gnumake", "jq","nix","rsync",
deps = [ "gnumake", "jq", "nix",
"(import <stockholm>).pkgs.populate",
"(import <stockholm>).pkgs.test.infest-cac-centos7" ]
# TODO: --pure , prepare ENV in nix-shell command:
# SSL_CERT_FILE,LOGNAME,NIX_REMOTE
@ -95,8 +96,7 @@
for i in [ "test-centos7", "wolf", "test-failing" ]:
addShell(f,name="populate-{}".format(i),env=env,
command=nixshell + \
["{}( make system={} eval.config.krebs.build.populate \
| jq -er .)".format("!" if "failing" in i else "",i)])
["{}(make system={} populate debug=true)".format("!" if "failing" in i else "",i)])
# XXX we must prepare ./retiolum.rsa_key.priv for secrets to work
addShell(f,name="instantiate-test-all-modules",env=env,
@ -179,7 +179,7 @@
masterhost = "localhost";
username = "testslave";
password = "krebspass";
packages = with pkgs;[ git nix gnumake jq rsync ];
packages = with pkgs; [ gnumake jq nix populate ];
# all nix commands will need a working nixpkgs installation
extraEnviron = {
NIX_PATH="nixpkgs=/var/src/nixpkgs:nixos-config=./shared/1systems/wolf.nix"; };

20
tv/2configs/default.nix
View file

@ -7,18 +7,18 @@ with config.krebs.lib;
krebs.build = {
user = config.krebs.users.tv;
source = mapAttrs (_: mkDefault) ({
nixos-config = "symlink:stockholm/tv/1systems/${config.krebs.build.host.name}.nix";
secrets = "/home/tv/secrets/${config.krebs.build.host.name}";
secrets-common = "/home/tv/secrets/common";
stockholm = "/home/tv/stockholm";
nixpkgs = {
source = let inherit (config.krebs.build) host; in {
nixos-config.symlink = "stockholm/tv/1systems/${host.name}.nix";
secrets.file = "/home/tv/secrets/${host.name}";
secrets-common.file = "/home/tv/secrets/common";
stockholm.file = "/home/tv/stockholm";
nixpkgs.git = {
url = https://github.com/NixOS/nixpkgs;
rev = "8bf31d7d27cae435d7c1e9e0ccb0a320b424066f";
ref = "8bf31d7d27cae435d7c1e9e0ccb0a320b424066f";
};
} // optionalAttrs config.krebs.build.host.secure {
secrets-master = "/home/tv/secrets/master";
});
} // optionalAttrs host.secure {
secrets-master.file = "/home/tv/secrets/master";
};
};
networking.hostName = config.krebs.build.host.name;