summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorlassulus <lass@aidsballs.de>2016-02-17 15:02:45 +0100
committerlassulus <lass@aidsballs.de>2016-02-17 15:02:45 +0100
commitf48e90915f9c1477a7068050e06a8686ceb03427 (patch)
tree7f3cb5987b2f962fa7f93ffab0c8d47267f70764
parent5c8606e6f6b9337d6f9c1241bf0af84af9db0bdf (diff)
parentd923ede6e33c57901039da59d50c45938228fd7a (diff)
Merge remote-tracking branch 'cd/master'
-rw-r--r--Makefile61
-rw-r--r--krebs/3modules/build.nix117
-rw-r--r--krebs/3modules/buildbot/master.nix4
-rw-r--r--krebs/3modules/buildbot/slave.nix6
-rw-r--r--krebs/3modules/default.nix1
-rw-r--r--krebs/3modules/repo-sync.nix109
-rw-r--r--krebs/5pkgs/repo-sync/default.nix6
-rw-r--r--shared/1systems/wolf.nix1
-rw-r--r--shared/2configs/cgit-mirror.nix9
-rw-r--r--shared/2configs/repo-sync.nix28
-rw-r--r--shared/2configs/shack-drivedroid.nix4
-rw-r--r--shared/2configs/shared-buildbot.nix28
12 files changed, 275 insertions, 99 deletions
diff --git a/Makefile b/Makefile
index 384c872ab..60dfe8030 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,32 @@
-ifndef system
-$(error unbound variable: system)
+stockholm ?= .
+
+ifndef nixos-config
+$(if $(system),,$(error unbound variable: system))
+nixos-config = ./$(LOGNAME)/1systems/$(system).nix
+endif
+
+# target = [target_user@]target_host[:target_port][/target_path]
+ifdef target
+_target_user != echo $(target) | sed -n 's/@.*//p'
+_target_path != echo $(target) | sed -n 's/^[^/]*//p'
+_target_port != echo $(target) | sed -En 's|^.*:([^/]*)(/.*)?$$|\1|p'
+_target_host != echo $(target) | sed -En 's/^(.*@)?([^:/]*).*/\2/p'
+ifneq ($(_target_host),)
+$(if $(target_host),$(error cannot define both, target_host and host in target))
+target_host ?= $(_target_host)
+endif
+ifneq ($(_target_user),)
+$(if $(target_user),$(error cannot define both, target_user and user in target))
+target_user ?= $(_target_user)
+endif
+ifneq ($(_target_port),)
+$(if $(target_port),$(error cannot define both, target_port and port in target))
+target_port ?= $(_target_port)
+endif
+ifneq ($(_target_path),)
+$(if $(target_path),$(error cannot define both, target_path and path in target))
+target_path ?= $(_target_path)
+endif
endif
export target_host ?= $(system)
@@ -7,13 +34,18 @@ export target_user ?= root
export target_port ?= 22
export target_path ?= /var/src
+$(if $(target_host),,$(error unbound variable: target_host))
+$(if $(target_user),,$(error unbound variable: target_user))
+$(if $(target_port),,$(error unbound variable: target_port))
+$(if $(target_path),,$(error unbound variable: target_path))
+
evaluate = \
nix-instantiate \
--eval \
--readonly-mode \
--show-trace \
- -I nixos-config=./$(LOGNAME)/1systems/$(system).nix \
- -I stockholm=. \
+ -I nixos-config=$(nixos-config) \
+ -I stockholm=$(stockholm) \
$(1)
execute = \
@@ -22,9 +54,10 @@ execute = \
echo "$$script" | sh
# usage: make deploy system=foo [target_host=bar]
+deploy: ssh ?= ssh
deploy:
$(call execute,populate)
- ssh $(target_user)@$(target_host) -p $(target_port) \
+ $(ssh) $(target_user)@$(target_host) -p $(target_port) \
nixos-rebuild switch --show-trace -I $(target_path)
# usage: make LOGNAME=shared system=wolf eval.config.krebs.build.host.name
@@ -41,3 +74,21 @@ install:
$(ssh) $(target_user)@$(target_host) -p $(target_port) \
env NIXOS_CONFIG=$(target_path)/nixos-config \
nixos-install
+
+# usage: make test system=foo [target=bar] [method={eval,build}]
+method ?= eval
+ifeq ($(method),build)
+test: command = nix-build --no-out-link
+else
+ifeq ($(method),eval)
+test: command ?= nix-instantiate --eval --json --readonly-mode --strict
+else
+$(error bad method: $(method))
+endif
+endif
+test: ssh ?= ssh
+test:
+ $(call execute,populate)
+ $(ssh) $(target_user)@$(target_host) -p $(target_port) \
+ $(command) --show-trace -I $(target_path) \
+ -A config.system.build.toplevel $(target_path)/stockholm
diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix
index a1f446188..c700fbc56 100644
--- a/krebs/3modules/build.nix
+++ b/krebs/3modules/build.nix
@@ -20,35 +20,19 @@ let
type = types.user;
};
- options.krebs.build.source = let
- raw = types.either types.str types.path;
- url = types.submodule {
+ options.krebs.build.source = mkOption {
+ type = with types; attrsOf (either str (submodule {
options = {
- url = mkOption {
- type = types.str;
- };
- rev = mkOption {
- type = types.str;
- };
- dev = mkOption {
- type = types.str;
- };
+ url = str;
+ rev = str;
};
- };
- in mkOption {
- type = types.attrsOf (types.either types.str url);
- apply = let f = mapAttrs (_: value: {
- string = value;
- path = toString value;
- set = f value;
- }.${typeOf value}); in f;
+ }));
default = {};
};
options.krebs.build.populate = mkOption {
type = types.str;
default = let
- source = config.krebs.build.source;
target-user = maybeEnv "target_user" "root";
target-host = maybeEnv "target_host" config.krebs.build.host.name;
target-port = maybeEnv "target_port" "22";
@@ -75,24 +59,21 @@ let
tmpdir=$(mktemp -dt stockholm.XXXXXXXX)
chmod 0755 "$tmpdir"
- ${concatStringsSep "\n"
- (mapAttrsToList
- (name: spec: let dst = removePrefix "symlink:" (get-url spec); in
- "verbose ln -s ${shell.escape dst} $tmpdir/${shell.escape name}")
- symlink-specs)}
+ ${concatStringsSep "\n" (mapAttrsToList (name: symlink: ''
+ verbose ln -s ${shell.escape symlink.target} \
+ "$tmpdir"/${shell.escape name}
+ '') source-by-method.symlink)}
verbose proot \
- -b $tmpdir:${shell.escape target-path} \
- ${concatStringsSep " \\\n "
- (mapAttrsToList
- (name: spec:
- "-b ${shell.escape "${get-url spec}:${target-path}/${name}"}")
- file-specs)} \
+ -b "$tmpdir":${shell.escape target-path} \
+ ${concatStringsSep " \\\n " (mapAttrsToList (name: file:
+ "-b ${shell.escape "${file.path}:${target-path}/${name}"}"
+ ) source-by-method.file)} \
rsync \
-f ${shell.escape "P /*"} \
- ${concatMapStringsSep " \\\n "
- (name: "-f ${shell.escape "R /${name}"}")
- (attrNames file-specs)} \
+ ${concatMapStringsSep " \\\n " (name:
+ "-f ${shell.escape "R /${name}"}"
+ ) (attrNames source-by-method.file)} \
--delete \
-vFrlptD \
-e ${shell.escape "ssh -p ${target-port}"} \
@@ -100,30 +81,6 @@ let
${shell.escape "${target-user}@${target-host}:${target-path}"}
'';
- get-schema = uri:
- if substring 0 1 uri == "/"
- then "file"
- else head (splitString ":" uri);
-
- has-schema = schema: uri: get-schema uri == schema;
-
- get-url = spec: {
- string = spec;
- path = toString spec;
- set = get-url spec.url;
- }.${typeOf spec};
-
- git-specs =
- filterAttrs (_: spec: has-schema "https" (get-url spec)) source //
- filterAttrs (_: spec: has-schema "http" (get-url spec)) source //
- filterAttrs (_: spec: has-schema "git" (get-url spec)) source;
-
- file-specs =
- filterAttrs (_: spec: has-schema "file" (get-url spec)) source;
-
- symlink-specs =
- filterAttrs (_: spec: has-schema "symlink" (get-url spec)) source;
-
git-script = ''
#! /bin/sh
set -efu
@@ -162,20 +119,42 @@ let
git clean -dxf
)}
- ${concatStringsSep "\n"
- (mapAttrsToList
- (name: spec: toString (map shell.escape [
- "verbose"
- "fetch_git"
- "${target-path}/${name}"
- spec.url
- spec.rev
- ]))
- git-specs)}
+ ${concatStringsSep "\n" (mapAttrsToList (name: git: ''
+ verbose fetch_git ${concatMapStringsSep " " shell.escape [
+ "${target-path}/${name}"
+ git.url
+ git.rev
+ ]}
+ '') source-by-method.git)}
'';
in out;
};
};
+ source-by-method = let
+ known-methods = ["git" "file" "symlink"];
+ in genAttrs known-methods (const {}) // recursiveUpdate source-by-scheme {
+ git = source-by-scheme.http or {} //
+ source-by-scheme.https or {};
+ };
+
+ source-by-scheme = foldl' (out: { k, v }: recursiveUpdate out {
+ ${v.scheme}.${k} = v;
+ }) {} (mapAttrsToList (k: v: { inherit k v; }) normalized-source);
+
+ normalized-source = mapAttrs (name: let f = x: getAttr (typeOf x) {
+ path = f (toString x);
+ string = f {
+ url = if substring 0 1 x == "/" then "file://${x}" else x;
+ };
+ set = let scheme = head (splitString ":" x.url); in recursiveUpdate x {
+ inherit scheme;
+ } // {
+ symlink.target = removePrefix "symlink:" x.url;
+ file.path = # TODO file://host/...
+ assert hasPrefix "file:///" x.url;
+ removePrefix "file://" x.url;
+ }.${scheme} or {};
+ }; in f) config.krebs.build.source;
in out
diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index 825cb3413..2a1dbe31a 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -338,8 +338,8 @@ let
SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
};
serviceConfig = let
- workdir="${lib.shell.escape cfg.workDir}";
- secretsdir="${lib.shell.escape (toString <secrets>)}";
+ workdir = shell.escape cfg.workDir;
+ secretsdir = shell.escape (toString <secrets>);
in {
PermissionsStartOnly = true;
Type = "forking";
diff --git a/krebs/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix
index 7705ac31c..248b46132 100644
--- a/krebs/3modules/buildbot/slave.nix
+++ b/krebs/3modules/buildbot/slave.nix
@@ -149,9 +149,9 @@ let
} // cfg.extraEnviron;
serviceConfig = let
- workdir = "${lib.shell.escape cfg.workDir}";
- contact = "${lib.shell.escape cfg.contact}";
- description = "${lib.shell.escape cfg.description}";
+ workdir = shell.escape cfg.workDir;
+ contact = shell.escape cfg.contact;
+ description = shell.escape cfg.description;
buildbot = pkgs.buildbot-slave;
# TODO:make this
in {
diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix
index 16a74e7c1..c06f3754e 100644
--- a/krebs/3modules/default.nix
+++ b/krebs/3modules/default.nix
@@ -31,6 +31,7 @@ let
./setuid.nix
./tinc_graphs.nix
./urlwatch.nix
+ ./repo-sync.nix
];
options.krebs = api;
config = lib.mkIf cfg.enable imp;
diff --git a/krebs/3modules/repo-sync.nix b/krebs/3modules/repo-sync.nix
new file mode 100644
index 000000000..7a7c80a75
--- /dev/null
+++ b/krebs/3modules/repo-sync.nix
@@ -0,0 +1,109 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+ cfg = config.krebs.repo-sync;
+
+ out = {
+ options.krebs.repo-sync = api;
+ config = mkIf cfg.enable imp;
+ };
+
+ api = {
+ enable = mkEnableOption "repo-sync";
+ config = mkOption {
+ type = with types;attrsOf (attrsOf (attrsOf str));
+ example = literalExample ''
+ # see `repo-sync --help`
+ # `ref` provides sane defaults and can be omitted
+
+ # attrset will be converted to json and be used as config
+ {
+ makefu = {
+ origin = {
+ url = http://github.com/makefu/repo ;
+ ref = "heads/dev" ;
+ };
+ mirror = {
+ url = "git@internal:mirror" ;
+ ref = "heads/github-mirror-dev" ;
+ };
+ };
+ lass = {
+ origin = {
+ url = http://github.com/lass/repo ;
+ };
+ mirror = {
+ url = "git@internal:mirror" ;
+ };
+ };
+ "@latest" = {
+ mirror = {
+ url = "git@internal:mirror";
+ ref = "heads/master";
+ };
+ };
+ };
+ '';
+ };
+ timerConfig = mkOption {
+ type = types.attrsOf types.str;
+ default = {
+ OnCalendar = "*:00,15,30,45";
+ };
+ };
+ stateDir = mkOption {
+ type = types.str;
+ default = "/var/lib/repo-sync";
+ };
+ privateKeyFile = mkOption {
+ type = types.str;
+ description = ''
+ used by repo-sync to identify with ssh service
+ '';
+ default = toString <secrets/wolf-repo-sync.rsa_key.priv>;
+ };
+ };
+ repo-sync-config = pkgs.writeText "repo-sync-config.json"
+ (builtins.toJSON cfg.config);
+
+ imp = {
+ users.users.repo-sync = {
+ name = "repo-sync";
+ uid = config.krebs.lib.genid "repo-sync";
+ description = "repo-sync user";
+ home = cfg.stateDir;
+ createHome = true;
+ };
+
+ systemd.timers.repo-sync = {
+ description = "repo-sync timer";
+ wantedBy = [ "timers.target" ];
+
+ timerConfig = cfg.timerConfig;
+ };
+ systemd.services.repo-sync = {
+ description = "repo-sync";
+ after = [ "network.target" ];
+
+ path = with pkgs; [ ];
+
+ environment = {
+ GIT_SSH_COMMAND = "${pkgs.openssh}/bin/ssh -i ${cfg.stateDir}/ssh.priv";
+ };
+
+ serviceConfig = {
+ Type = "simple";
+ PermissionsStartOnly = true;
+ ExecStartPre = pkgs.writeScript "prepare-repo-sync-user" ''
+ #! /bin/sh
+ cp -v ${config.krebs.lib.shell.escape cfg.privateKeyFile} ${cfg.stateDir}/ssh.priv
+ chown repo-sync ${cfg.stateDir}/ssh.priv
+ '';
+ ExecStart = "${pkgs.repo-sync}/bin/repo-sync ${repo-sync-config}";
+ WorkingDirectory = cfg.stateDir;
+ User = "repo-sync";
+ };
+ };
+ };
+in out
diff --git a/krebs/5pkgs/repo-sync/default.nix b/krebs/5pkgs/repo-sync/default.nix
index 90f838de9..789c03f36 100644
--- a/krebs/5pkgs/repo-sync/default.nix
+++ b/krebs/5pkgs/repo-sync/default.nix
@@ -1,15 +1,17 @@
{ lib, pkgs, python3Packages, fetchurl, ... }:
+
with python3Packages; buildPythonPackage rec {
name = "repo-sync-${version}";
- version = "0.1.1";
+ version = "0.2.5";
disabled = isPy26 || isPy27;
propagatedBuildInputs = [
docopt
GitPython
+ pkgs.git
];
src = fetchurl {
url = "https://pypi.python.org/packages/source/r/repo-sync/repo-sync-${version}.tar.gz";
- sha256 = "01r30l2bbsld90ps13ip0zi2a41b53dv4q6fxrzvkfrprr64c0vv";
+ sha256 = "1a59bj0vc5ajq8indkvkdk022yzvvv5mjb57hk3xf1j3wpr85p84";
};
meta = {
homepage = http://github.com/makefu/repo-sync;
diff --git a/shared/1systems/wolf.nix b/shared/1systems/wolf.nix
index 317591433..96691aed8 100644
--- a/shared/1systems/wolf.nix
+++ b/shared/1systems/wolf.nix
@@ -14,6 +14,7 @@ in
../2configs/shack-drivedroid.nix
../2configs/shared-buildbot.nix
../2configs/cgit-mirror.nix
+ ../2configs/repo-sync.nix
# ../2configs/graphite.nix
];
# use your own binary cache, fallback use cache.nixos.org (which is used by
diff --git a/shared/2configs/cgit-mirror.nix b/shared/2configs/cgit-mirror.nix
index d30f1444f..b984535c9 100644
--- a/shared/2configs/cgit-mirror.nix
+++ b/shared/2configs/cgit-mirror.nix
@@ -3,7 +3,7 @@
with config.krebs.lib;
let
rules = with git; singleton {
- user = [ git-sync ];
+ user = [ wolf-repo-sync ];
repo = [ stockholm-mirror ];
perm = push ''refs/*'' [ non-fast-forward create delete merge ];
};
@@ -22,14 +22,15 @@ let
};
};
- git-sync = {
- name = "git-sync";
+ wolf-repo-sync = {
+ name = "wolf-repo-sync";
mail = "spam@krebsco.de";
# TODO put git-sync pubkey somewhere more appropriate
- pubkey = ''ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCzUuzyoAhMgJmsiaTVWNSXqcrZNTpKpv0nfFBOMcNXUWEbvfAq5eNpg5cX+P8eoYl6UQgfftbYi06flKK3yJdntxoZKLwJGgJt9NZr8yZTsiIfMG8XosvGNQtGPkBtpLusgmPpu7t2RQ9QrqumBvoUDGYEauKTslLwupp1QeyWKUGEhihn4CuqQKiPrz+9vbNd75XOfVZMggk3j4F7HScatmA+p1EQXWyq5Jj78jQN5ZIRnHjMQcIZ4DOz1U96atwSKMviI1xEZIODYfgoGjjiWYeEtKaLVPtSqtLRGI7l+RNouMfwHLdTWOJSlIdFncfPXC6R19hTll3UHeHLtqLP git-sync'';
+ pubkey = ''ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCwuAZB3wtAvBJFYh+gWdyGaZU4mtqM2dFXmh2rORlbXeh02msu1uv07ck1VKkQ4LgvCBcBsAOeVa1NTz99eLqutwgcqMCytvRNUCibcoEWwHObsK53KhDJj+zotwlFhnPPeK9+EpOP4ngh/tprJikttos5BwBwe2K+lfiid3fmVPZcTTYa77nCwijimMvWEx6CEjq1wiXMUc4+qcEn8Swbwomz/EEQdNE2hgoC3iMW9RqduTFdIJWnjVi0KaxenX9CvQRGbVK5SSu2gwzN59D/okQOCP6+p1gL5r3QRHSLSSRiEHctVQTkpKOifrtLZGSr5zArEmLd/cOVyssHQPCX repo-sync@wolf'';
};
in {
+ krebs.users.wolf-repo-sync = wolf-repo-sync;
krebs.git = {
enable = true;
root-title = "Shared Repos";
diff --git a/shared/2configs/repo-sync.nix b/shared/2configs/repo-sync.nix
new file mode 100644
index 000000000..b23cb1675
--- /dev/null
+++ b/shared/2configs/repo-sync.nix
@@ -0,0 +1,28 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+{
+ krebs.repo-sync = let
+ # TODO addMirrorURL function
+ mirror = "git@wolf:stockholm-mirror";
+ in {
+ enable = true;
+ config = {
+ makefu = {
+ origin.url = http://cgit.gum/stockholm ;
+ mirror.url = mirror;
+ };
+ tv = {
+ origin.url = http://cgit.cd/stockholm ;
+ mirror.url = mirror;
+ };
+ lassulus = {
+ origin.url = http://cgit.cloudkrebs/stockholm ;
+ mirror.url = mirror;
+ };
+ "@latest" = {
+ mirror.url = mirror;
+ };
+ };
+ };
+}
diff --git a/shared/2configs/shack-drivedroid.nix b/shared/2configs/shack-drivedroid.nix
index 08a6b0697..6133ccc99 100644
--- a/shared/2configs/shack-drivedroid.nix
+++ b/shared/2configs/shack-drivedroid.nix
@@ -1,7 +1,8 @@
{ pkgs, lib, config, ... }:
+with config.krebs.lib;
let
repodir = "/var/srv/drivedroid";
- srepodir = lib.shell.escape repodir;
+ srepodir = shell.escape repodir;
in
{
environment.systemPackages = [ pkgs.drivedroid-gen-repo ];
@@ -40,5 +41,4 @@ in
};
};
};
-
}
diff --git a/shared/2configs/shared-buildbot.nix b/shared/2configs/shared-buildbot.nix
index af877f5d8..ebf5f4a1e 100644
--- a/shared/2configs/shared-buildbot.nix
+++ b/shared/2configs/shared-buildbot.nix
@@ -1,18 +1,22 @@
{ lib, config, pkgs, ... }:
-# The buildbot config is seilf-contained and provides a way to test "shared"
-# configuration (infrastructure to be used by every krebsminister).
+# The buildbot config is self-contained and currently provides a way
+# to test "shared" configuration (infrastructure to be used by every krebsminister).
# You can add your own test, test steps as required. Deploy the config on a
# shared host like wolf and everything should be fine.
+
+# TODO for all users schedule a build for fast tests
{
networking.firewall.allowedTCPPorts = [ 8010 9989 ];
- krebs.buildbot.master = {
+ krebs.buildbot.master = let
+ stockholm-mirror-url = http://cgit.wolf/stockholm-mirror ;
+ in {
secrets = [ "retiolum-ci.rsa_key.priv" "cac.json" ];
slaves = {
testslave = "krebspass";
};
change_source.stockholm = ''
- stockholm_repo = 'http://cgit.wolf/stockholm-mirror'
+ stockholm_repo = '${stockholm-mirror-url}'
cs.append(changes.GitPoller(
stockholm_repo,
workdir='stockholm-poller', branches=True,
@@ -23,16 +27,15 @@
force-scheduler = ''
sched.append(schedulers.ForceScheduler(
name="force",
- builderNames=["full-tests"]))
+ builderNames=["full-tests","fast-tests"]))
'';
fast-tests-scheduler = ''
- # test the master real quick
+ # test everything real quick
sched.append(schedulers.SingleBranchScheduler(
## all branches
change_filter=util.ChangeFilter(branch_re=".*"),
- # change_filter=util.ChangeFilter(branch="master"),
- treeStableTimer=10, #only test the latest push
- name="fast-master-test",
+ # treeStableTimer=10,
+ name="fast-test-all-branches",
builderNames=["fast-tests"]))
'';
test-cac-infest-master = ''
@@ -61,7 +64,7 @@
# prepare nix-shell
# the dependencies which are used by the test script
deps = [ "gnumake", "jq","nix","rsync",
- "(import <stockholm> {}).pkgs.test.infest-cac-centos7" ]
+ "(import <stockholm>).pkgs.test.infest-cac-centos7" ]
# TODO: --pure , prepare ENV in nix-shell command:
# SSL_CERT_FILE,LOGNAME,NIX_REMOTE
nixshell = ["nix-shell",
@@ -133,7 +136,7 @@
};
irc = {
enable = true;
- nick = "shared-buildbot";
+ nick = "wolfbot";
server = "cd.retiolum";
channels = [ "retiolum" ];
allowForce = true;
@@ -147,6 +150,7 @@
password = "krebspass";
packages = with pkgs;[ git nix ];
# all nix commands will need a working nixpkgs installation
- extraEnviron = { NIX_PATH="/var/src"; };
+ extraEnviron = {
+ NIX_PATH="nixpkgs=/var/src/upstream-nixpkgs:nixos-config=./shared/1systems/wolf.nix"; };
};
}