Merge remote-tracking branch 'cd/master'

6 files changed, 167 insertions, 76 deletions

diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix
index a1f446188..c700fbc56 100644
--- a/krebs/3modules/build.nix
+++ b/krebs/3modules/build.nix
@@ -20,35 +20,19 @@ let
       type = types.user;
     };
 
-    options.krebs.build.source = let
-      raw = types.either types.str types.path;
-      url = types.submodule {
+    options.krebs.build.source = mkOption {
+      type = with types; attrsOf (either str (submodule {
         options = {
-          url = mkOption {
-            type = types.str;
-          };
-          rev = mkOption {
-            type = types.str;
-          };
-          dev = mkOption {
-            type = types.str;
-          };
+          url = str;
+          rev = str;
         };
-      };
-    in mkOption {
-      type = types.attrsOf (types.either types.str url);
-      apply = let f = mapAttrs (_: value: {
-        string = value;
-        path = toString value;
-        set = f value;
-      }.${typeOf value}); in f;
+      }));
       default = {};
     };
 
     options.krebs.build.populate = mkOption {
       type = types.str;
       default = let
-        source = config.krebs.build.source;
         target-user = maybeEnv "target_user" "root";
         target-host = maybeEnv "target_host" config.krebs.build.host.name;
         target-port = maybeEnv "target_port" "22";
@@ -75,24 +59,21 @@ let
           tmpdir=$(mktemp -dt stockholm.XXXXXXXX)
           chmod 0755 "$tmpdir"
 
-          ${concatStringsSep "\n"
-            (mapAttrsToList
-              (name: spec: let dst = removePrefix "symlink:" (get-url spec); in
-                "verbose ln -s ${shell.escape dst} $tmpdir/${shell.escape name}")
-              symlink-specs)}
+          ${concatStringsSep "\n" (mapAttrsToList (name: symlink: ''
+            verbose ln -s ${shell.escape symlink.target} \
+                          "$tmpdir"/${shell.escape name}
+          '') source-by-method.symlink)}
 
           verbose proot \
-              -b $tmpdir:${shell.escape target-path} \
-              ${concatStringsSep " \\\n    "
-                (mapAttrsToList
-                  (name: spec:
-                    "-b ${shell.escape "${get-url spec}:${target-path}/${name}"}")
-                  file-specs)} \
+              -b "$tmpdir":${shell.escape target-path} \
+              ${concatStringsSep " \\\n    " (mapAttrsToList (name: file:
+                "-b ${shell.escape "${file.path}:${target-path}/${name}"}"
+              ) source-by-method.file)} \
               rsync \
                   -f ${shell.escape "P /*"} \
-                  ${concatMapStringsSep " \\\n        "
-                    (name: "-f ${shell.escape "R /${name}"}")
-                    (attrNames file-specs)} \
+                  ${concatMapStringsSep " \\\n        " (name:
+                    "-f ${shell.escape "R /${name}"}"
+                  ) (attrNames source-by-method.file)} \
                   --delete \
                   -vFrlptD \
                   -e ${shell.escape "ssh -p ${target-port}"} \
@@ -100,30 +81,6 @@ let
                   ${shell.escape "${target-user}@${target-host}:${target-path}"}
         '';
 
-        get-schema = uri:
-          if substring 0 1 uri == "/"
-            then "file"
-            else head (splitString ":" uri);
-
-        has-schema = schema: uri: get-schema uri == schema;
-
-        get-url = spec: {
-          string = spec;
-          path = toString spec;
-          set = get-url spec.url;
-        }.${typeOf spec};
-
-        git-specs =
-          filterAttrs (_: spec: has-schema "https" (get-url spec)) source //
-          filterAttrs (_: spec: has-schema "http" (get-url spec)) source //
-          filterAttrs (_: spec: has-schema "git" (get-url spec)) source;
-
-        file-specs =
-          filterAttrs (_: spec: has-schema "file" (get-url spec)) source;
-
-        symlink-specs =
-          filterAttrs (_: spec: has-schema "symlink" (get-url spec)) source;
-
         git-script = ''
           #! /bin/sh
           set -efu
@@ -162,20 +119,42 @@ let
             git clean -dxf
           )}
 
-          ${concatStringsSep "\n"
-            (mapAttrsToList
-              (name: spec: toString (map shell.escape [
-                "verbose"
-                "fetch_git"
-                "${target-path}/${name}"
-                spec.url
-                spec.rev
-              ]))
-              git-specs)}
+          ${concatStringsSep "\n" (mapAttrsToList (name: git: ''
+            verbose fetch_git ${concatMapStringsSep " " shell.escape [
+              "${target-path}/${name}"
+              git.url
+              git.rev
+            ]}
+          '') source-by-method.git)}
         '';
       in out;
     };
 
   };
 
+  source-by-method = let
+    known-methods = ["git" "file" "symlink"];
+  in genAttrs known-methods (const {}) // recursiveUpdate source-by-scheme {
+    git = source-by-scheme.http or {} //
+          source-by-scheme.https or {};
+  };
+
+  source-by-scheme = foldl' (out: { k, v }: recursiveUpdate out {
+    ${v.scheme}.${k} = v;
+  }) {} (mapAttrsToList (k: v: { inherit k v; }) normalized-source);
+
+  normalized-source = mapAttrs (name: let f = x: getAttr (typeOf x) {
+    path = f (toString x);
+    string = f {
+      url = if substring 0 1 x == "/" then "file://${x}" else x;
+    };
+    set = let scheme = head (splitString ":" x.url); in recursiveUpdate x {
+      inherit scheme;
+    } // {
+      symlink.target = removePrefix "symlink:" x.url;
+      file.path = # TODO file://host/...
+                  assert hasPrefix "file:///" x.url;
+                  removePrefix "file://" x.url;
+    }.${scheme} or {};
+  }; in f) config.krebs.build.source;
 in out
diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index 825cb3413..2a1dbe31a 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -338,8 +338,8 @@ let
         SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
       };
       serviceConfig = let
-        workdir="${lib.shell.escape cfg.workDir}";
-        secretsdir="${lib.shell.escape (toString <secrets>)}";
+        workdir = shell.escape cfg.workDir;
+        secretsdir = shell.escape (toString <secrets>);
       in {
         PermissionsStartOnly = true;
         Type = "forking";
diff --git a/krebs/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix
index 7705ac31c..248b46132 100644
--- a/krebs/3modules/buildbot/slave.nix
+++ b/krebs/3modules/buildbot/slave.nix
@@ -149,9 +149,9 @@ let
       } // cfg.extraEnviron;
 
       serviceConfig = let
-        workdir = "${lib.shell.escape cfg.workDir}";
-        contact = "${lib.shell.escape cfg.contact}";
-        description = "${lib.shell.escape cfg.description}";
+        workdir = shell.escape cfg.workDir;
+        contact = shell.escape cfg.contact;
+        description = shell.escape cfg.description;
         buildbot = pkgs.buildbot-slave;
         # TODO:make this
       in {
diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix
index 16a74e7c1..c06f3754e 100644
--- a/krebs/3modules/default.nix
+++ b/krebs/3modules/default.nix
@@ -31,6 +31,7 @@ let
       ./setuid.nix
       ./tinc_graphs.nix
       ./urlwatch.nix
+      ./repo-sync.nix
     ];
     options.krebs = api;
     config = lib.mkIf cfg.enable imp;
diff --git a/krebs/3modules/repo-sync.nix b/krebs/3modules/repo-sync.nix
new file mode 100644
index 000000000..7a7c80a75
--- /dev/null
+++ b/krebs/3modules/repo-sync.nix
@@ -0,0 +1,109 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  cfg = config.krebs.repo-sync;
+
+  out = {
+    options.krebs.repo-sync = api;
+    config = mkIf cfg.enable imp;
+  };
+
+  api = {
+    enable = mkEnableOption "repo-sync";
+    config = mkOption {
+      type = with types;attrsOf (attrsOf (attrsOf str));
+      example = literalExample ''
+        # see `repo-sync --help`
+        #   `ref` provides sane defaults and can be omitted
+
+        # attrset will be converted to json and be used as config
+        {
+            makefu = {
+                origin = {
+                    url = http://github.com/makefu/repo ;
+                    ref = "heads/dev" ;
+                };
+                mirror = {
+                    url = "git@internal:mirror" ;
+                    ref = "heads/github-mirror-dev" ;
+                };
+            };
+            lass = {
+                origin = {
+                    url = http://github.com/lass/repo ;
+                };
+                mirror = {
+                    url = "git@internal:mirror" ;
+                };
+            };
+            "@latest" = {
+                mirror = {
+                    url = "git@internal:mirror";
+                    ref = "heads/master";
+                };
+            };
+        };
+      '';
+    };
+    timerConfig = mkOption {
+      type = types.attrsOf types.str;
+      default = {
+        OnCalendar = "*:00,15,30,45";
+      };
+    };
+    stateDir = mkOption {
+      type = types.str;
+      default = "/var/lib/repo-sync";
+    };
+    privateKeyFile = mkOption {
+      type = types.str;
+      description = ''
+        used by repo-sync to identify with ssh service
+      '';
+      default = toString <secrets/wolf-repo-sync.rsa_key.priv>;
+    };
+  };
+  repo-sync-config = pkgs.writeText "repo-sync-config.json"
+    (builtins.toJSON cfg.config);
+
+  imp = {
+    users.users.repo-sync = {
+      name = "repo-sync";
+      uid = config.krebs.lib.genid "repo-sync";
+      description = "repo-sync user";
+      home = cfg.stateDir;
+      createHome = true;
+    };
+
+    systemd.timers.repo-sync = {
+      description = "repo-sync timer";
+      wantedBy = [ "timers.target" ];
+
+      timerConfig = cfg.timerConfig;
+    };
+    systemd.services.repo-sync = {
+      description = "repo-sync";
+      after = [ "network.target" ];
+
+      path = with pkgs; [ ];
+
+      environment = {
+        GIT_SSH_COMMAND = "${pkgs.openssh}/bin/ssh -i ${cfg.stateDir}/ssh.priv";
+      };
+
+      serviceConfig = {
+        Type = "simple";
+        PermissionsStartOnly = true;
+        ExecStartPre = pkgs.writeScript "prepare-repo-sync-user" ''
+          #! /bin/sh
+          cp -v ${config.krebs.lib.shell.escape cfg.privateKeyFile} ${cfg.stateDir}/ssh.priv
+          chown repo-sync ${cfg.stateDir}/ssh.priv
+        '';
+        ExecStart = "${pkgs.repo-sync}/bin/repo-sync ${repo-sync-config}";
+        WorkingDirectory = cfg.stateDir;
+        User = "repo-sync";
+      };
+    };
+  };
+in out
diff --git a/krebs/5pkgs/repo-sync/default.nix b/krebs/5pkgs/repo-sync/default.nix
index 90f838de9..789c03f36 100644
--- a/krebs/5pkgs/repo-sync/default.nix
+++ b/krebs/5pkgs/repo-sync/default.nix
@@ -1,15 +1,17 @@
 { lib, pkgs, python3Packages, fetchurl, ... }:
+
 with python3Packages; buildPythonPackage rec {
   name = "repo-sync-${version}";
-  version = "0.1.1";
+  version = "0.2.5";
   disabled = isPy26 || isPy27;
   propagatedBuildInputs = [
     docopt
     GitPython
+    pkgs.git
   ];
   src = fetchurl {
     url = "https://pypi.python.org/packages/source/r/repo-sync/repo-sync-${version}.tar.gz";
-    sha256 = "01r30l2bbsld90ps13ip0zi2a41b53dv4q6fxrzvkfrprr64c0vv";
+    sha256 = "1a59bj0vc5ajq8indkvkdk022yzvvv5mjb57hk3xf1j3wpr85p84";
   };
   meta = {
     homepage = http://github.com/makefu/repo-sync;