From 26d716c7812cde3b2b7ea184daf0b772c1e63ef2 Mon Sep 17 00:00:00 2001 From: tv Date: Wed, 13 Jul 2016 10:06:22 +0200 Subject: urlwatch: 2.2 -> 2.5 --- krebs/5pkgs/urlwatch/default.nix | 8 ++------ krebs/5pkgs/urlwatch/setup.patch | 42 ---------------------------------------- 2 files changed, 2 insertions(+), 48 deletions(-) delete mode 100644 krebs/5pkgs/urlwatch/setup.patch diff --git a/krebs/5pkgs/urlwatch/default.nix b/krebs/5pkgs/urlwatch/default.nix index ae1416204..7a4df5c7c 100644 --- a/krebs/5pkgs/urlwatch/default.nix +++ b/krebs/5pkgs/urlwatch/default.nix @@ -1,11 +1,11 @@ { stdenv, fetchurl, python3Packages }: python3Packages.buildPythonPackage rec { - name = "urlwatch-2.2"; + name = "urlwatch-2.5"; src = fetchurl { url = "https://thp.io/2008/urlwatch/${name}.tar.gz"; - sha256 = "0s9056mm1hkj5gpzsb5bz6fwxk0nm73i0dhnqwa1bfddjnvpl9d3"; + sha256 = "0qirpymdmpsx0klmhbx3icmiwpm6fx4wjma646gl9m90pifs8430"; }; propagatedBuildInputs = with python3Packages; [ @@ -15,10 +15,6 @@ python3Packages.buildPythonPackage rec { requests2 ]; - patches = [ - ./setup.patch - ]; - postFixup = '' wrapProgram "$out/bin/urlwatch" --prefix "PYTHONPATH" : "$PYTHONPATH" ''; diff --git a/krebs/5pkgs/urlwatch/setup.patch b/krebs/5pkgs/urlwatch/setup.patch deleted file mode 100644 index 66626dbf0..000000000 --- a/krebs/5pkgs/urlwatch/setup.patch +++ /dev/null @@ -1,42 +0,0 @@ -From ebe7b90100a3d960f53fdc9409d2d89eaa61bf11 Mon Sep 17 00:00:00 2001 -From: Thomas Perl -Date: Tue, 28 Jun 2016 18:15:51 +0200 -Subject: [PATCH] Check current directory and use os.path.relpath (Fixes #73) - ---- - setup.py | 11 ++++++++--- - 1 file changed, 8 insertions(+), 3 deletions(-) - -diff --git a/setup.py b/setup.py -index 947a7c8..45405cd 100644 ---- a/setup.py -+++ b/setup.py -@@ -7,10 +7,15 @@ - - import os - import re -+import sys - - PACKAGE_NAME = 'urlwatch' - DEPENDENCIES = ['minidb', 'PyYAML', 'requests'] --HERE = os.path.dirname(__file__) -+HERE = os.path.abspath(os.path.dirname(__file__)) -+ -+if os.path.normpath(os.getcwd()) != os.path.normpath(HERE): -+ print('You must run {} inside {} (cwd={})'.format(os.path.basename(__file__), HERE, os.getcwd())) -+ sys.exit(1) - - # Assumptions: - # 1. Package name equals main script file name (and only one script) -@@ -29,9 +34,9 @@ - - m['scripts'] = [os.path.join(HERE, PACKAGE_NAME)] - m['package_dir'] = {'': os.path.join(HERE, 'lib')} --m['packages'] = ['.'.join(dirname[len(HERE)+1:].split(os.sep)[1:]) -+m['packages'] = ['.'.join(os.path.relpath(dirname, HERE).split(os.sep)[1:]) - for dirname, _, files in os.walk(os.path.join(HERE, 'lib')) if '__init__.py' in files] --m['data_files'] = [(dirname[len(HERE)+1:], [os.path.join(dirname[len(HERE)+1:], fn) for fn in files]) -+m['data_files'] = [(os.path.relpath(dirname, HERE), [os.path.join(os.path.relpath(dirname, HERE), fn) for fn in files]) - for dirname, _, files in os.walk(os.path.join(HERE, 'share')) if files] - m['install_requires'] = DEPENDENCIES - -- cgit v1.2.3 From 447ead94a3ad13ac2691b5e5e996efa1cfcbdddd Mon Sep 17 00:00:00 2001 From: makefu Date: Fri, 15 Jul 2016 08:34:44 +0200 Subject: m wbob: start siem website --- makefu/5pkgs/awesomecfg/kiosk.lua | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/makefu/5pkgs/awesomecfg/kiosk.lua b/makefu/5pkgs/awesomecfg/kiosk.lua index 81ec99225..ec255a8af 100644 --- a/makefu/5pkgs/awesomecfg/kiosk.lua +++ b/makefu/5pkgs/awesomecfg/kiosk.lua @@ -521,13 +521,15 @@ awful.rules.rules = { } -- awful.util.spawn_with_shell("chromium --new-window --kiosk https://www.checkpoint.com/ThreatPortal/livemap.html") -awful.util.spawn_with_shell("chromium --new-window --kiosk http://wolf:3000/dashboard/db/soc-critical-values") +--awful.util.spawn_with_shell("chromium --new-window --kiosk http://wolf:3000/dashboard/db/soc-critical-values") -- awful.util.spawn_with_shell("sleep 0.5;chromium --new-window --kiosk http://wolf:3000/dashboard/db/aralast") --awful.util.spawn_with_shell("chromium --new-window --kiosk http://gast.aramark.de/thales-deutschland/menu/pdf/woche_de.php") -awful.util.spawn_with_shell("sleep 0.5;chromium --new-window --kiosk http://map.norsecorp.com") +--awful.util.spawn_with_shell("sleep 0.5;chromium --new-window --kiosk http://map.norsecorp.com") --awful.util.spawn_with_shell("sleep 0.5;chromium --new-window --kiosk http://threatmap.fortiguard.com") +awful.util.spawn_with_shell("chromium --new-window --kiosk 'https://ossim.siem/ossim/#dashboard/overview/overview'") +awful.util.spawn_with_shell("chromium --new-window --kiosk 'https://ossim.siem/ossim/#analysis/alarms/alarms'") -- }}} -- cgit v1.2.3 From d0e0cde0d0b07b6932b8b591de73a62e41b4abfb Mon Sep 17 00:00:00 2001 From: makefu Date: Fri, 15 Jul 2016 08:35:27 +0200 Subject: m 1 vbob: cleanup, add workaround for virtualbox --- makefu/1systems/vbob.nix | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/makefu/1systems/vbob.nix b/makefu/1systems/vbob.nix index 8b71b1393..3fcb173ce 100644 --- a/makefu/1systems/vbob.nix +++ b/makefu/1systems/vbob.nix @@ -5,23 +5,23 @@ imports = [ # Include the results of the hardware scan. ../. - + (toString ) + (toString ) ../2configs/main-laptop.nix #< base-gui + # (toString )/extra-hosts.nix # environment ]; - nixpkgs.config.allowUnfree = true; + # workaround for https://github.com/NixOS/nixpkgs/issues/16641 + services.xserver.videoDrivers = lib.mkOverride 45 [ "virtualbox" "modesetting" ]; + nixpkgs.config.allowUnfree = true; fileSystems."/nix" = { device ="/dev/disk/by-label/nixstore"; fsType = "ext4"; }; - fileSystems."/var/lib/docker" = { - device ="/dev/disk/by-label/nix-docker"; - fsType = "ext4"; - }; - #makefu.buildbot.master.enable = true; + # allow vbob to deploy self users.extraUsers = { root = { @@ -52,11 +52,7 @@ "gum" ]; }; - - networking.extraHosts = '' - 172.17.20.190 gitlab - 172.17.62.27 svbittool01 tool - ''; + virtualisation.docker.enable = false; fileSystems."/media/share" = { fsType = "vboxsf"; -- cgit v1.2.3 From 2ff9a5900870dfab9b69f7def3a7f0902e122220 Mon Sep 17 00:00:00 2001 From: makefu Date: Fri, 15 Jul 2016 08:37:37 +0200 Subject: m 5 git-xlsx-textconv: init --- makefu/5pkgs/git-xlsx-textconv/default.nix | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 makefu/5pkgs/git-xlsx-textconv/default.nix diff --git a/makefu/5pkgs/git-xlsx-textconv/default.nix b/makefu/5pkgs/git-xlsx-textconv/default.nix new file mode 100644 index 000000000..1f631f020 --- /dev/null +++ b/makefu/5pkgs/git-xlsx-textconv/default.nix @@ -0,0 +1,30 @@ +{ stdenv, lib, goPackages, fetchFromGitHub }: +let + go-xlsx = goPackages.buildGoPackage rec { + name = "go-xlsx-${version}"; + version = "46e6e472d"; + + goPackagePath = "github.com/tealeg/xlsx"; + src = fetchFromGitHub { + rev = version; + owner = "tealeg"; + repo = "xlsx"; + sha256 = "1vls05asms7azhyszbqpgdby9l45jpgisbzzmbrzi30n6cvs89zg"; + }; +}; +in +(goPackages.buildGoPackage rec { + name = "git-xlsx-textconv-${version}"; + version = "70685e7f8"; + + + goPackagePath = "github.com/tokuhirom/git-xlsx-textconv"; + + src = fetchFromGitHub { + rev = version; + owner = "tokuhirom"; + repo = "git-xlsx-textconv"; + sha256 = "055f3caj1y8v7sc2pz9q0dfyi2ij77d499pby4sjfvm5kjy9msdi"; + }; + propagatedBuildInputs = [ go-xlsx ]; +}).bin -- cgit v1.2.3 From 49ad0a990d00cc351e8339b9d22cfa79856008c1 Mon Sep 17 00:00:00 2001 From: tv Date: Sat, 16 Jul 2016 19:10:20 +0200 Subject: populate: init at 1.0.0 --- krebs/5pkgs/populate/default.nix | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 krebs/5pkgs/populate/default.nix diff --git a/krebs/5pkgs/populate/default.nix b/krebs/5pkgs/populate/default.nix new file mode 100644 index 000000000..0c73d936d --- /dev/null +++ b/krebs/5pkgs/populate/default.nix @@ -0,0 +1,35 @@ +{ coreutils, fetchgit, jq, openssh, proot, rsync, stdenv, ... }: + +let + PATH = stdenv.lib.makeBinPath [ + coreutils + jq + openssh + proot + rsync + ]; +in + +stdenv.mkDerivation rec { + name = "populate"; + version = "1.0.0"; + + src = fetchgit { + url = http://cgit.cd.krebsco.de/populate; + rev = "refs/tags/v${version}"; + sha256 = "12d8p8k6c8096rn423xjnbhbrxacbfpdm7c78g4g5j5nqxknghp5"; + }; + + phases = [ + "unpackPhase" + "installPhase" + ]; + + installPhase = '' + sed \ + '1s,.*,&\nPATH=${PATH},' \ + -i bin/populate + + cp -r . $out + ''; +} -- cgit v1.2.3 From e349c7467c56e2a288f90d9ffe0d5793126f4784 Mon Sep 17 00:00:00 2001 From: tv Date: Sat, 16 Jul 2016 19:10:49 +0200 Subject: tv git: add populate --- tv/2configs/git.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/tv/2configs/git.nix b/tv/2configs/git.nix index 9bcf8f31f..4bc971370 100644 --- a/tv/2configs/git.nix +++ b/tv/2configs/git.nix @@ -36,6 +36,7 @@ let much = {}; newsbot-js = {}; nixpkgs = {}; + populate.desc = "source code installer"; push = {}; regfish = {}; soundcloud = { -- cgit v1.2.3 From 514daf3d4611c3d6f451964b5f7ebce22219e6d3 Mon Sep 17 00:00:00 2001 From: tv Date: Sat, 16 Jul 2016 21:43:38 +0200 Subject: replace krebs.build.populate by populate --- Makefile | 23 ++--- krebs/3modules/build.nix | 138 +----------------------------- krebs/4lib/types.nix | 69 +++++++++++++++ krebs/5pkgs/test/infest-cac-centos7/notes | 4 +- lass/2configs/buildbot-standalone.nix | 7 +- lass/2configs/default.nix | 18 ++-- makefu/2configs/default.nix | 20 +++-- shared/2configs/base.nix | 13 ++- shared/2configs/shared-buildbot.nix | 8 +- tv/2configs/default.nix | 20 ++--- 10 files changed, 127 insertions(+), 193 deletions(-) diff --git a/Makefile b/Makefile index aa5d5d8ca..be069ec3e 100644 --- a/Makefile +++ b/Makefile @@ -68,23 +68,27 @@ evaluate = \ -I stockholm=$(stockholm) \ -E "let eval = import ; in with eval; $(1)" -execute = \ - result=$$($(call evaluate,config.krebs.build.$(1))) && \ - script=$$(echo "$$result" | jq -r .) && \ - echo "$$script" | PS5=% sh - ifeq ($(MAKECMDGOALS),) $(error No goals specified) endif # usage: make deploy system=foo [target_host=bar] deploy: ssh ?= ssh -deploy: - $(call execute,populate) +deploy: populate $(ssh) $(target_user)@$(target_host) -p $(target_port) \ env STOCKHOLM_VERSION="$$STOCKHOLM_VERSION" \ nixos-rebuild switch --show-trace -I $(target_path) +# usage: make populate system=foo +ifeq ($(debug),true) +populate: populate-flags = --debug +endif +populate: + source=$$($(call evaluate,config.krebs.build.source) --json --strict) && \ + echo $$source | populate \ + $(target_user)@$(target_host):$(target_port)$(target_path) \ + $(populate-flags) + # usage: make build.pkgs.get build build.:;@$(call build,$${expr-eval}) build.%:;@$(call build,$@) @@ -99,7 +103,7 @@ install: $(ssh) $(target_user)@$(target_host) -p $(target_port) \ env target_path=$(target_path) \ sh -s prepare < krebs/4lib/infest/prepare.sh - target_path=/mnt$(target_path) $(call execute,populate) + $(MAKE) populate target_path=/mnt$(target_path) $(ssh) $(target_user)@$(target_host) -p $(target_port) \ env NIXOS_CONFIG=$(target_path)/nixos-config \ STOCKHOLM_VERSION="$$STOCKHOLM_VERSION" \ @@ -117,8 +121,7 @@ $(error bad method: $(method)) endif endif test: ssh ?= ssh -test: - $(call execute,populate) +test: populate $(ssh) $(target_user)@$(target_host) -p $(target_port) \ $(command) --show-trace -I $(target_path) \ -A config.system.build.toplevel $(target_path)/stockholm diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix index 9cd095622..5924d1033 100644 --- a/krebs/3modules/build.nix +++ b/krebs/3modules/build.nix @@ -21,145 +21,9 @@ let }; options.krebs.build.source = mkOption { - type = with types; attrsOf (either str (submodule { - options = { - url = str; - rev = str; - }; - })); + type = types.attrsOf types.source; default = {}; }; - - options.krebs.build.populate = mkOption { - type = types.str; - default = let - target-user = maybeEnv "target_user" "root"; - target-host = maybeEnv "target_host" config.krebs.build.host.name; - target-port = maybeEnv "target_port" "22"; - target-path = maybeEnv "target_path" "/var/src"; - out = '' - #! /bin/sh - set -eu - - ssh=''${ssh-ssh} - - verbose() { - printf '%s%s\n' "$PS5$(printf ' %q' "$@")" >&2 - "$@" - } - - { printf 'PS5=%q%q\n' @ "$PS5" - echo ${shell.escape git-script} - } | verbose $ssh -p ${shell.escape target-port} \ - ${shell.escape "${target-user}@${target-host}"} -T - - unset tmpdir - trap ' - rm -f "$tmpdir"/* - rmdir "$tmpdir" - trap - EXIT INT QUIT - ' EXIT INT QUIT - tmpdir=$(mktemp -dt stockholm.XXXXXXXX) - chmod 0755 "$tmpdir" - - ${concatStringsSep "\n" (mapAttrsToList (name: symlink: '' - verbose ln -s ${shell.escape symlink.target} \ - "$tmpdir"/${shell.escape name} - '') source-by-method.symlink)} - - verbose proot \ - -b "$tmpdir":${shell.escape target-path} \ - ${concatStringsSep " \\\n " (mapAttrsToList (name: file: - "-b ${shell.escape "${file.path}:${target-path}/${name}"}" - ) source-by-method.file)} \ - rsync \ - -f ${shell.escape "P /*"} \ - ${concatMapStringsSep " \\\n " (name: - "-f ${shell.escape "R /${name}"}" - ) (attrNames source-by-method.file)} \ - --delete \ - -vFrlptD \ - -e "$ssh -p ${shell.escape target-port}" \ - ${shell.escape target-path}/ \ - ${shell.escape "${target-user}@${target-host}:${target-path}"} - ''; - - git-script = '' - #! /bin/sh - set -efu - - export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt - - verbose() { - printf '%s%s\n' "$PS5$(printf ' %q' "$@")" >&2 - "$@" - } - - fetch_git() {( - dst_dir=$1 - src_url=$2 - src_ref=$3 - - if ! test -e "$dst_dir"; then - git clone "$src_url" "$dst_dir" - fi - - cd "$dst_dir" - - if ! url=$(git config remote.origin.url); then - git remote add origin "$src_url" - elif test "$url" != "$src_url"; then - git remote set-url origin "$src_url" - fi - - # TODO resolve src_ref to commit hash - hash=$src_ref - - if ! test "$(git log --format=%H -1)" = "$hash"; then - git fetch origin - git checkout "$hash" -- "$dst_dir" - git checkout -f "$hash" - fi - - git clean -dxf - )} - - ${concatStringsSep "\n" (mapAttrsToList (name: git: '' - verbose fetch_git ${concatMapStringsSep " " shell.escape [ - "${target-path}/${name}" - git.url - git.rev - ]} - '') source-by-method.git)} - ''; - in out; - }; - - }; - - source-by-method = let - known-methods = ["git" "file" "symlink"]; - in genAttrs known-methods (const {}) // recursiveUpdate source-by-scheme { - git = source-by-scheme.http or {} // - source-by-scheme.https or {}; }; - source-by-scheme = foldl' (out: { k, v }: recursiveUpdate out { - ${v.scheme}.${k} = v; - }) {} (mapAttrsToList (k: v: { inherit k v; }) normalized-source); - - normalized-source = mapAttrs (name: let f = x: getAttr (typeOf x) { - path = f (toString x); - string = f { - url = if substring 0 1 x == "/" then "file://${x}" else x; - }; - set = let scheme = head (splitString ":" x.url); in recursiveUpdate x { - inherit scheme; - } // { - symlink.target = removePrefix "symlink:" x.url; - file.path = # TODO file://host/... - assert hasPrefix "file:///" x.url; - removePrefix "file://" x.url; - }.${scheme} or {}; - }; in f) config.krebs.build.source; in out diff --git a/krebs/4lib/types.nix b/krebs/4lib/types.nix index aa7b7a9f5..8906eff4a 100644 --- a/krebs/4lib/types.nix +++ b/krebs/4lib/types.nix @@ -188,6 +188,75 @@ types // rec { }; }); + + source = submodule ({ config, ... }: { + options = { + type = let + types = ["file" "git" "symlink"]; + in mkOption { + type = enum types; + default = let + cands = filter (k: config.${k} != null) types; + in + if length cands == 1 + then head cands + else throw "cannot determine type"; + }; + file = let + file-path = (file-source.getSubOptions "FIXME").path.type; + in mkOption { + type = nullOr (either file-source file-path); + default = null; + apply = x: + if file-path.check x + then { path = x; } + else x; + }; + git = mkOption { + type = nullOr git-source; + default = null; + }; + symlink = let + symlink-target = (symlink-source.getSubOptions "FIXME").target.type; + in mkOption { + type = nullOr (either symlink-source symlink-target); + default = null; + apply = x: + if symlink-target.check x + then { target = x; } + else x; + }; + }; + }); + + file-source = submodule { + options = { + path = mkOption { + type = absolute-pathname; + }; + }; + }; + + git-source = submodule { + options = { + ref = mkOption { + type = str; # TODO types.git.ref + }; + url = mkOption { + type = str; # TODO types.git.url + }; + }; + }; + + symlink-source = submodule { + options = { + target = mkOption { + type = pathname; # TODO relative-pathname + }; + }; + }; + + suffixed-str = suffs: mkOptionType { name = "string suffixed by ${concatStringsSep ", " suffs}"; diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes index ab6bc557c..2a3ebd6fc 100755 --- a/krebs/5pkgs/test/infest-cac-centos7/notes +++ b/krebs/5pkgs/test/infest-cac-centos7/notes @@ -138,8 +138,8 @@ ip=$(cac-api getserver $id | jq -r .ip) cat > shared/2configs/temp/dirs.nix <).pkgs.populate" ] # TODO: --pure , prepare ENV in nix-shell command: # SSL_CERT_FILE,LOGNAME,NIX_REMOTE nixshell = ["nix-shell", @@ -112,8 +112,7 @@ in { for i in [ "prism", "mors", "echelon" ]: addShell(f,name="populate-{}".format(i),env=env_lass, command=nixshell + \ - ["{}( make system={} eval.config.krebs.build.populate \ - | jq -er .)".format("!" if "failing" in i else "",i)]) + ["{}(make system={} populate debug=true)".format("!" if "failing" in i else "",i)]) addShell(f,name="build-test-minimal",env=env_lass, command=nixshell + \ @@ -146,7 +145,7 @@ in { masterhost = "localhost"; username = "testslave"; password = "lasspass"; - packages = with pkgs;[ git nix gnumake jq rsync ]; + packages = with pkgs; [ gnumake jq nix populate ]; extraEnviron = { NIX_PATH="nixpkgs=/var/src/nixpkgs"; }; diff --git a/lass/2configs/default.nix b/lass/2configs/default.nix index e3065ba84..b8c50f1aa 100644 --- a/lass/2configs/default.nix +++ b/lass/2configs/default.nix @@ -53,16 +53,14 @@ with config.krebs.lib; search-domain = "retiolum"; build = { user = config.krebs.users.lass; - source = mapAttrs (_: mkDefault) ({ - nixos-config = "symlink:stockholm/lass/1systems/${config.krebs.build.host.name}.nix"; - secrets = if getEnv "dummy_secrets" == "true" - then toString - else "/home/lass/secrets/${config.krebs.build.host.name}"; - #secrets-common = "/home/lass/secrets/common"; - stockholm = getEnv "PWD"; - } // optionalAttrs config.krebs.build.host.secure { - #secrets-master = "/home/lass/secrets/master"; - }); + source = let inherit (config.krebs.build) host; in { + nixos-config.symlink = "stockholm/lass/1systems/${host.name}.nix"; + secrets.file = + if getEnv "dummy_secrets" == "true" + then toString + else "/home/lass/secrets/${host.name}"; + stockholm.file = getEnv "PWD"; + }; }; }; diff --git a/makefu/2configs/default.nix b/makefu/2configs/default.nix index 422927b28..58a537a2b 100644 --- a/makefu/2configs/default.nix +++ b/makefu/2configs/default.nix @@ -19,20 +19,22 @@ with config.krebs.lib; dns.providers.siem = "hosts"; search-domain = "retiolum"; - build = { + build = { user = config.krebs.users.makefu; - source = mapAttrs (_: mkDefault) { - nixpkgs = { + source = let inherit (config.krebs.build) host user; in { + nixpkgs.git = { url = https://github.com/nixos/nixpkgs; - rev = "63b9785"; # stable @ 2016-06-01 + ref = "63b9785"; # stable @ 2016-06-01 }; - secrets = if getEnv "dummy_secrets" == "true" - then toString - else "/home/makefu/secrets/${config.krebs.build.host.name}"; - stockholm = "/home/makefu/stockholm"; + secrets.file = + if getEnv "dummy_secrets" == "true" + then toString + else "/home/makefu/secrets/${host.name}"; + stockholm.file = "/home/makefu/stockholm"; # Defaults for all stockholm users? - nixos-config = "symlink:stockholm/${config.krebs.build.user.name}/1systems/${config.krebs.build.host.name}.nix"; + nixos-config.symlink = + "stockholm/${user.name}/1systems/${host.name}.nix"; }; }; }; diff --git a/shared/2configs/base.nix b/shared/2configs/base.nix index bbb089c2c..a92a0df35 100644 --- a/shared/2configs/base.nix +++ b/shared/2configs/base.nix @@ -7,15 +7,14 @@ with config.krebs.lib; # TODO rename shared user to "krebs" krebs.build.user = mkDefault config.krebs.users.shared; - krebs.build.source = { - nixpkgs = mkDefault { + krebs.build.source = let inherit (config.krebs.build) host user; in { + nixos-config.symlink = "stockholm/${user.name}/1systems/${host.name}.nix"; + nixpkgs.git = { url = https://github.com/NixOS/nixpkgs; - rev = "63b9785"; # stable @ 2016-06-01 + ref = "63b9785"; # stable @ 2016-06-01 }; - secrets = mkDefault "${getEnv "HOME"}/secrets/krebs/${config.krebs.build.host.name}"; - stockholm = mkDefault "${getEnv "HOME"}/stockholm"; - - nixos-config = "symlink:stockholm/${config.krebs.build.user.name}/1systems/${config.krebs.build.host.name}.nix"; + secrets.file = "${getEnv "HOME"}/secrets/krebs/${host.name}"; + stockholm.file = "${getEnv "HOME"}/stockholm"; }; networking.hostName = config.krebs.build.host.name; diff --git a/shared/2configs/shared-buildbot.nix b/shared/2configs/shared-buildbot.nix index 6c40d9966..688f8f9aa 100644 --- a/shared/2configs/shared-buildbot.nix +++ b/shared/2configs/shared-buildbot.nix @@ -75,7 +75,8 @@ # prepare nix-shell # the dependencies which are used by the test script - deps = [ "gnumake", "jq","nix","rsync", + deps = [ "gnumake", "jq", "nix", + "(import ).pkgs.populate", "(import ).pkgs.test.infest-cac-centos7" ] # TODO: --pure , prepare ENV in nix-shell command: # SSL_CERT_FILE,LOGNAME,NIX_REMOTE @@ -95,8 +96,7 @@ for i in [ "test-centos7", "wolf", "test-failing" ]: addShell(f,name="populate-{}".format(i),env=env, command=nixshell + \ - ["{}( make system={} eval.config.krebs.build.populate \ - | jq -er .)".format("!" if "failing" in i else "",i)]) + ["{}(make system={} populate debug=true)".format("!" if "failing" in i else "",i)]) # XXX we must prepare ./retiolum.rsa_key.priv for secrets to work addShell(f,name="instantiate-test-all-modules",env=env, @@ -179,7 +179,7 @@ masterhost = "localhost"; username = "testslave"; password = "krebspass"; - packages = with pkgs;[ git nix gnumake jq rsync ]; + packages = with pkgs; [ gnumake jq nix populate ]; # all nix commands will need a working nixpkgs installation extraEnviron = { NIX_PATH="nixpkgs=/var/src/nixpkgs:nixos-config=./shared/1systems/wolf.nix"; }; diff --git a/tv/2configs/default.nix b/tv/2configs/default.nix index a9ba1eadd..04009f54d 100644 --- a/tv/2configs/default.nix +++ b/tv/2configs/default.nix @@ -7,18 +7,18 @@ with config.krebs.lib; krebs.build = { user = config.krebs.users.tv; - source = mapAttrs (_: mkDefault) ({ - nixos-config = "symlink:stockholm/tv/1systems/${config.krebs.build.host.name}.nix"; - secrets = "/home/tv/secrets/${config.krebs.build.host.name}"; - secrets-common = "/home/tv/secrets/common"; - stockholm = "/home/tv/stockholm"; - nixpkgs = { + source = let inherit (config.krebs.build) host; in { + nixos-config.symlink = "stockholm/tv/1systems/${host.name}.nix"; + secrets.file = "/home/tv/secrets/${host.name}"; + secrets-common.file = "/home/tv/secrets/common"; + stockholm.file = "/home/tv/stockholm"; + nixpkgs.git = { url = https://github.com/NixOS/nixpkgs; - rev = "8bf31d7d27cae435d7c1e9e0ccb0a320b424066f"; + ref = "8bf31d7d27cae435d7c1e9e0ccb0a320b424066f"; }; - } // optionalAttrs config.krebs.build.host.secure { - secrets-master = "/home/tv/secrets/master"; - }); + } // optionalAttrs host.secure { + secrets-master.file = "/home/tv/secrets/master"; + }; }; networking.hostName = config.krebs.build.host.name; -- cgit v1.2.3 From 6f51a211251c40846b9a68666a3dd4f7a02aec67 Mon Sep 17 00:00:00 2001 From: tv Date: Sat, 16 Jul 2016 23:59:56 +0200 Subject: populate: 1.0.0 -> 1.1.0 --- krebs/5pkgs/populate/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/krebs/5pkgs/populate/default.nix b/krebs/5pkgs/populate/default.nix index 0c73d936d..af7b05227 100644 --- a/krebs/5pkgs/populate/default.nix +++ b/krebs/5pkgs/populate/default.nix @@ -12,12 +12,12 @@ in stdenv.mkDerivation rec { name = "populate"; - version = "1.0.0"; + version = "1.1.0"; src = fetchgit { url = http://cgit.cd.krebsco.de/populate; rev = "refs/tags/v${version}"; - sha256 = "12d8p8k6c8096rn423xjnbhbrxacbfpdm7c78g4g5j5nqxknghp5"; + sha256 = "19drrzji4zdpfn9r1cfms8j4jrn7xgqpyrs6bfd5zszrx4wyw3gw"; }; phases = [ -- cgit v1.2.3 From 9c4ed31565f3491f055963bcd7869bd0c1e62aa5 Mon Sep 17 00:00:00 2001 From: tv Date: Sun, 17 Jul 2016 00:31:53 +0200 Subject: make populate: drop redundant variable: source --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index be069ec3e..064cba484 100644 --- a/Makefile +++ b/Makefile @@ -84,10 +84,10 @@ ifeq ($(debug),true) populate: populate-flags = --debug endif populate: - source=$$($(call evaluate,config.krebs.build.source) --json --strict) && \ - echo $$source | populate \ - $(target_user)@$(target_host):$(target_port)$(target_path) \ - $(populate-flags) + $(call evaluate,config.krebs.build.source) --json --strict | \ + populate \ + $(target_user)@$(target_host):$(target_port)$(target_path) \ + $(populate-flags) # usage: make build.pkgs.get build build.:;@$(call build,$${expr-eval}) -- cgit v1.2.3 From d8616439191014697d3f99776b5c51a2d799d907 Mon Sep 17 00:00:00 2001 From: tv Date: Sun, 17 Jul 2016 00:35:30 +0200 Subject: krebs.build: simplify structure --- krebs/3modules/build.nix | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix index 5924d1033..bc04345d6 100644 --- a/krebs/3modules/build.nix +++ b/krebs/3modules/build.nix @@ -1,29 +1,28 @@ -{ config, lib, ... }: +{ config, ... }: with config.krebs.lib; -let - out = { +{ + options.krebs.build = { # TODO deprecate krebs.build.host - options.krebs.build.host = mkOption { + host = mkOption { type = types.host; }; # TODO make krebs.build.profile shell safe - options.krebs.build.profile = mkOption { + profile = mkOption { type = types.str; default = "/nix/var/nix/profiles/system"; }; - # TODO deprecate krebs.build.user - options.krebs.build.user = mkOption { - type = types.user; - }; - - options.krebs.build.source = mkOption { + source = mkOption { type = types.attrsOf types.source; default = {}; }; - }; -in out + # TODO deprecate krebs.build.user + user = mkOption { + type = types.user; + }; + }; +} -- cgit v1.2.3 From 0bc55750cfe132aaf7314a60dc26765766369aa6 Mon Sep 17 00:00:00 2001 From: tv Date: Sun, 17 Jul 2016 00:38:59 +0200 Subject: krebs.build.profile :: str => absolute-path --- krebs/3modules/build.nix | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix index bc04345d6..4848748cd 100644 --- a/krebs/3modules/build.nix +++ b/krebs/3modules/build.nix @@ -9,9 +9,8 @@ with config.krebs.lib; type = types.host; }; - # TODO make krebs.build.profile shell safe profile = mkOption { - type = types.str; + type = types.absolute-path; default = "/nix/var/nix/profiles/system"; }; -- cgit v1.2.3 From a428b4dcbb92ddd90f184f72a97ffdb300083e6e Mon Sep 17 00:00:00 2001 From: tv Date: Sun, 17 Jul 2016 00:52:21 +0200 Subject: make build -> make pkgs --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 064cba484..510d7b5c9 100644 --- a/Makefile +++ b/Makefile @@ -57,7 +57,7 @@ build = \ --show-trace \ -I nixos-config=$(nixos-config) \ -I stockholm=$(stockholm) \ - -E "let build = import ; in $(1)" + -E "with import ; $(1)" evaluate = \ nix-instantiate \ @@ -89,9 +89,9 @@ populate: $(target_user)@$(target_host):$(target_port)$(target_path) \ $(populate-flags) -# usage: make build.pkgs.get -build build.:;@$(call build,$${expr-eval}) -build.%:;@$(call build,$@) +# usage: make pkgs.populate +pkgs:;@$(error no package selected) +pkgs.%:;@$(call build,$@) # usage: make LOGNAME=shared system=wolf eval.config.krebs.build.host.name eval eval.:;@$(call evaluate,$${expr-eval}) -- cgit v1.2.3 From b8d9b1507e444b2a80864c87769c67409d5c7f6e Mon Sep 17 00:00:00 2001 From: tv Date: Sun, 17 Jul 2016 01:12:30 +0200 Subject: populate: add git to PATH --- krebs/5pkgs/populate/default.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/krebs/5pkgs/populate/default.nix b/krebs/5pkgs/populate/default.nix index af7b05227..e68efa594 100644 --- a/krebs/5pkgs/populate/default.nix +++ b/krebs/5pkgs/populate/default.nix @@ -1,8 +1,9 @@ -{ coreutils, fetchgit, jq, openssh, proot, rsync, stdenv, ... }: +{ coreutils, fetchgit, git, jq, openssh, proot, rsync, stdenv, ... }: let PATH = stdenv.lib.makeBinPath [ coreutils + git jq openssh proot -- cgit v1.2.3 From 534850dc5723775706ae2d7121ccef4b1768956c Mon Sep 17 00:00:00 2001 From: tv Date: Sun, 17 Jul 2016 01:26:26 +0200 Subject: populate: 1.1.0 -> 1.1.1 --- krebs/5pkgs/populate/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/krebs/5pkgs/populate/default.nix b/krebs/5pkgs/populate/default.nix index e68efa594..09b29c6c3 100644 --- a/krebs/5pkgs/populate/default.nix +++ b/krebs/5pkgs/populate/default.nix @@ -13,12 +13,12 @@ in stdenv.mkDerivation rec { name = "populate"; - version = "1.1.0"; + version = "1.1.1"; src = fetchgit { url = http://cgit.cd.krebsco.de/populate; rev = "refs/tags/v${version}"; - sha256 = "19drrzji4zdpfn9r1cfms8j4jrn7xgqpyrs6bfd5zszrx4wyw3gw"; + sha256 = "139f4lzn56lca3qgqy9g33r94m3xi1mqns9340lkb4qm6626yvqd"; }; phases = [ -- cgit v1.2.3 From 10ff2cf70a11ae7a624d34c09bd04ff3f1b51983 Mon Sep 17 00:00:00 2001 From: tv Date: Sun, 17 Jul 2016 01:30:11 +0200 Subject: make populate: admit $ssh --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 510d7b5c9..8279637cf 100644 --- a/Makefile +++ b/Makefile @@ -83,6 +83,9 @@ deploy: populate ifeq ($(debug),true) populate: populate-flags = --debug endif +ifneq ($(ssh),) +populate: populate-flags += --ssh=$(ssh) +endif populate: $(call evaluate,config.krebs.build.source) --json --strict | \ populate \ -- cgit v1.2.3 From 0b84e3f161cee0ac20c6a85f18ad946ce980e0c2 Mon Sep 17 00:00:00 2001 From: tv Date: Sun, 17 Jul 2016 02:37:46 +0200 Subject: Makefile: define default target --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 8279637cf..e6a8eeb95 100644 --- a/Makefile +++ b/Makefile @@ -51,6 +51,8 @@ $(if $(target_user),,$(error unbound variable: target_user)) $(if $(target_port),,$(error unbound variable: target_port)) $(if $(target_path),,$(error unbound variable: target_path)) +export target ?= $(target_user)@$(target_host):$(target_port)$(target_path) + build = \ nix-build \ --no-out-link \ @@ -81,16 +83,14 @@ deploy: populate # usage: make populate system=foo ifeq ($(debug),true) -populate: populate-flags = --debug +populate: populate-flags += --debug endif ifneq ($(ssh),) populate: populate-flags += --ssh=$(ssh) endif populate: $(call evaluate,config.krebs.build.source) --json --strict | \ - populate \ - $(target_user)@$(target_host):$(target_port)$(target_path) \ - $(populate-flags) + populate $(target) $(populate-flags) # usage: make pkgs.populate pkgs:;@$(error no package selected) -- cgit v1.2.3 From 14ccdf48b61e400fd45e180cd0a16e8f99bd0678 Mon Sep 17 00:00:00 2001 From: tv Date: Sun, 17 Jul 2016 02:58:20 +0200 Subject: make deploy: admit debug --- Makefile | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index e6a8eeb95..3857a2390 100644 --- a/Makefile +++ b/Makefile @@ -41,17 +41,17 @@ target_path ?= $(_target_path) endif endif -export target_host ?= $(system) -export target_user ?= root -export target_port ?= 22 -export target_path ?= /var/src +target_host ?= $(system) +target_user ?= root +target_port ?= 22 +target_path ?= /var/src $(if $(target_host),,$(error unbound variable: target_host)) $(if $(target_user),,$(error unbound variable: target_user)) $(if $(target_port),,$(error unbound variable: target_port)) $(if $(target_path),,$(error unbound variable: target_path)) -export target ?= $(target_user)@$(target_host):$(target_port)$(target_path) +target ?= $(target_user)@$(target_host):$(target_port)$(target_path) build = \ nix-build \ @@ -75,11 +75,17 @@ $(error No goals specified) endif # usage: make deploy system=foo [target_host=bar] +ifeq ($(debug),true) +deploy: rebuild-command = dry-activate +else +deploy: rebuild-command = switch +endif deploy: ssh ?= ssh -deploy: populate +deploy: + $(MAKE) populate debug=false $(ssh) $(target_user)@$(target_host) -p $(target_port) \ env STOCKHOLM_VERSION="$$STOCKHOLM_VERSION" \ - nixos-rebuild switch --show-trace -I $(target_path) + nixos-rebuild $(rebuild-command) --show-trace -I $(target_path) # usage: make populate system=foo ifeq ($(debug),true) -- cgit v1.2.3 From e2157dade8a359f81b1e6260f3c9c6e8d36360e5 Mon Sep 17 00:00:00 2001 From: tv Date: Sun, 17 Jul 2016 19:55:38 +0200 Subject: alnus: init --- krebs/3modules/tv/default.nix | 27 +++++++++++ tv/1systems/alnus.nix | 103 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 tv/1systems/alnus.nix diff --git a/krebs/3modules/tv/default.nix b/krebs/3modules/tv/default.nix index 075066961..d04f1cab2 100644 --- a/krebs/3modules/tv/default.nix +++ b/krebs/3modules/tv/default.nix @@ -7,6 +7,30 @@ with config.krebs.lib; "viljetic.de" = "regfish"; }; hosts = mapAttrs (_: setAttr "owner" config.krebs.users.tv) { + alnus = { + cores = 2; + nets = { + retiolum = { + ip4.addr = "10.243.21.1"; + ip6.addr = "42:0:0:0:0:0:0:2101"; + aliases = [ + "alnus.r" + ]; + tinc.pubkey = '' + -----BEGIN RSA PUBLIC KEY----- + MIIBCgKCAQEAyDGucukxY1xFSkqDaicpiCXZe3NX1Max7N+E9PKXO2yE0EFoGdUP + /4hZFO9IbteDwlsTd/RQIhhUWF818TLWzwasUxgmqBFN4d23IIDLHJxgRZ8cPzAs + gmBWwnVWRetDETc6HZK6m2rLU6PG53rRLvheZHW/B9nSfUp7n+puehJdGLnBQ8W+ + q5d/yUmN8hqS6h62yfAZEJSr7Gh/AW6Irmf3gjKRJlRmD2z28hR5tFH+Q/ulxJXQ + rNVzusASjRBO9VYOSWnNWI3Zl9vaUtbtEnvyl3PaV9N3gcHzB2HHlyDIotjqXvxU + cPLMN0lWOZeDae/9SDT62l/YuETYQo6TxwIDAQAB + -----END RSA PUBLIC KEY----- + ''; + }; + }; + ssh.privkey.path = ; + ssh.pubkey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDP9JS2Nyjx4Pn+/4MrFi1EvBBYVKkGm2Q4lhgaAiSuiGLol53OSsL2KIo01mbcSSBWow9QpQpn8KDoRnT2aMLDrdTFqL20ztDLOXmtrSsz3flgCjmW4f6uOaoZF0RNjAybd1coqwSJ7EINugwoqOsg1zzN2qeIGKYFvqFIKibYFAnQ8hcksmkvPdIO5O8CbdIiP9sZSrSDp0ZyLK2T0PML2jensVZOeqSPulQDFqLsbmavpVLkpDjdzzPRwbZWNB4++YeipbYNOkX4GR1EB4wMZ93IbBV7kpJtib2Zb2AnUf7UW37hxWBjILdstj9ClwNOQggn8kD9ub7YxBzH1dz0Xd8a0mPOAWIDJz9MypXgFRc3vdvPB/W1I4Se0CLbgOkORun9CkgijKr9oEY8JNt8HFd6viZcAaQxOyIm6PNHZTnHfdSc7bIBS2n3e3IZBv0fTd77knGLXg402aTuu2bm/kxsKivxsILXIaGbeXe4ceN3Fynr3FzSM2bUkzHb0mAHu1BQ9YaX0xzCwjVueA5nzGls7ODSFkXsiBfg2FvMN/sTLFca6tnwyqcnD6nujoiS5+BxjDWPgnZYqCaW3B/IkpTsRMsX6QrfhOFcsP8qlJ2Cp82orWoDK/D0vZ9pdzAc6PFGga0RofuJKY2yiq+SRZ7/e9E6VncIVCYZ1OfN0Q=="; + }; caxi = { cores = 2; extraZones = { @@ -391,6 +415,9 @@ with config.krebs.lib; }; }; users = { + dv = { + mail = "dv@alnus.r"; + }; mv = { mail = "mv@cd.r"; pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGer9e2+Lew7vnisgBbsFNECEIkpNJgEaqQqgb9inWkQ mv@vod"; diff --git a/tv/1systems/alnus.nix b/tv/1systems/alnus.nix new file mode 100644 index 000000000..360390c09 --- /dev/null +++ b/tv/1systems/alnus.nix @@ -0,0 +1,103 @@ +{ config, pkgs, ... }: + +with config.krebs.lib; + +{ + imports = [ + ../. + ../2configs/hw/x220.nix + ../2configs/exim-retiolum.nix + ../2configs/retiolum.nix + ]; + + # TODO remove non-hardware stuff from ../2configs/hw/x220.nix + # networking.wireless.enable collides with networkmanager + networking.wireless.enable = mkForce false; + + boot = { + initrd = { + availableKernelModules = [ "ahci" ]; + luks = { + cryptoModules = [ "aes" "sha512" "xts" ]; + devices = [ { name = "luksroot"; device = "/dev/sda2"; } ]; + }; + }; + loader = { + efi.canTouchEfiVariables = true; + gummiboot.enable = true; + }; + }; + + environment.systemPackages = with pkgs; [ + chromium + firefoxWrapper + networkmanagerapplet + pidginotr + pidgin-with-plugins + ]; + + fileSystems = { + "/boot" = { + device = "/dev/sda1"; + }; + "/" = { + device = "/dev/mapper/main-root"; + fsType = "btrfs"; + options = [ "defaults" "noatime" ]; + }; + "/home" = { + device = "/dev/mapper/main-home"; + fsType = "btrfs"; + options = [ "defaults" "noatime" ]; + }; + }; + + hardware = { + enableAllFirmware = true; + opengl.driSupport32Bit = true; + pulseaudio.enable = true; + }; + + i18n.defaultLocale = "de_DE.UTF-8"; + + krebs.build = { + host = config.krebs.hosts.alnus; + user = mkForce config.krebs.users.dv; + source.nixpkgs.git.ref = mkForce "d7450443c42228832c68fba203a7c15cfcfb264e"; + }; + + networking.networkmanager.enable = true; + + nixpkgs.config = { + allowUnfree = true; + chromium.enablePepperFlash = true; + firefox.enableAdobeFlash = true; + }; + + services.xserver = { + enable = true; + layout = "de"; + xkbOptions = "eurosign:e"; + synaptics = { + enable = true; + twoFingerScroll = true; + }; + desktopManager.xfce.enable = true; + displayManager.auto = { + enable = true; + user = "dv"; + }; + }; + + swapDevices =[ ]; + + users.users.dv = { + inherit (config.krebs.users.dv) home uid; + isNormalUser = true; + extraGroups = [ + "audio" + "video" + "networkmanager" + ]; + }; +} -- cgit v1.2.3 From d9028c51320d29d8ed15dff7a9fb7a3eb16f52ff Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 18 Jul 2016 12:14:05 +0200 Subject: m 2 default: lan dns provider --- makefu/2configs/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/makefu/2configs/default.nix b/makefu/2configs/default.nix index acd34b0d3..25c2996c6 100644 --- a/makefu/2configs/default.nix +++ b/makefu/2configs/default.nix @@ -18,6 +18,7 @@ with config.krebs.lib; enable = true; dns.providers.siem = "hosts"; + dns.providers.lan = "hosts"; search-domain = "retiolum"; build = { user = config.krebs.users.makefu; -- cgit v1.2.3