From 6ab41de256066d9870a8f2e260781a9a10365a94 Mon Sep 17 00:00:00 2001
From: lassulus <lass@aidsballs.de>
Date: Sun, 13 Dec 2015 15:08:35 +0100
Subject: [PATCH 01/65] l 5 xmonad: add binding for mute buttons
---
lass/5pkgs/xmonad-lass/Main.hs | 2 ++
1 file changed, 2 insertions(+)
diff --git a/lass/5pkgs/xmonad-lass/Main.hs b/lass/5pkgs/xmonad-lass/Main.hs
index ce5afe33a..faaa00aab 100644
--- a/lass/5pkgs/xmonad-lass/Main.hs
+++ b/lass/5pkgs/xmonad-lass/Main.hs
@@ -125,6 +125,8 @@ myKeyMap =
--, ("M4-r", spawn "exe=$(yeganesh -x) && eval \"exec $exe\"")
, ("<XF86AudioRaiseVolume>", spawn "pactl -- set-sink-volume 0 +4%")
, ("<XF86AudioLowerVolume>", spawn "pactl -- set-sink-volume 0 -4%")
+ , ("<XF86AudioMute>", spawn "pactl -- set-sink-mute 0 toggle")
+ , ("<XF86AudioMicMute>", spawn "pactl -- set-source-mute 1 toggle")
, ("<XF86Launch1>", gridselectWorkspace myWSConfig W.view)
, ("M4-a", focusUrgent)
From 9c1207a52825da2f7d9c55304f864d68055cedb8 Mon Sep 17 00:00:00 2001
From: lassulus <lass@aidsballs.de>
Date: Mon, 14 Dec 2015 14:06:05 +0100
Subject: [PATCH 02/65] l 5 newsbot-js: rev b227296 -> 6ee4884
---
lass/5pkgs/newsbot-js/default.nix | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lass/5pkgs/newsbot-js/default.nix b/lass/5pkgs/newsbot-js/default.nix
index ace2a976f..0d194e6f3 100644
--- a/lass/5pkgs/newsbot-js/default.nix
+++ b/lass/5pkgs/newsbot-js/default.nix
@@ -26,8 +26,8 @@ in nodePackages.buildNodePackage {
src = fetchgit {
url = "http://cgit.echelon/newsbot-js/";
- rev = "b22729670236bfa6491207d57c5d7565137625ca";
- sha256 = "8ff00de56d85543399776c82d41d92ccc68000e5dce0f008d926748e188f3c69";
+ rev = "6ee488430c6915eeae03f1569084577d39cef51d";
+ sha256 = "00xmn7hzcs0mm6hjf5i37d9nna5rcd0gra0ynch7x2id8liazksx";
};
phases = [
From 57feffb3f65876cca3f10ef82e6e82283c02852d Mon Sep 17 00:00:00 2001
From: lassulus <lass@aidsballs.de>
Date: Mon, 21 Dec 2015 13:47:22 +0100
Subject: [PATCH 03/65] l 2 base: nixpkgs rev: 363c843 -> 93d8671
---
lass/2configs/base.nix | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lass/2configs/base.nix b/lass/2configs/base.nix
index 40f4e12c7..66e12b262 100644
--- a/lass/2configs/base.nix
+++ b/lass/2configs/base.nix
@@ -50,7 +50,7 @@ with lib;
source = {
git.nixpkgs = {
url = https://github.com/Lassulus/nixpkgs;
- rev = "363c8430f1efad8b03d5feae6b3a4f2fe7b29251";
+ rev = "93d8671e2c6d1d25f126ed30e5e6f16764330119";
};
dir.secrets = {
host = config.krebs.hosts.mors;
From 14d1655deb456d6be95463af2ca1524f7a1b7a98 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Tue, 22 Dec 2015 16:30:01 +0100
Subject: [PATCH 04/65] s 1 test-centos7: prepare for ci
---
shared/1systems/test-centos7.nix | 3 ++-
shared/2configs/base.nix | 4 ++--
shared/2configs/temp/dirs.nix | 1 +
shared/2configs/temp/networking.nix | 1 +
4 files changed, 6 insertions(+), 3 deletions(-)
create mode 100644 shared/2configs/temp/dirs.nix
create mode 100644 shared/2configs/temp/networking.nix
diff --git a/shared/1systems/test-centos7.nix b/shared/1systems/test-centos7.nix
index 077a5d61b..48cecc877 100644
--- a/shared/1systems/test-centos7.nix
+++ b/shared/1systems/test-centos7.nix
@@ -7,7 +7,8 @@ in {
imports = [
../2configs/base.nix
../2configs/os-templates/CAC-CentOS-7-64bit.nix
- ../2configs/os-templates/temp-networking.nix
+ ../2configs/temp/networking.nix
+ ../2configs/temp/dirs.nix
];
sound.enable = false;
diff --git a/shared/2configs/base.nix b/shared/2configs/base.nix
index df41eae1a..fceea67d1 100644
--- a/shared/2configs/base.nix
+++ b/shared/2configs/base.nix
@@ -20,11 +20,11 @@ with lib;
};
dir.secrets = {
host = config.krebs.current.host;
- path = "${getEnv "HOME"}/secrets/krebs/wolf";
+ path = mkDefault "${getEnv "HOME"}/secrets/krebs/wolf";
};
dir.stockholm = {
host = config.krebs.current.host;
- path = "${getEnv "HOME"}/stockholm";
+ path = mkDefault "${getEnv "HOME"}/stockholm";
};
};
diff --git a/shared/2configs/temp/dirs.nix b/shared/2configs/temp/dirs.nix
new file mode 100644
index 000000000..958608a54
--- /dev/null
+++ b/shared/2configs/temp/dirs.nix
@@ -0,0 +1 @@
+_: { }
diff --git a/shared/2configs/temp/networking.nix b/shared/2configs/temp/networking.nix
new file mode 100644
index 000000000..958608a54
--- /dev/null
+++ b/shared/2configs/temp/networking.nix
@@ -0,0 +1 @@
+_: { }
From e6b1003fe26e340be21a12e6e531259fd698f33f Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Tue, 22 Dec 2015 16:30:23 +0100
Subject: [PATCH 05/65] k 5 krebs-ci: initial commit
---
krebs/5pkgs/krebs-ci/default.nix | 37 +++++++++++
krebs/5pkgs/krebs-ci/notes | 111 +++++++++++++++++++++++++++++++
2 files changed, 148 insertions(+)
create mode 100644 krebs/5pkgs/krebs-ci/default.nix
create mode 100755 krebs/5pkgs/krebs-ci/notes
diff --git a/krebs/5pkgs/krebs-ci/default.nix b/krebs/5pkgs/krebs-ci/default.nix
new file mode 100644
index 000000000..f5b302b52
--- /dev/null
+++ b/krebs/5pkgs/krebs-ci/default.nix
@@ -0,0 +1,37 @@
+{ stdenv, coreutils,makeWrapper, cac, cacpanel, gnumake, gnused, jq, openssh, ... }:
+
+stdenv.mkDerivation rec {
+ name = "krebs-ci-0.1.0";
+
+ src = ./notes;
+
+ phases = [
+ "installPhase"
+ ];
+ buildInputs = [ makeWrapper ];
+
+ path = stdenv.lib.makeSearchPath "bin" [
+ coreutils
+ cac
+ cacpanel
+ gnumake
+ gnused
+ jq
+ openssh
+ ];
+
+ installPhase =
+ ''
+ mkdir -p $out/bin
+ cp ${src} $out/bin/krebs-ci
+ chmod +x $out/bin/krebs-ci
+ wrapProgram $out/bin/krebs-ci \
+ --prefix PATH : ${path}
+ '';
+ meta = with stdenv.lib; {
+ homepage = http://krebsco.de;
+ description = "Krebs CI Scripts";
+ license = licenses.wtfpl;
+ maintainers = [ maintainers.makefu ];
+ };
+}
diff --git a/krebs/5pkgs/krebs-ci/notes b/krebs/5pkgs/krebs-ci/notes
new file mode 100755
index 000000000..7e34d6a28
--- /dev/null
+++ b/krebs/5pkgs/krebs-ci/notes
@@ -0,0 +1,111 @@
+#! /bin/sh
+
+# nix-shell -p gnumake jq openssh cac cacpanel
+set -euf
+
+# 2 secrets are required:
+krebs_cred=${krebs_cred-./cac.json}
+retiolum_key=${retiolum_key-./retiolum.rsa_key.priv}
+
+# Sanity
+if test ! -r "$krebs_cred";then
+ echo "\$krebs_cred=$krebs_cred must be readable"; exit 1
+fi
+if test ! -r "$retiolum_key";then
+ echo "\$retiolum_key=$retiolum_key must be readable"; exit 1
+fi
+
+krebs_secrets=$(mktemp -d)
+sec_file=$krebs_secrets/cac_config
+krebs_ssh=$krebs_secrets/tempssh
+# we need to receive this key from buildmaster to speed up tinc bootstrap
+TRAP="rm $sec_file;rm -r $krebs_secrets"
+trap "$TRAP" INT TERM EXIT
+
+cat > $sec_file <<EOF
+cac_login="$(jq -r .email $krebs_cred)"
+cac_key="$(cac-cli panel --config $krebs_cred settings | jq -r .apicode)"
+EOF
+
+export cac_secrets=$sec_file
+cac-cli panel --config $krebs_cred update-api-ip
+
+# test login:
+cac update
+cac servers
+
+# Template 26: CentOS7
+# TODO: use cac templates to determine the real Centos7 template in case it changes
+name=$( cac build cpu=1 ram=512 storage=10 os=26 2>&1\
+ | jq -r .servername)
+
+id=servername:$name
+trap "cac delete $id;$TRAP" INT TERM EXIT
+# TODO: timeout?
+always_update=true cac waitstatus $id "Powered On"
+
+wait_login_cac(){
+ # timeout
+ for t in `seq 60`;do
+ # now we have a working cac server
+ if cac ssh $1 cat /etc/redhat-release | \
+ grep CentOS ;then
+ return 0
+ fi
+ sleep 10
+ done
+ return 1
+}
+# die on timeout
+wait_login_cac $id
+
+mkdir -p shared/2configs/temp
+cac generatenetworking $id > \
+ shared/2configs/temp/networking.nix
+# new temporary ssh key we will use to log in after infest
+ssh-keygen -f $krebs_ssh -N ""
+cp $retiolum_key $krebs_secrets/retiolum.rsa_key.priv
+# we override the directories for secrets and stockholm
+# additionally we set the ssh key we generated
+ip=$(cac getserver $id | jq -r .ip)
+
+cat > shared/2configs/temp/dirs.nix <<EOF
+_: {
+ krebs.build.source.dir = {
+ secrets.path = "$krebs_secrets";
+ stockholm.path = "$(pwd)";
+ };
+ users.extraUsers.root.openssh.authorizedKeys.keys = [
+ "$(cat ${krebs_ssh}.pub)"
+ ];
+ krebs.build.target = "$ip";
+}
+EOF
+
+LOGNAME=shared make eval get=krebs.infest \
+ target=derp system=test-centos7 filter=json \
+ | sed -e "s#^ssh.*<<#cac ssh $id<<#" \
+ -e "/^rsync/a -e 'cac ssh $id' \\\\" \
+ -e "s#root.derp:#:#" > $krebs_secrets/infest
+sh -x $krebs_secrets/infest
+
+# TODO: generate secrets directory $krebs_secrets for nix import
+cac powerop $id reset
+
+wait_login(){
+ # timeout
+ for t in `seq 20`;do
+ # now we have a working cac server
+ if ssh -o StrictHostKeyChecking=no \
+ -o UserKnownHostsFile=/dev/null \
+ -i $krebs_ssh \
+ -o ConnectTimeout=10 \
+ -o BatchMode=yes \
+ root@$1 nixos-version ;then
+ return 0
+ fi
+ sleep 10
+ done
+ return 1
+}
+wait_login $ip
From adbe4a5b4aaed8ea9a7edf20e088f2b74ec3216b Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Tue, 22 Dec 2015 16:38:31 +0100
Subject: [PATCH 06/65] m 2 default: bump revision
---
makefu/2configs/default.nix | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/makefu/2configs/default.nix b/makefu/2configs/default.nix
index c0d7685e3..a0b49edaf 100644
--- a/makefu/2configs/default.nix
+++ b/makefu/2configs/default.nix
@@ -24,7 +24,7 @@ with lib;
git.nixpkgs = {
#url = https://github.com/NixOS/nixpkgs;
url = mkDefault https://github.com/makefu/nixpkgs;
- rev = mkDefault "78340b042463fd35caa587b0db2e400e5666dbe1"; # nixos-15.09 + cherry-picking
+ rev = mkDefault "3fd2c24685f604edc925f73ed56600b8c66236b3"; # nixos-15.09 + cherry-picking
target-path = "/var/src/nixpkgs";
};
From 9ec4c5df3d132db078e89cb577860b6ec416be04 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Tue, 22 Dec 2015 16:42:28 +0100
Subject: [PATCH 07/65] s 2 base: fix user,pubkey,secrets path
---
shared/2configs/base.nix | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/shared/2configs/base.nix b/shared/2configs/base.nix
index fceea67d1..c36061e38 100644
--- a/shared/2configs/base.nix
+++ b/shared/2configs/base.nix
@@ -13,6 +13,8 @@ with lib;
];
};
+ # TODO rename shared user to "krebs"
+ krebs.build.user = config.krebs.users.shared;
krebs.build.source = {
git.nixpkgs = {
url = https://github.com/NixOS/nixpkgs;
@@ -20,7 +22,7 @@ with lib;
};
dir.secrets = {
host = config.krebs.current.host;
- path = mkDefault "${getEnv "HOME"}/secrets/krebs/wolf";
+ path = mkDefault "${getEnv "HOME"}/secrets/krebs/${config.krebs.build.host.name}";
};
dir.stockholm = {
host = config.krebs.current.host;
@@ -65,7 +67,7 @@ with lib;
config.krebs.users.lass.pubkey
config.krebs.users.makefu.pubkey
# TODO HARDER:
- (readFile ../../krebs/Zpubkeys/makefu_omo.ssh.pub)
+ config.krebs.users.makefu-omo.pubkey
config.krebs.users.tv.pubkey
];
From c26ba8d7e674a02995ae613327208f4d9771546b Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Tue, 22 Dec 2015 18:53:53 +0100
Subject: [PATCH 08/65] m 2 base: build user defaults to shared
---
shared/1systems/wolf.nix | 2 --
shared/2configs/base.nix | 2 +-
2 files changed, 1 insertion(+), 3 deletions(-)
diff --git a/shared/1systems/wolf.nix b/shared/1systems/wolf.nix
index 2c51ac8fe..fba4bd9b9 100644
--- a/shared/1systems/wolf.nix
+++ b/shared/1systems/wolf.nix
@@ -33,8 +33,6 @@ in
# uninteresting stuff
#####################
krebs.build.host = config.krebs.hosts.wolf;
- # TODO rename shared user to "krebs"
- krebs.build.user = config.krebs.users.shared;
krebs.build.target = "wolf";
boot.kernel.sysctl = {
diff --git a/shared/2configs/base.nix b/shared/2configs/base.nix
index c36061e38..0ce336558 100644
--- a/shared/2configs/base.nix
+++ b/shared/2configs/base.nix
@@ -14,7 +14,7 @@ with lib;
};
# TODO rename shared user to "krebs"
- krebs.build.user = config.krebs.users.shared;
+ krebs.build.user = mkDefault config.krebs.users.shared;
krebs.build.source = {
git.nixpkgs = {
url = https://github.com/NixOS/nixpkgs;
From bf1b6482ce3535ef7e7b3f77879def12ff454c0c Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Tue, 22 Dec 2015 19:36:19 +0100
Subject: [PATCH 09/65] mv makefu->krebs 3 buildbot
---
.../3modules/buildbot/master.nix | 4 +--
{makefu => krebs}/3modules/buildbot/slave.nix | 4 +--
krebs/3modules/default.nix | 2 ++
makefu/3modules/default.nix | 2 --
shared/1systems/wolf.nix | 2 +-
shared/2configs/buildbot-standalone.nix | 31 +++++++++++++++++++
shared/2configs/cac-ci.nix | 11 -------
7 files changed, 38 insertions(+), 18 deletions(-)
rename {makefu => krebs}/3modules/buildbot/master.nix (99%)
rename {makefu => krebs}/3modules/buildbot/slave.nix (98%)
create mode 100644 shared/2configs/buildbot-standalone.nix
delete mode 100644 shared/2configs/cac-ci.nix
diff --git a/makefu/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
similarity index 99%
rename from makefu/3modules/buildbot/master.nix
rename to krebs/3modules/buildbot/master.nix
index 58e2f8175..2f73e44bc 100644
--- a/makefu/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -143,7 +143,7 @@ let
${cfg.extraConfig}
'';
- cfg = config.makefu.buildbot.master;
+ cfg = config.krebs.buildbot.master;
api = {
enable = mkEnableOption "Buildbot Master";
@@ -258,6 +258,6 @@ let
};
in
{
- options.makefu.buildbot.master = api;
+ options.krebs.buildbot.master = api;
config = mkIf cfg.enable imp;
}
diff --git a/makefu/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix
similarity index 98%
rename from makefu/3modules/buildbot/slave.nix
rename to krebs/3modules/buildbot/slave.nix
index 69d0361bf..65291f63e 100644
--- a/makefu/3modules/buildbot/slave.nix
+++ b/krebs/3modules/buildbot/slave.nix
@@ -39,7 +39,7 @@ let
s.setServiceParent(application)
'';
default-packages = [ pkgs.git pkgs.bash ];
- cfg = config.makefu.buildbot.slave;
+ cfg = config.krebs.buildbot.slave;
api = {
enable = mkEnableOption "Buildbot Slave";
@@ -180,6 +180,6 @@ let
};
in
{
- options.makefu.buildbot.slave = api;
+ options.krebs.buildbot.slave = api;
config = mkIf cfg.enable imp;
}
diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix
index 740ba67b8..cbc1291fa 100644
--- a/krebs/3modules/default.nix
+++ b/krebs/3modules/default.nix
@@ -9,6 +9,8 @@ let
./apt-cacher-ng.nix
./bepasty-server.nix
./build.nix
+ ./buildbot/master.nix
+ ./buildbot/slave.nix
./current.nix
./exim-retiolum.nix
./exim-smarthost.nix
diff --git a/makefu/3modules/default.nix b/makefu/3modules/default.nix
index ffbf54cc0..a8a1f69d0 100644
--- a/makefu/3modules/default.nix
+++ b/makefu/3modules/default.nix
@@ -2,8 +2,6 @@ _:
{
imports = [
- ./buildbot/master.nix
- ./buildbot/slave.nix
];
}
diff --git a/shared/1systems/wolf.nix b/shared/1systems/wolf.nix
index fba4bd9b9..f05356f0f 100644
--- a/shared/1systems/wolf.nix
+++ b/shared/1systems/wolf.nix
@@ -11,7 +11,7 @@ in
../2configs/collectd-base.nix
../2configs/shack-nix-cacher.nix
../2configs/shack-drivedroid.nix
- ../2configs/cac-ci.nix
+ ../2configs/buildbot-standalone.nix
../2configs/graphite.nix
];
# use your own binary cache, fallback use cache.nixos.org (which is used by
diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix
new file mode 100644
index 000000000..adf44cada
--- /dev/null
+++ b/shared/2configs/buildbot-standalone.nix
@@ -0,0 +1,31 @@
+{ lib, config, pkgs, ... }:
+let
+ pkgs-unst = import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-unstable.tar.gz) {};
+in {
+ nixpkgs.config.packageOverrides = pkgs: {
+ buildbot = pkgs-unst.buildbot;
+ buildbot-slave = pkgs-unst.buildbot-slave;
+ };
+ networking.firewall.allowedTCPPorts = [ 8010 ];
+ krebs.buildbot.master = {
+ enable = true;
+ irc = {
+ enable = true;
+ server = "cd.retiolum";
+ channel = "retiolum";
+ allowForce = true;
+ };
+ extraConfig = ''
+ c['buildbotURL'] = "http://${config.krebs.build.host.name}:8010/"
+ '';
+ };
+
+ krebs.buildbot.slave = {
+ enable = true;
+ masterhost = "localhost";
+ username = "testslave";
+ password = "krebspass";
+ packages = with pkgs;[ git nix ];
+ extraEnviron = { NIX_PATH="nixpkgs=${toString <nixpkgs>}"; };
+ };
+}
diff --git a/shared/2configs/cac-ci.nix b/shared/2configs/cac-ci.nix
deleted file mode 100644
index 06cce2746..000000000
--- a/shared/2configs/cac-ci.nix
+++ /dev/null
@@ -1,11 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-with lib;
-{
- environment.systemPackages = with pkgs;[
- get
- cac
- cacpanel
- jq
- ];
-}
From 6c5921c9fc84211b42a93ab715a25dc7d77a1907 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Tue, 22 Dec 2015 20:31:21 +0100
Subject: [PATCH 10/65] Makefile: fail if nix-instantiate fails
---
Makefile | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/Makefile b/Makefile
index aefd17147..5b898c54c 100644
--- a/Makefile
+++ b/Makefile
@@ -35,7 +35,7 @@ ifeq ($(filter),json)
else
filter() { cat; }
endif
- nix-instantiate \
+ result=$$(nix-instantiate \
$${extraArgs-} \
--eval \
-A "$$get" \
@@ -45,8 +45,9 @@ endif
--argstr current-host-name "$$HOSTNAME" \
--argstr current-user-name "$$LOGNAME" \
$${system+--argstr system "$$system"} \
- $${target+--argstr target "$$target"} \
- | filter
+ $${target+--argstr target "$$target"})
+ echo "$$result" | filter
+
else
$(error unbound variable: system[s])
endif
From 56e8346faa75fc42f65d11ea3569a3e5bdd252ec Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Tue, 22 Dec 2015 20:53:11 +0100
Subject: [PATCH 11/65] k 5 krebs-ci: remove obsolete trap rm
---
krebs/3modules/buildbot/master.nix | 2 +-
krebs/5pkgs/krebs-ci/notes | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index 2f73e44bc..e66e0d6b2 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -44,7 +44,7 @@ let
# files everyone depends on or are part of the share branch
def shared_files(change):
- r =re.compile("^((krebs|share)/.*|Makefile|default.nix)")
+ r =re.compile("^((krebs|shared)/.*|Makefile|default.nix)")
for file in change.files:
if r.match(file):
return True
diff --git a/krebs/5pkgs/krebs-ci/notes b/krebs/5pkgs/krebs-ci/notes
index 7e34d6a28..f6b193ddb 100755
--- a/krebs/5pkgs/krebs-ci/notes
+++ b/krebs/5pkgs/krebs-ci/notes
@@ -19,7 +19,7 @@ krebs_secrets=$(mktemp -d)
sec_file=$krebs_secrets/cac_config
krebs_ssh=$krebs_secrets/tempssh
# we need to receive this key from buildmaster to speed up tinc bootstrap
-TRAP="rm $sec_file;rm -r $krebs_secrets"
+TRAP="rm -r $krebs_secrets"
trap "$TRAP" INT TERM EXIT
cat > $sec_file <<EOF
From 9de08634c0718363a9ff7ee5c3a6825a4a4a7d9c Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Tue, 22 Dec 2015 22:00:30 +0100
Subject: [PATCH 12/65] s 1 test-failing: add for CI
---
shared/1systems/test-failing.nix | 6 ++++++
1 file changed, 6 insertions(+)
create mode 100644 shared/1systems/test-failing.nix
diff --git a/shared/1systems/test-failing.nix b/shared/1systems/test-failing.nix
new file mode 100644
index 000000000..81a9e48d6
--- /dev/null
+++ b/shared/1systems/test-failing.nix
@@ -0,0 +1,6 @@
+{ config, pkgs, ... }:
+
+{
+ programs.ssh.startAgent = true;
+ programs.ssh.startAgent = false;
+}
From 1a184c98a21ed32447bb4a88f7c865adef5a535f Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Tue, 22 Dec 2015 23:37:12 +0100
Subject: [PATCH 13/65] k 5 krebs-ci: set cache files manually
---
krebs/5pkgs/krebs-ci/notes | 13 +++++++++----
1 file changed, 9 insertions(+), 4 deletions(-)
diff --git a/krebs/5pkgs/krebs-ci/notes b/krebs/5pkgs/krebs-ci/notes
index f6b193ddb..f6f3da8db 100755
--- a/krebs/5pkgs/krebs-ci/notes
+++ b/krebs/5pkgs/krebs-ci/notes
@@ -1,9 +1,10 @@
#! /bin/sh
# nix-shell -p gnumake jq openssh cac cacpanel
-set -euf
+set -eufx
# 2 secrets are required:
+
krebs_cred=${krebs_cred-./cac.json}
retiolum_key=${retiolum_key-./retiolum.rsa_key.priv}
@@ -18,8 +19,12 @@ fi
krebs_secrets=$(mktemp -d)
sec_file=$krebs_secrets/cac_config
krebs_ssh=$krebs_secrets/tempssh
+cac_resources_cache=$krebs_secrets/res_cache.json
+cac_servers_cache=$krebs_secrets/servers_cache.json
+cac_tasks_cache=$krebs_secrets/tasks_cache.json
+cac_templates_cache=$krebs_secrets/templates_cache.json
# we need to receive this key from buildmaster to speed up tinc bootstrap
-TRAP="rm -r $krebs_secrets"
+TRAP="rm -r $krebs_secrets;exit"
trap "$TRAP" INT TERM EXIT
cat > $sec_file <<EOF
@@ -42,11 +47,11 @@ name=$( cac build cpu=1 ram=512 storage=10 os=26 2>&1\
id=servername:$name
trap "cac delete $id;$TRAP" INT TERM EXIT
# TODO: timeout?
-always_update=true cac waitstatus $id "Powered On"
+# cac_always_update=true cac waitstatus $id "Powered On"
wait_login_cac(){
# timeout
- for t in `seq 60`;do
+ for t in `seq 180`;do
# now we have a working cac server
if cac ssh $1 cat /etc/redhat-release | \
grep CentOS ;then
From f59080e76f950a5a8e33d1edd4314ffaa14187fc Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 23 Dec 2015 00:06:27 +0100
Subject: [PATCH 14/65] m 3 buildbot: add new slow factory to complete
integration test
---
krebs/3modules/buildbot/master.nix | 47 ++++++++++++++++++++----------
krebs/5pkgs/krebs-ci/notes | 8 ++---
2 files changed, 35 insertions(+), 20 deletions(-)
diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index e66e0d6b2..0d9c53977 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -59,27 +59,28 @@ let
###### The actual build
# couple of fast steps:
f = util.BuildFactory()
+ # some slow steps
+ s = util.BuildFactory()
## fetch repo
grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
f.addStep(grab_repo)
+ s.addStep(grab_repo)
# the dependencies which are used by the test script
- deps = [ "gnumake", "jq" ]
- nixshell = ["nix-shell", "-p" ] + deps + [ "--run" ]
+ deps = [ "gnumake", "jq", "(import <stockholm> {}).pkgs.krebs-ci" ]
+ nixshell = ["nix-shell", "-I", ".", "-p" ] + deps + [ "--run" ]
+
def addShell(f,**kwargs):
f.addStep(steps.ShellCommand(**kwargs))
- addShell(f,name="centos7-eval",env={"LOGNAME": "shared",
- "get" : "krebs.deploy",
- "filter" : "json"
- },
- command=nixshell + ["make -s eval system=test-centos7"])
+ addShell(f,name="centos7-eval",env={"LOGNAME": "shared"},
+ command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"])
- addShell(f,name="wolf-eval",env={"LOGNAME": "shared",
- "get" : "krebs.deploy",
- "filter" : "json"
- },
- command=nixshell + ["make -s eval system=wolf"])
+ addShell(f,name="wolf-eval",env={"LOGNAME": "shared"},
+ command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"])
+
+ addShell(f,name="eval-cross-check",env={"LOGNAME": "shared"},
+ command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"])
c['builders'] = []
c['builders'].append(
@@ -87,11 +88,20 @@ let
slavenames=slavenames,
factory=f))
- # TODO slow build
+ # slave needs 2 files:
+ # * cac.json
+ # * retiolum
+ for file in ["cac.json", "retiolum.rsa_key.priv"]:
+ s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file),
+ slavedest=file))
+
+ addShell(s,name="complete-build-centos7",env={"LOGNAME": "shared"},
+ command=nix-shell + ["krebs-ci"])
+
c['builders'].append(
util.BuilderConfig(name="full-tests",
slavenames=slavenames,
- factory=f))
+ factory=s))
####### Status of Builds
c['status'] = []
@@ -119,8 +129,8 @@ let
# TODO: multiple channels
channels=["${cfg.irc.channel}"],
notify_events={
- #'success': 1,
- #'failure': 1,
+ 'success': 1,
+ 'failure': 1,
'exception': 1,
'successToFailure': 1,
'failureToSuccess': 1,
@@ -221,6 +231,7 @@ let
path = [ pkgs.git ];
serviceConfig = let
workdir="${lib.shell.escape cfg.workDir}";
+ secretsdir="${lib.shell.escape (toString <secrets>)}";
# TODO: check if git is the only dep
in {
PermissionsStartOnly = true;
@@ -236,6 +247,10 @@ let
fi
# always override the master.cfg
cp ${buildbot-master-config} ${workdir}/master.cfg
+ # copy secrets
+ cp ${secretsdir}/cac.json ${workdir}
+ cp ${secretsdir}/retiolum-ci.rsa_key.priv \
+ ${workdir}/retiolum.rsa_key.priv
# sanity
${buildbot}/bin/buildbot checkconfig ${workdir}
diff --git a/krebs/5pkgs/krebs-ci/notes b/krebs/5pkgs/krebs-ci/notes
index f6f3da8db..f162656f7 100755
--- a/krebs/5pkgs/krebs-ci/notes
+++ b/krebs/5pkgs/krebs-ci/notes
@@ -19,10 +19,10 @@ fi
krebs_secrets=$(mktemp -d)
sec_file=$krebs_secrets/cac_config
krebs_ssh=$krebs_secrets/tempssh
-cac_resources_cache=$krebs_secrets/res_cache.json
-cac_servers_cache=$krebs_secrets/servers_cache.json
-cac_tasks_cache=$krebs_secrets/tasks_cache.json
-cac_templates_cache=$krebs_secrets/templates_cache.json
+export cac_resources_cache=$krebs_secrets/res_cache.json
+export cac_servers_cache=$krebs_secrets/servers_cache.json
+export cac_tasks_cache=$krebs_secrets/tasks_cache.json
+export cac_templates_cache=$krebs_secrets/templates_cache.json
# we need to receive this key from buildmaster to speed up tinc bootstrap
TRAP="rm -r $krebs_secrets;exit"
trap "$TRAP" INT TERM EXIT
From dc8e270d2a5346e4316b7c2050b26fd428ec3fc3 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 23 Dec 2015 00:06:27 +0100
Subject: [PATCH 15/65] m 3 buildbot: add new slow factory to complete
integration test
---
krebs/3modules/buildbot/master.nix | 52 ++++++++++++++++++++----------
krebs/3modules/buildbot/slave.nix | 1 +
krebs/5pkgs/krebs-ci/notes | 8 ++---
3 files changed, 40 insertions(+), 21 deletions(-)
diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index e66e0d6b2..b4fd6bb2f 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -59,27 +59,28 @@ let
###### The actual build
# couple of fast steps:
f = util.BuildFactory()
+ # some slow steps
+ s = util.BuildFactory()
## fetch repo
grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
f.addStep(grab_repo)
+ s.addStep(grab_repo)
# the dependencies which are used by the test script
- deps = [ "gnumake", "jq" ]
- nixshell = ["nix-shell", "-p" ] + deps + [ "--run" ]
+ deps = [ "gnumake", "jq", "(import <stockholm> {}).pkgs.krebs-ci" ]
+ nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ]
+
def addShell(f,**kwargs):
f.addStep(steps.ShellCommand(**kwargs))
- addShell(f,name="centos7-eval",env={"LOGNAME": "shared",
- "get" : "krebs.deploy",
- "filter" : "json"
- },
- command=nixshell + ["make -s eval system=test-centos7"])
+ addShell(f,name="centos7-eval",env={"LOGNAME": "shared"},
+ command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"])
- addShell(f,name="wolf-eval",env={"LOGNAME": "shared",
- "get" : "krebs.deploy",
- "filter" : "json"
- },
- command=nixshell + ["make -s eval system=wolf"])
+ addShell(f,name="wolf-eval",env={"LOGNAME": "shared"},
+ command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"])
+
+ addShell(f,name="eval-cross-check",env={"LOGNAME": "shared"},
+ command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"])
c['builders'] = []
c['builders'].append(
@@ -87,11 +88,20 @@ let
slavenames=slavenames,
factory=f))
- # TODO slow build
+ # slave needs 2 files:
+ # * cac.json
+ # * retiolum
+ for file in ["cac.json", "retiolum.rsa_key.priv"]:
+ s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file),
+ slavedest=file))
+
+ addShell(s,name="complete-build-centos7",env={"LOGNAME": "shared"},
+ command=nixshell + ["krebs-ci"])
+
c['builders'].append(
util.BuilderConfig(name="full-tests",
slavenames=slavenames,
- factory=f))
+ factory=s))
####### Status of Builds
c['status'] = []
@@ -106,7 +116,7 @@ let
forceBuild = 'auth',
forceAllBuilds = 'auth',
pingBuilder = False,
- stopBuild = False,
+ stopBuild = 'auth',
stopAllBuilds = False,
cancelPendingBuild = False,
)
@@ -119,8 +129,8 @@ let
# TODO: multiple channels
channels=["${cfg.irc.channel}"],
notify_events={
- #'success': 1,
- #'failure': 1,
+ 'success': 1,
+ 'failure': 1,
'exception': 1,
'successToFailure': 1,
'failureToSuccess': 1,
@@ -219,8 +229,12 @@ let
after = [ "network.target" ];
wantedBy = [ "multi-user.target" ];
path = [ pkgs.git ];
+ environment = {
+ SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
+ };
serviceConfig = let
workdir="${lib.shell.escape cfg.workDir}";
+ secretsdir="${lib.shell.escape (toString <secrets>)}";
# TODO: check if git is the only dep
in {
PermissionsStartOnly = true;
@@ -236,6 +250,10 @@ let
fi
# always override the master.cfg
cp ${buildbot-master-config} ${workdir}/master.cfg
+ # copy secrets
+ cp ${secretsdir}/cac.json ${workdir}
+ cp ${secretsdir}/retiolum-ci.rsa_key.priv \
+ ${workdir}/retiolum.rsa_key.priv
# sanity
${buildbot}/bin/buildbot checkconfig ${workdir}
diff --git a/krebs/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix
index 65291f63e..8711a287a 100644
--- a/krebs/3modules/buildbot/slave.nix
+++ b/krebs/3modules/buildbot/slave.nix
@@ -144,6 +144,7 @@ let
path = default-packages ++ cfg.packages;
environment = {
+ SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
NIX_REMOTE="daemon";
} // cfg.extraEnviron;
diff --git a/krebs/5pkgs/krebs-ci/notes b/krebs/5pkgs/krebs-ci/notes
index f6f3da8db..f162656f7 100755
--- a/krebs/5pkgs/krebs-ci/notes
+++ b/krebs/5pkgs/krebs-ci/notes
@@ -19,10 +19,10 @@ fi
krebs_secrets=$(mktemp -d)
sec_file=$krebs_secrets/cac_config
krebs_ssh=$krebs_secrets/tempssh
-cac_resources_cache=$krebs_secrets/res_cache.json
-cac_servers_cache=$krebs_secrets/servers_cache.json
-cac_tasks_cache=$krebs_secrets/tasks_cache.json
-cac_templates_cache=$krebs_secrets/templates_cache.json
+export cac_resources_cache=$krebs_secrets/res_cache.json
+export cac_servers_cache=$krebs_secrets/servers_cache.json
+export cac_tasks_cache=$krebs_secrets/tasks_cache.json
+export cac_templates_cache=$krebs_secrets/templates_cache.json
# we need to receive this key from buildmaster to speed up tinc bootstrap
TRAP="rm -r $krebs_secrets;exit"
trap "$TRAP" INT TERM EXIT
From 14ddb767eb10dbe43d3112c4b4674f6c1d4ff32a Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 23 Dec 2015 11:18:00 +0100
Subject: [PATCH 16/65] k 5 mv krebs-ci test/infest-cac-centos7
---
krebs/3modules/buildbot/master.nix | 6 +++---
krebs/5pkgs/default.nix | 4 ++++
.../{krebs-ci => test/infest-cac-centos7}/default.nix | 10 ++++++----
.../5pkgs/{krebs-ci => test/infest-cac-centos7}/notes | 11 ++++++-----
4 files changed, 19 insertions(+), 12 deletions(-)
rename krebs/5pkgs/{krebs-ci => test/infest-cac-centos7}/default.nix (74%)
rename krebs/5pkgs/{krebs-ci => test/infest-cac-centos7}/notes (91%)
diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index b4fd6bb2f..483ba18e7 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -67,7 +67,7 @@ let
s.addStep(grab_repo)
# the dependencies which are used by the test script
- deps = [ "gnumake", "jq", "(import <stockholm> {}).pkgs.krebs-ci" ]
+ deps = [ "gnumake", "jq", "(import <stockholm> {}).pkgs.test.infest-cac-centos7" ]
nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ]
def addShell(f,**kwargs):
@@ -95,8 +95,8 @@ let
s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file),
slavedest=file))
- addShell(s,name="complete-build-centos7",env={"LOGNAME": "shared"},
- command=nixshell + ["krebs-ci"])
+ addShell(s,name="infest-cac-centos7",env={"LOGNAME": "shared"},
+ command=nixshell + ["infest-cac-centos7"])
c['builders'].append(
util.BuilderConfig(name="full-tests",
diff --git a/krebs/5pkgs/default.nix b/krebs/5pkgs/default.nix
index 7df7b7d3c..0562fe836 100644
--- a/krebs/5pkgs/default.nix
+++ b/krebs/5pkgs/default.nix
@@ -40,6 +40,10 @@ subdirs // rec {
}
'';
+ test = {
+ infest-cac-centos7 = pkgs.callPackage ./test/infest-cac-centos7 {};
+ };
+
execveBin = name: cfg: execve name (cfg // { destination = "/bin/${name}"; });
writeC = name: { destination ? "" }: src: pkgs.runCommand name {} ''
diff --git a/krebs/5pkgs/krebs-ci/default.nix b/krebs/5pkgs/test/infest-cac-centos7/default.nix
similarity index 74%
rename from krebs/5pkgs/krebs-ci/default.nix
rename to krebs/5pkgs/test/infest-cac-centos7/default.nix
index f5b302b52..7f2e3f231 100644
--- a/krebs/5pkgs/krebs-ci/default.nix
+++ b/krebs/5pkgs/test/infest-cac-centos7/default.nix
@@ -1,7 +1,9 @@
{ stdenv, coreutils,makeWrapper, cac, cacpanel, gnumake, gnused, jq, openssh, ... }:
stdenv.mkDerivation rec {
- name = "krebs-ci-0.1.0";
+ name = "${shortname}-${version}";
+ shortname = "infest-cac-centos7";
+ version = "0.2.0";
src = ./notes;
@@ -23,9 +25,9 @@ stdenv.mkDerivation rec {
installPhase =
''
mkdir -p $out/bin
- cp ${src} $out/bin/krebs-ci
- chmod +x $out/bin/krebs-ci
- wrapProgram $out/bin/krebs-ci \
+ cp ${src} $out/bin/${shortname}
+ chmod +x $out/bin/${shortname}
+ wrapProgram $out/bin/${shortname} \
--prefix PATH : ${path}
'';
meta = with stdenv.lib; {
diff --git a/krebs/5pkgs/krebs-ci/notes b/krebs/5pkgs/test/infest-cac-centos7/notes
similarity index 91%
rename from krebs/5pkgs/krebs-ci/notes
rename to krebs/5pkgs/test/infest-cac-centos7/notes
index f162656f7..1e350084c 100755
--- a/krebs/5pkgs/krebs-ci/notes
+++ b/krebs/5pkgs/test/infest-cac-centos7/notes
@@ -24,7 +24,7 @@ export cac_servers_cache=$krebs_secrets/servers_cache.json
export cac_tasks_cache=$krebs_secrets/tasks_cache.json
export cac_templates_cache=$krebs_secrets/templates_cache.json
# we need to receive this key from buildmaster to speed up tinc bootstrap
-TRAP="rm -r $krebs_secrets;exit"
+TRAP="rm -r $krebs_secrets;trap - INT TERM EXIT"
trap "$TRAP" INT TERM EXIT
cat > $sec_file <<EOF
@@ -45,16 +45,17 @@ name=$( cac build cpu=1 ram=512 storage=10 os=26 2>&1\
| jq -r .servername)
id=servername:$name
-trap "cac delete $id;$TRAP" INT TERM EXIT
+trap "cac delete $id;$TRAP;exit" INT TERM EXIT
# TODO: timeout?
-# cac_always_update=true cac waitstatus $id "Powered On"
wait_login_cac(){
# timeout
for t in `seq 180`;do
# now we have a working cac server
- if cac ssh $1 cat /etc/redhat-release | \
- grep CentOS ;then
+ if cac ssh $1 -o ConnectTimeout=10 \
+ -o BatchMode=yes \
+ cat /etc/redhat-release | \
+ grep CentOS ;then
return 0
fi
sleep 10
From cf3391704d88c49afba652715e1153888bf46099 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 23 Dec 2015 16:02:58 +0100
Subject: [PATCH 17/65] k 5 test/infest*: remove batch mode from cac ssh call
this leads to "permission denied"
---
krebs/5pkgs/test/infest-cac-centos7/notes | 1 -
1 file changed, 1 deletion(-)
diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes
index 1e350084c..5fd0cae61 100755
--- a/krebs/5pkgs/test/infest-cac-centos7/notes
+++ b/krebs/5pkgs/test/infest-cac-centos7/notes
@@ -53,7 +53,6 @@ wait_login_cac(){
for t in `seq 180`;do
# now we have a working cac server
if cac ssh $1 -o ConnectTimeout=10 \
- -o BatchMode=yes \
cat /etc/redhat-release | \
grep CentOS ;then
return 0
From 9a386718714d70f7100b5de297dfd0869d98e47b Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 23 Dec 2015 16:06:41 +0100
Subject: [PATCH 18/65] k 3 buildbot: fix merge fuckup
---
krebs/3modules/buildbot/master.nix | 5 -----
1 file changed, 5 deletions(-)
diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index 19aecead1..483ba18e7 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -95,13 +95,8 @@ let
s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file),
slavedest=file))
-<<<<<<< HEAD
addShell(s,name="infest-cac-centos7",env={"LOGNAME": "shared"},
command=nixshell + ["infest-cac-centos7"])
-=======
- addShell(s,name="complete-build-centos7",env={"LOGNAME": "shared"},
- command=nix-shell + ["krebs-ci"])
->>>>>>> f59080e76f950a5a8e33d1edd4314ffaa14187fc
c['builders'].append(
util.BuilderConfig(name="full-tests",
From 1ef9af2c9a49490a2dda21884ad761675c520d1a Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 23 Dec 2015 16:20:27 +0100
Subject: [PATCH 19/65] k 3 buildbot/master: send sigterm before sigkill for
cleanup
---
krebs/3modules/buildbot/master.nix | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index 483ba18e7..6ce708769 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -95,8 +95,10 @@ let
s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file),
slavedest=file))
- addShell(s,name="infest-cac-centos7",env={"LOGNAME": "shared"},
- command=nixshell + ["infest-cac-centos7"])
+ addShell(s, name="infest-cac-centos7",env={"LOGNAME": "shared"},
+ sigtermTime=60, # SIGTERM 1 minute before SIGKILL
+ timeout=5400, # 1.5h timeout
+ command=nixshell + ["infest-cac-centos7"])
c['builders'].append(
util.BuilderConfig(name="full-tests",
From 6b7506dc672b4bd658088bf37fad06fd64c777fe Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 23 Dec 2015 16:37:02 +0100
Subject: [PATCH 20/65] k 3 buildbot: add rsync as explicit dep
do not be pure yet
---
krebs/3modules/buildbot/master.nix | 17 +++++++++++------
1 file changed, 11 insertions(+), 6 deletions(-)
diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index 6ce708769..6bf3fda2c 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -67,19 +67,24 @@ let
s.addStep(grab_repo)
# the dependencies which are used by the test script
- deps = [ "gnumake", "jq", "(import <stockholm> {}).pkgs.test.infest-cac-centos7" ]
+ deps = [ "gnumake", "jq","nix",
+ "(import <stockholm> {}).pkgs.test.infest-cac-centos7",
+ "rsync" ]
+ # TODO: --pure , prepare ENV in nix-shell command:
+ # SSL_CERT_FILE,LOGNAME,NIX_REMOTE
nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ]
-
+ env = {"LOGNAME": "shared",
+ "NIX_REMOTE": "daemon"}
def addShell(f,**kwargs):
f.addStep(steps.ShellCommand(**kwargs))
- addShell(f,name="centos7-eval",env={"LOGNAME": "shared"},
+ addShell(f,name="centos7-eval",env=env,
command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"])
- addShell(f,name="wolf-eval",env={"LOGNAME": "shared"},
+ addShell(f,name="wolf-eval",env=env,
command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"])
- addShell(f,name="eval-cross-check",env={"LOGNAME": "shared"},
+ addShell(f,name="eval-cross-check",env=env,
command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"])
c['builders'] = []
@@ -95,7 +100,7 @@ let
s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file),
slavedest=file))
- addShell(s, name="infest-cac-centos7",env={"LOGNAME": "shared"},
+ addShell(s, name="infest-cac-centos7",env=env,
sigtermTime=60, # SIGTERM 1 minute before SIGKILL
timeout=5400, # 1.5h timeout
command=nixshell + ["infest-cac-centos7"])
From 3adf78473d2deff0b991d7222e928fa2888529f6 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Thu, 24 Dec 2015 00:02:59 +0100
Subject: [PATCH 21/65] k 3 buildbot.master: refactor
see buildbot-standalone.nix in shared/2configs for the current buildbot config
---
krebs/3modules/buildbot/master.nix | 325 +++++++++++++++---------
shared/2configs/buildbot-standalone.nix | 107 +++++++-
2 files changed, 309 insertions(+), 123 deletions(-)
diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index 6bf3fda2c..7078000fe 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -7,134 +7,81 @@ let
# -*- python -*-
from buildbot.plugins import *
import re
-
+ import json
c = BuildmasterConfig = {}
c['slaves'] = []
- # TODO: template potential buildslaves
- # TODO: set password?
- slavenames= [ 'testslave' ]
- for i in slavenames:
- c['slaves'].append(buildslave.BuildSlave(i, "krebspass"))
+ slaves = json.loads('${builtins.toJSON cfg.slaves}')
+ slavenames = [ s for s in slaves ]
+ for k,v in slaves.items():
+ c['slaves'].append(buildslave.BuildSlave(k, v))
+ # TODO: configure protocols?
c['protocols'] = {'pb': {'port': 9989}}
####### Build Inputs
- stockholm_repo = 'http://cgit.gum/stockholm'
- c['change_source'] = []
- c['change_source'].append(changes.GitPoller(
- stockholm_repo,
- workdir='stockholm-poller', branch='master',
- project='stockholm',
- pollinterval=120))
+ c['change_source'] = cs = []
+
+ ${ concatStringsSep "\n"
+ (mapAttrsToList (n: v: ''
+ #### Change_Source: Begin of ${n}
+ ${v}
+ #### Change_Source: End of ${n}
+ '') cfg.change_source )}
####### Build Scheduler
- # TODO: configure scheduler
- c['schedulers'] = []
+ c['schedulers'] = sched = []
- # test the master real quick
- fast = schedulers.SingleBranchScheduler(
- change_filter=util.ChangeFilter(branch="master"),
- name="fast-master-test",
- builderNames=["fast-tests"])
+ ${ concatStringsSep "\n"
+ (mapAttrsToList (n: v: ''
+ #### Schedulers: Begin of ${n}
+ ${v}
+ #### Schedulers: End of ${n}
+ '') cfg.scheduler )}
- force = schedulers.ForceScheduler(
- name="force",
- builderNames=["full-tests"])
+ ###### Builder
+ c['builders'] = bu = []
+
+ # Builder Pre: Begin
+ ${cfg.builder_pre}
+ # Builder Pre: End
- # files everyone depends on or are part of the share branch
- def shared_files(change):
- r =re.compile("^((krebs|shared)/.*|Makefile|default.nix)")
- for file in change.files:
- if r.match(file):
- return True
- return False
+ ${ concatStringsSep "\n"
+ (mapAttrsToList (n: v: ''
+ #### Builder: Begin of ${n}
+ ${v}
+ #### Builder: End of ${n}
+ '') cfg.builder )}
- full = schedulers.SingleBranchScheduler(
- change_filter=util.ChangeFilter(branch="master"),
- fileIsImportant=shared_files,
- name="full-master-test",
- builderNames=["full-tests"])
- c['schedulers'] = [ fast, force, full ]
- ###### The actual build
- # couple of fast steps:
- f = util.BuildFactory()
- # some slow steps
- s = util.BuildFactory()
- ## fetch repo
- grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
- f.addStep(grab_repo)
- s.addStep(grab_repo)
- # the dependencies which are used by the test script
- deps = [ "gnumake", "jq","nix",
- "(import <stockholm> {}).pkgs.test.infest-cac-centos7",
- "rsync" ]
- # TODO: --pure , prepare ENV in nix-shell command:
- # SSL_CERT_FILE,LOGNAME,NIX_REMOTE
- nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ]
- env = {"LOGNAME": "shared",
- "NIX_REMOTE": "daemon"}
- def addShell(f,**kwargs):
- f.addStep(steps.ShellCommand(**kwargs))
+ ####### Status
+ c['status'] = st = []
- addShell(f,name="centos7-eval",env=env,
- command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"])
+ # If you want to configure this url, override with extraConfig
+ c['buildbotURL'] = "http://${config.networking.hostName}:${toString cfg.web.port}/"
- addShell(f,name="wolf-eval",env=env,
- command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"])
+ ${optionalString (cfg.web.enable) ''
+ from buildbot.status import html
+ from buildbot.status.web import authz, auth
+ authz_cfg=authz.Authz(
+ auth=auth.BasicAuth([ ("${cfg.web.username}","${cfg.web.password}") ]),
+ # TODO: configure harder
+ gracefulShutdown = False,
+ forceBuild = 'auth',
+ forceAllBuilds = 'auth',
+ pingBuilder = False,
+ stopBuild = 'auth',
+ stopAllBuilds = 'auth',
+ cancelPendingBuild = 'auth'
+ )
+ # TODO: configure krebs.nginx
+ st.append(html.WebStatus(http_port=${toString cfg.web.port}, authz=authz_cfg))
+ ''}
- addShell(f,name="eval-cross-check",env=env,
- command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"])
-
- c['builders'] = []
- c['builders'].append(
- util.BuilderConfig(name="fast-tests",
- slavenames=slavenames,
- factory=f))
-
- # slave needs 2 files:
- # * cac.json
- # * retiolum
- for file in ["cac.json", "retiolum.rsa_key.priv"]:
- s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file),
- slavedest=file))
-
- addShell(s, name="infest-cac-centos7",env=env,
- sigtermTime=60, # SIGTERM 1 minute before SIGKILL
- timeout=5400, # 1.5h timeout
- command=nixshell + ["infest-cac-centos7"])
-
- c['builders'].append(
- util.BuilderConfig(name="full-tests",
- slavenames=slavenames,
- factory=s))
-
- ####### Status of Builds
- c['status'] = []
-
- from buildbot.status import html
- from buildbot.status.web import authz, auth
- # TODO: configure if http is wanted
- authz_cfg=authz.Authz(
- # TODO: configure user/pw
- auth=auth.BasicAuth([("krebs","bob")]),
- gracefulShutdown = False,
- forceBuild = 'auth',
- forceAllBuilds = 'auth',
- pingBuilder = False,
- stopBuild = 'auth',
- stopAllBuilds = False,
- cancelPendingBuild = False,
- )
- # TODO: configure nginx
- c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
-
- from buildbot.status import words
${optionalString (cfg.irc.enable) ''
- irc = words.IRC("${cfg.irc.server}", "krebsbuild",
- # TODO: multiple channels
- channels=["${cfg.irc.channel}"],
+ from buildbot.status import words
+ irc = words.IRC("${cfg.irc.server}", "${cfg.irc.nick}",
+ channels=${builtins.toJSON cfg.irc.channels},
notify_events={
'success': 1,
'failure': 1,
@@ -145,15 +92,20 @@ let
c['status'].append(irc)
''}
+ ${ concatStringsSep "\n"
+ (mapAttrsToList (n: v: ''
+ #### Status: Begin of ${n}
+ ${v}
+ #### Status: End of ${n}
+ '') cfg.status )}
+
####### PROJECT IDENTITY
- c['title'] = "Stockholm"
+ c['title'] = "${cfg.title}"
c['titleURL'] = "http://krebsco.de"
- #c['buildbotURL'] = "http://buildbot.krebsco.de/"
- # TODO: configure url
- c['buildbotURL'] = "http://vbob:8010/"
####### DB URL
+ # TODO: configure
c['db'] = {
'db_url' : "sqlite:///state.sqlite",
}
@@ -164,6 +116,13 @@ let
api = {
enable = mkEnableOption "Buildbot Master";
+ title = mkOption {
+ default = "Buildbot CI";
+ type = types.str;
+ description = ''
+ Title of the Buildbot Installation
+ '';
+ };
workDir = mkOption {
default = "/var/lib/buildbot/master";
type = types.str;
@@ -172,16 +131,144 @@ let
Will be created on startup.
'';
};
+
+ slaves = mkOption {
+ default = {};
+ type = types.attrsOf types.str;
+ description = ''
+ Attrset of slavenames with their passwords
+ slavename = slavepassword
+ '';
+ };
+
+ change_source = mkOption {
+ default = {};
+ type = types.attrsOf types.str;
+ example = {
+ stockholm = ''
+ cs.append(changes.GitPoller(
+ 'http://cgit.gum/stockholm',
+ workdir='stockholm-poller', branch='master',
+ project='stockholm',
+ pollinterval=120))
+ '';
+ };
+ description = ''
+ Attrset of all the change_sources which should be configured.
+ It will be directly included into the master configuration.
+
+ At the end an change object should be appended to <literal>cs</literal>
+ '';
+ };
+
+ scheduler = mkOption {
+ default = {};
+ type = types.attrsOf types.str;
+ example = {
+ force-scheduler = ''
+ sched.append(schedulers.ForceScheduler(
+ name="force",
+ builderNames=["full-tests"]))
+ '';
+ };
+ description = ''
+ Attrset of all the schedulers which should be configured.
+ It will be directly included into the master configuration.
+
+ At the end an change object should be appended to <literal>sched</literal>
+ '';
+ };
+
+ builder_pre = mkOption {
+ default = "";
+ type = types.lines;
+ example = ''
+ grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
+ '';
+ description = ''
+ some code before the builders are being assembled.
+ can be used to define functions used by multiple builders
+ '';
+ };
+
+ builder = mkOption {
+ default = {};
+ type = types.attrsOf types.str;
+ example = {
+ fast-test = ''
+ '';
+ };
+ description = ''
+ Attrset of all the builder which should be configured.
+ It will be directly included into the master configuration.
+
+ At the end an change object should be appended to <literal>bu</literal>
+ '';
+ };
+
+ status = mkOption {
+ default = {};
+ type = types.attrsOf types.str;
+ description = ''
+ Attrset of all the extra status which should be configured.
+ It will be directly included into the master configuration.
+
+ At the end an change object should be appended to <literal>st</literal>
+
+ Right now IRC and Web status can be configured by setting
+ <literal>buildbot.master.irc.enable</literal> and
+ <literal>buildbot.master.web.enable</literal>
+ '';
+ };
+
+ # Configurable Stati
+ web = mkOption {
+ default = {};
+ type = types.submodule ({ config2, ... }: {
+ options = {
+ enable = mkEnableOption "Buildbot Master Web Status";
+ username = mkOption {
+ default = "krebs";
+ type = types.str;
+ description = ''
+ username for web authentication
+ '';
+ };
+ hostname = mkOption {
+ default = config.networking.hostName;
+ type = types.str;
+ description = ''
+ web interface Hostname
+ '';
+ };
+ password = mkOption {
+ default = "bob";
+ type = types.str;
+ description = ''
+ password for web authentication
+ '';
+ };
+ port = mkOption {
+ default = 8010;
+ type = types.int;
+ description = ''
+ port for buildbot web status
+ '';
+ };
+ };
+ });
+ };
+
irc = mkOption {
default = {};
type = types.submodule ({ config, ... }: {
options = {
enable = mkEnableOption "Buildbot Master IRC Status";
- channel = mkOption {
- default = "nix-buildbot-meetup";
- type = types.str;
+ channels = mkOption {
+ default = [ "nix-buildbot-meetup" ];
+ type = with types; listOf str;
description = ''
- irc channel the bot should connect to
+ irc channels the bot should connect to
'';
};
allowForce = mkOption {
@@ -235,6 +322,7 @@ let
description = "Buildbot Master";
after = [ "network.target" ];
wantedBy = [ "multi-user.target" ];
+ # TODO: add extra dependencies to master like svn and cvs
path = [ pkgs.git ];
environment = {
SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
@@ -242,7 +330,6 @@ let
serviceConfig = let
workdir="${lib.shell.escape cfg.workDir}";
secretsdir="${lib.shell.escape (toString <secrets>)}";
- # TODO: check if git is the only dep
in {
PermissionsStartOnly = true;
Type = "forking";
diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix
index adf44cada..baab059c9 100644
--- a/shared/2configs/buildbot-standalone.nix
+++ b/shared/2configs/buildbot-standalone.nix
@@ -8,16 +8,115 @@ in {
};
networking.firewall.allowedTCPPorts = [ 8010 ];
krebs.buildbot.master = {
+ slaves = {
+ testslave = "krebspass";
+ testslave2 = "krebspass";
+ };
+ change_source.stockholm = ''
+ stockholm_repo = 'http://cgit.gum/stockholm'
+ cs.append(changes.GitPoller(
+ stockholm_repo,
+ workdir='stockholm-poller', branch='master',
+ project='stockholm',
+ pollinterval=120))
+ '';
+ scheduler = {
+ force-scheduler = ''
+ sched.append(schedulers.ForceScheduler(
+ name="force",
+ builderNames=["full-tests"]))
+ '';
+ fast-tests-scheduler = ''
+ # test the master real quick
+ sched.append(schedulers.SingleBranchScheduler(
+ change_filter=util.ChangeFilter(branch="master"),
+ name="fast-master-test",
+ builderNames=["fast-tests"]))
+ '';
+ full-master-scheduler = ''
+ # files everyone depends on or are part of the share branch
+ def shared_files(change):
+ r =re.compile("^((krebs|shared)/.*|Makefile|default.nix)")
+ for file in change.files:
+ if r.match(file):
+ return True
+ return False
+
+ sched.append(schedulers.SingleBranchScheduler(
+ change_filter=util.ChangeFilter(branch="master"),
+ fileIsImportant=shared_files,
+ name="full-master-test",
+ builderNames=["full-tests"]))
+ '';
+ };
+ builder_pre = ''
+ # prepare grab_repo step for stockholm
+ stockholm_repo = "http://cgit.gum.retiolum/stockholm"
+ grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
+
+ env = {"LOGNAME": "shared", "NIX_REMOTE": "daemon"}
+
+ # prepare nix-shell
+ # the dependencies which are used by the test script
+ deps = [ "gnumake", "jq","nix","rsync",
+ "(import <stockholm> {}).pkgs.test.infest-cac-centos7" ]
+ # TODO: --pure , prepare ENV in nix-shell command:
+ # SSL_CERT_FILE,LOGNAME,NIX_REMOTE
+ nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ]
+
+ # prepare addShell function
+ def addShell(factory,**kwargs):
+ factory.addStep(steps.ShellCommand(**kwargs))
+ '';
+ builder = {
+ fast-tests = ''
+ f = util.BuildFactory()
+ f.addStep(grab_repo)
+ addShell(f,name="centos7-eval",env=env,
+ command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"])
+
+ addShell(f,name="wolf-eval",env=env,
+ command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"])
+
+ addShell(f,name="eval-cross-check",env=env,
+ command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"])
+
+ bu.append(util.BuilderConfig(name="fast-tests",
+ slavenames=slavenames,
+ factory=f))
+ '';
+ slow-tests = ''
+ s = util.BuildFactory()
+ s.addStep(grab_repo)
+
+ # slave needs 2 files:
+ # * cac.json
+ # * retiolum
+ for file in ["cac.json", "retiolum.rsa_key.priv"]:
+ s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/{}".format(file),
+ slavedest=file))
+
+ addShell(s, name="infest-cac-centos7",env=env,
+ sigtermTime=60, # SIGTERM 1 minute before SIGKILL
+ timeout=5400, # 1.5h timeout
+ command=nixshell + ["infest-cac-centos7"])
+
+ bu.append(util.BuilderConfig(name="full-tests",
+ slavenames=slavenames,
+ factory=s))
+ '';
+ };
enable = true;
+ web = {
+ enable = true;
+ };
irc = {
enable = true;
+ nick = "shared-buildbot";
server = "cd.retiolum";
- channel = "retiolum";
+ channels = [ "retiolum" ];
allowForce = true;
};
- extraConfig = ''
- c['buildbotURL'] = "http://${config.krebs.build.host.name}:8010/"
- '';
};
krebs.buildbot.slave = {
From 6e4351044195f1f3b5708785e760b9d118e2c229 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Thu, 24 Dec 2015 02:20:24 +0100
Subject: [PATCH 22/65] k 5 test/infest*: up limit of final connect
---
krebs/5pkgs/test/infest-cac-centos7/notes | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes
index 5fd0cae61..5bb5de2c4 100755
--- a/krebs/5pkgs/test/infest-cac-centos7/notes
+++ b/krebs/5pkgs/test/infest-cac-centos7/notes
@@ -99,7 +99,7 @@ cac powerop $id reset
wait_login(){
# timeout
- for t in `seq 20`;do
+ for t in `seq 90`;do
# now we have a working cac server
if ssh -o StrictHostKeyChecking=no \
-o UserKnownHostsFile=/dev/null \
From 9ae664209328f6030bf3773e09dce7bcd14e82b4 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Thu, 24 Dec 2015 20:37:04 +0100
Subject: [PATCH 23/65] k 5 cacpanel: bump version to 0.2.3
---
krebs/5pkgs/cacpanel/default.nix | 4 ++--
krebs/5pkgs/test/infest-cac-centos7/notes | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/krebs/5pkgs/cacpanel/default.nix b/krebs/5pkgs/cacpanel/default.nix
index 3e3e2e1fc..3df4dffed 100644
--- a/krebs/5pkgs/cacpanel/default.nix
+++ b/krebs/5pkgs/cacpanel/default.nix
@@ -2,11 +2,11 @@
python3Packages.buildPythonPackage rec {
name = "cacpanel-${version}";
- version = "0.2.1";
+ version = "0.2.3";
src = pkgs.fetchurl {
url = "https://pypi.python.org/packages/source/c/cacpanel/cacpanel-${version}.tar.gz";
- sha256 = "1zaazg5r10kgva32zh4fhpw6l6h51ijkwpa322na0kh4x6f6aqj3";
+ sha256 = "1fib7416qqv8yzrj75kxra7ccpz9abqh58b6gkaavws2fa6m3mm8";
};
propagatedBuildInputs = with python3Packages; [
diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes
index 5bb5de2c4..cfb074423 100755
--- a/krebs/5pkgs/test/infest-cac-centos7/notes
+++ b/krebs/5pkgs/test/infest-cac-centos7/notes
@@ -29,11 +29,11 @@ trap "$TRAP" INT TERM EXIT
cat > $sec_file <<EOF
cac_login="$(jq -r .email $krebs_cred)"
-cac_key="$(cac-cli panel --config $krebs_cred settings | jq -r .apicode)"
+cac_key="$(cac-cli --config $krebs_cred panel settings | jq -r .apicode)"
EOF
export cac_secrets=$sec_file
-cac-cli panel --config $krebs_cred update-api-ip
+cac-cli --config $krebs_cred panel add-api-ip
# test login:
cac update
From 1ff8d0b8c5c6f631d71408eeff61778d90f9789f Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Thu, 24 Dec 2015 20:50:23 +0100
Subject: [PATCH 24/65] s 2 buildbot: add treestabletimer
---
shared/2configs/base.nix | 2 ++
shared/2configs/buildbot-standalone.nix | 7 ++++---
2 files changed, 6 insertions(+), 3 deletions(-)
diff --git a/shared/2configs/base.nix b/shared/2configs/base.nix
index 0ce336558..4d509d7a6 100644
--- a/shared/2configs/base.nix
+++ b/shared/2configs/base.nix
@@ -19,6 +19,7 @@ with lib;
git.nixpkgs = {
url = https://github.com/NixOS/nixpkgs;
rev = "6d31e9b81dcd4ab927bb3dc91b612dd5abfa2f80";
+ target-path = "/var/src/nixpkgs";
};
dir.secrets = {
host = config.krebs.current.host;
@@ -27,6 +28,7 @@ with lib;
dir.stockholm = {
host = config.krebs.current.host;
path = mkDefault "${getEnv "HOME"}/stockholm";
+ target-path = "/var/src/stockholm";
};
};
diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix
index baab059c9..51c600329 100644
--- a/shared/2configs/buildbot-standalone.nix
+++ b/shared/2configs/buildbot-standalone.nix
@@ -6,11 +6,11 @@ in {
buildbot = pkgs-unst.buildbot;
buildbot-slave = pkgs-unst.buildbot-slave;
};
- networking.firewall.allowedTCPPorts = [ 8010 ];
+ networking.firewall.allowedTCPPorts = [ 8010 9989 ];
krebs.buildbot.master = {
slaves = {
testslave = "krebspass";
- testslave2 = "krebspass";
+ omo = "krebspass";
};
change_source.stockholm = ''
stockholm_repo = 'http://cgit.gum/stockholm'
@@ -33,7 +33,7 @@ in {
name="fast-master-test",
builderNames=["fast-tests"]))
'';
- full-master-scheduler = ''
+ test-cac-infest-master = ''
# files everyone depends on or are part of the share branch
def shared_files(change):
r =re.compile("^((krebs|shared)/.*|Makefile|default.nix)")
@@ -45,6 +45,7 @@ in {
sched.append(schedulers.SingleBranchScheduler(
change_filter=util.ChangeFilter(branch="master"),
fileIsImportant=shared_files,
+ treeStableTimer=60*60, # master was stable for the last hour
name="full-master-test",
builderNames=["full-tests"]))
'';
From 7f9f7a0cf65212618fbe3fcd85291868b571fae2 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Thu, 24 Dec 2015 20:50:53 +0100
Subject: [PATCH 25/65] m 2 urlwatch: add cvs2svn to watchlist
---
makefu/2configs/urlwatch.nix | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/makefu/2configs/urlwatch.nix b/makefu/2configs/urlwatch.nix
index cd05f0114..eadffa7dd 100644
--- a/makefu/2configs/urlwatch.nix
+++ b/makefu/2configs/urlwatch.nix
@@ -12,7 +12,7 @@
http://git.sysphere.org/vicious/log/?qt=grep&q=Next+release
https://pypi.python.org/simple/bepasty/
https://pypi.python.org/simple/xstatic/
-
+ http://cvs2svn.tigris.org/servlets/ProjectDocumentList?folderID=2976
];
};
}
From 58f37bde831877e467646d283b88c17251b84b7c Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Thu, 24 Dec 2015 20:51:58 +0100
Subject: [PATCH 26/65] m 1 gum: enable urlwatch service
---
makefu/1systems/gum.nix | 3 +++
makefu/1systems/pnp.nix | 3 ---
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/makefu/1systems/gum.nix b/makefu/1systems/gum.nix
index 417a020fa..93fb3dc3a 100644
--- a/makefu/1systems/gum.nix
+++ b/makefu/1systems/gum.nix
@@ -15,6 +15,9 @@ in {
../2configs/git/cgit-retiolum.nix
../2configs/mattermost-docker.nix
../2configs/nginx/euer.test.nix
+
+ ../2configs/exim-retiolum.nix
+ ../2configs/urlwatch.nix
];
diff --git a/makefu/1systems/pnp.nix b/makefu/1systems/pnp.nix
index 161bfa3e9..a1b73c0c9 100644
--- a/makefu/1systems/pnp.nix
+++ b/makefu/1systems/pnp.nix
@@ -28,9 +28,6 @@
../2configs/Reaktor/titlebot.nix
../2configs/Reaktor/shack-correct.nix
- ../2configs/exim-retiolum.nix
- ../2configs/urlwatch.nix
-
# ../2configs/graphite-standalone.nix
];
krebs.urlwatch.verbose = true;
From 70264d1e46dc17391f0a3a590ba0749d0a93eda2 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Fri, 25 Dec 2015 00:04:52 +0100
Subject: [PATCH 27/65] k 5 Reaktor: init plugin infrastructure
---
krebs/5pkgs/Reaktor/plugins.nix | 38 +++++++++++++++++++++
krebs/5pkgs/Reaktor/scripts/random-emoji.sh | 6 ++++
krebs/5pkgs/default.nix | 2 ++
3 files changed, 46 insertions(+)
create mode 100644 krebs/5pkgs/Reaktor/plugins.nix
create mode 100644 krebs/5pkgs/Reaktor/scripts/random-emoji.sh
diff --git a/krebs/5pkgs/Reaktor/plugins.nix b/krebs/5pkgs/Reaktor/plugins.nix
new file mode 100644
index 000000000..05ede38e1
--- /dev/null
+++ b/krebs/5pkgs/Reaktor/plugins.nix
@@ -0,0 +1,38 @@
+{ stdenv, lib, pkgs, makeWrapper }:
+
+rec {
+ buildReaktorPlugin = { name
+ # TODO: profiles
+ , extraConfig
+ , phases ? []
+ , ... } @ attrs:
+ stdenv.mkDerivation (attrs // {
+ name = "Reaktor-plugin-" + name;
+ phases = phases ++ [ "installPhase" ];
+ isReaktorPlugin = true;
+ });
+
+ random-emoji = buildReaktorPlugin rec {
+ name = "random-emoji";
+ src = ./scripts/random-emoji.sh;
+ phases = [ "installPhase" ];
+ buildInputs = [ makeWrapper ];
+ installPhase = ''
+ mkdir -p $out/bin
+ install -vm 755 ${src} $out/bin/random-emoji.sh
+ wrapProgram $out/bin/random-emoji.sh \
+ --prefix PATH : ${lib.makeSearchPath "bin" (with pkgs; [
+ coreutils
+ gnused
+ gnugrep
+ xmlstarlet
+ curl])};
+ '';
+ extraConfig = ''
+ public_commands.insert(0,{
+ 'capname' : "emoji",
+ 'pattern' : indirect_pattern.format("emoji"),
+ 'argv' : ["random-emoji.sh"])
+ '';
+ };
+}
diff --git a/krebs/5pkgs/Reaktor/scripts/random-emoji.sh b/krebs/5pkgs/Reaktor/scripts/random-emoji.sh
new file mode 100644
index 000000000..386aa68b9
--- /dev/null
+++ b/krebs/5pkgs/Reaktor/scripts/random-emoji.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+curl http://emojicons.com/random -s | \
+ grep data-text | \
+ sed -n 's/.*>\(.*\)<\/textarea>/\1/p' | \
+ head -n 1 | \
+ xmlstarlet unesc
diff --git a/krebs/5pkgs/default.nix b/krebs/5pkgs/default.nix
index 0562fe836..c4b1dafe4 100644
--- a/krebs/5pkgs/default.nix
+++ b/krebs/5pkgs/default.nix
@@ -26,6 +26,8 @@ subdirs // rec {
inherit (subdirs) get jq;
};
+ ReaktorPlugins = pkgs.callPackage ./Reaktor/plugins.nix {};
+
execve = name: { filename, argv, envp ? {}, destination ? "" }:
writeC name { inherit destination; } ''
#include <unistd.h>
From f55b44eb7cffbe0934785afd3a36001ba0713ad1 Mon Sep 17 00:00:00 2001
From: lassulus <lass@aidsballs.de>
Date: Sat, 26 Dec 2015 10:43:15 +0100
Subject: [PATCH 28/65] l 1 mors: add wordpress/owncloud test
---
lass/1systems/mors.nix | 36 +++++++++++++++++++++++++++++++-----
1 file changed, 31 insertions(+), 5 deletions(-)
diff --git a/lass/1systems/mors.nix b/lass/1systems/mors.nix
index 4ba9df6f9..9b2200c58 100644
--- a/lass/1systems/mors.nix
+++ b/lass/1systems/mors.nix
@@ -38,6 +38,10 @@
../3modules/wordpress_nginx.nix
];
lass.wordpress."testserver.de" = {
+ multiSite = {
+ "1" = "testserver.de";
+ "2" = "bla.testserver.de";
+ };
};
services.mysql = {
@@ -52,6 +56,27 @@
{ predicate = "-i retiolum -p tcp --dport 80"; target = "ACCEPT"; precedence = 9998; }
];
}
+ {
+ #owncloud-test
+ #imports = singleton (sitesGenerators.createWordpress "testserver.de");
+ imports = [
+ ../3modules/owncloud_nginx.nix
+ ];
+ lass.owncloud."owncloud-test.de" = {
+ };
+
+ #services.mysql = {
+ # enable = true;
+ # package = pkgs.mariadb;
+ # rootPassword = "<secrets>/mysql_rootPassword";
+ #};
+ networking.extraHosts = ''
+ 10.243.0.2 owncloud-test.de
+ '';
+ krebs.iptables.tables.filter.INPUT.rules = [
+ { predicate = "-i retiolum -p tcp --dport 80"; target = "ACCEPT"; precedence = 9998; }
+ ];
+ }
];
krebs.build.host = config.krebs.hosts.mors;
@@ -59,11 +84,12 @@
networking.wireless.enable = true;
networking.extraHosts = ''
- 10.243.206.102 habsys.de
- 10.243.206.102 pixelpocket.de
- 10.243.206.102 karlaskop.de
- 10.243.206.102 ubikmedia.de
- 10.243.206.102 apanowicz.de
+ 213.239.205.240 wohnprojekt-rhh.de
+ 213.239.205.240 karlaskop.de
+ 213.239.205.240 makeup.apanowicz.de
+ 213.239.205.240 pixelpocket.de
+ 213.239.205.240 reich-gebaeudereinigung.de
+ 213.239.205.240 o.ubikmedia.de
'';
hardware.enableAllFirmware = true;
From cef2be532b0cc76071b0b3515fc71214b37591f0 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Sat, 26 Dec 2015 10:44:56 +0100
Subject: [PATCH 29/65] m 3 Reaktor: add workdir/state_dir
---
krebs/3modules/Reaktor.nix | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/krebs/3modules/Reaktor.nix b/krebs/3modules/Reaktor.nix
index 1ec49b81e..d219d1800 100644
--- a/krebs/3modules/Reaktor.nix
+++ b/krebs/3modules/Reaktor.nix
@@ -62,6 +62,14 @@ let
configuration appended to the default or overridden configuration
'';
};
+
+ workdir = mkOption {
+ default = "/var/lib/Reaktor";
+ type = types.str;
+ description = ''
+ Reaktor working directory
+ '';
+ };
extraEnviron = mkOption {
default = {};
type = types.attrsOf types.str;
@@ -91,7 +99,7 @@ let
# uid = config.ids.uids.Reaktor;
uid = 2066439104; #genid Reaktor
description = "Reaktor user";
- home = "/var/lib/Reaktor";
+ home = cfg.workdir;
createHome = true;
};
@@ -113,6 +121,7 @@ let
GIT_SSL_CAINFO = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
REAKTOR_NICKNAME = cfg.nickname;
REAKTOR_DEBUG = (if cfg.debug then "True" else "False");
+ state_dir = cfg.workdir;
} // cfg.extraEnviron;
serviceConfig= {
ExecStartPre = pkgs.writeScript "Reaktor-init" ''
From 8f98ae9842963d801945c850e9da1e450e098ce3 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Sat, 26 Dec 2015 10:54:02 +0100
Subject: [PATCH 30/65] m 3 buildbot/master: use genid
---
krebs/3modules/buildbot/master.nix | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index 7078000fe..5870c3145 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -308,7 +308,7 @@ let
imp = {
users.extraUsers.buildbotMaster = {
- uid = 672626386; #genid buildbotMaster
+ uid = genid "buildbotMaster";
description = "Buildbot Master";
home = cfg.workDir;
createHome = false;
From 669e4be273ac2abe9505ca6411d5ee37f1771d4c Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Sat, 26 Dec 2015 11:06:11 +0100
Subject: [PATCH 31/65] k 5 Reaktor/plugins: converted plugins from
makefu/2/Reaktor
---
krebs/5pkgs/Reaktor/plugins.nix | 124 ++++++++++++++----
.../5pkgs/Reaktor/scripts}/random-issue.sh | 0
.../5pkgs/Reaktor/scripts}/sed-plugin.py | 0
.../5pkgs/Reaktor/scripts}/shack-correct.sh | 0
makefu/2configs/Reaktor/full.nix | 18 ---
makefu/2configs/Reaktor/random-emoji.nix | 26 ----
makefu/2configs/Reaktor/random-emoji.sh | 6 -
makefu/2configs/Reaktor/sed-plugin.nix | 18 ---
makefu/2configs/Reaktor/shack-correct.nix | 20 ---
makefu/2configs/Reaktor/simpleExtend.nix | 19 ---
makefu/2configs/Reaktor/stockholmLentil.nix | 27 ----
makefu/2configs/Reaktor/titlebot.nix | 38 ------
12 files changed, 102 insertions(+), 194 deletions(-)
rename {makefu/2configs/Reaktor => krebs/5pkgs/Reaktor/scripts}/random-issue.sh (100%)
rename {makefu/2configs/Reaktor => krebs/5pkgs/Reaktor/scripts}/sed-plugin.py (100%)
rename {makefu/2configs/Reaktor => krebs/5pkgs/Reaktor/scripts}/shack-correct.sh (100%)
delete mode 100644 makefu/2configs/Reaktor/full.nix
delete mode 100644 makefu/2configs/Reaktor/random-emoji.nix
delete mode 100644 makefu/2configs/Reaktor/random-emoji.sh
delete mode 100644 makefu/2configs/Reaktor/sed-plugin.nix
delete mode 100644 makefu/2configs/Reaktor/shack-correct.nix
delete mode 100644 makefu/2configs/Reaktor/simpleExtend.nix
delete mode 100644 makefu/2configs/Reaktor/stockholmLentil.nix
delete mode 100644 makefu/2configs/Reaktor/titlebot.nix
diff --git a/krebs/5pkgs/Reaktor/plugins.nix b/krebs/5pkgs/Reaktor/plugins.nix
index 05ede38e1..3b2508862 100644
--- a/krebs/5pkgs/Reaktor/plugins.nix
+++ b/krebs/5pkgs/Reaktor/plugins.nix
@@ -1,38 +1,118 @@
{ stdenv, lib, pkgs, makeWrapper }:
rec {
- buildReaktorPlugin = { name
- # TODO: profiles
- , extraConfig
+ # Begin API
+ buildBaseReaktorPlugin = { name
+ , config # python extra configuration for plugin
, phases ? []
, ... } @ attrs:
stdenv.mkDerivation (attrs // {
name = "Reaktor-plugin-" + name;
- phases = phases ++ [ "installPhase" ];
isReaktorPlugin = true;
});
- random-emoji = buildReaktorPlugin rec {
- name = "random-emoji";
- src = ./scripts/random-emoji.sh;
+ buildSimpleReaktorPlugin = name: { script
+ , path ? []
+ , env ? {}
+ , pattern ? ""
+ , ... } @ attrs:
+ let
+ path_env = { "PATH" = lib.makeSearchPath "bin" (path ++ [ pkgs.coreutils ]); };
+ src_dir = pkgs.substituteAll ( {
+ inherit name;
+ dir = "bin";
+ isExecutable = true;
+ src = script;
+ });
+ src_file = "${src_dir}/bin/${name}";
+ config = ''
+ public_commands.insert(0,{
+ 'capname' : "${name}",
+ 'pattern' : ${if pattern == "" then
+ ''indirect_pattern.format("${name}")'' else
+ ''"${pattern}"'' },
+ 'argv' : ["${src_file}"],
+ 'env' : ${builtins.toJSON path_env // env})})
+ '';
+ config_file = pkgs.writeText "plugin.py" config;
+ in buildBaseReaktorPlugin (attrs // rec {
+ inherit name config;
+
+ phases = [ "installPhase" ];
+ buildInputs = [ makeWrapper ];
+ installPhase = ''
+ mkdir -p $out/bin $out/etc/Reaktor
+ ln -s ${src_file} $out/bin
+ wrapProgram $out/bin/${name} \
+ --prefix PATH : ${path_env.PATH}
+ ln -s ${config_file} $out/etc/Reaktor/plugin.py
+ '';
+
+ });
+ # End API
+
+ # Begin Plugins
+ random-emoji = buildSimpleReaktorPlugin "emoji" {
+ path = with pkgs; [ gnused gnugrep xmlstarlet curl ];
+ script = ./scripts/random-emoji.sh;
+ };
+
+ sed-plugin = buildSimpleReaktorPlugin "sed-plugin" {
+ path = [ pkgs.gnused ];
+ # only support s///gi the plugin needs to see every msg
+ # TODO: this will eat up the last regex, fix Reaktor to support fallthru
+ pattern = "^(?P<args>.*)$$";
+ script = ./scripts/sed-plugin.py;
+ };
+
+ shack-correct = buildSimpleReaktorPlugin "shack-correct" {
+ path = [ pkgs.gnused ];
+ pattern = "^(?P<args>.*Shack.*)$$";
+ script = ./scripts/shack-correct.sh;
+ };
+
+ nixos-version = buildSimpleReaktorPlugin "nixos-version" {
+ script = pkgs.writeScript "nixos-version" ''
+ #! /bin/sh
+ . /etc/os-release
+ echo "$PRETTY_NAME"
+ '';
+ };
+ stockholm-issue = buildSimpleReaktorPlugin "stockholm-issue" {
+ script = ./scripts/random-issue.sh;
+ path = with pkgs; [ git gnused lentil ];
+ env = { "origin"= "http://cgit.gum/stockholm"; };
+ };
+
+ titlebot =
+ let
+ pypkgs = pkgs.python3Packages;
+ titlebot_cmds = pypkgs.buildPythonPackage {
+ name = "titlebot_cmds";
+ propagatedBuildInputs = with pypkgs; [ setuptools ];
+ src = pkgs.fetchurl {
+ url = "https://github.com/makefu/reaktor-titlebot/archive/2.1.0.tar.gz";
+ sha256 = "0wvf09wmk8b52f9j65qrw81nwrhs9pfhijwrlkzp5l7l2q8cjkp6";
+ };
+ };
+ in buildBaseReaktorPlugin rec {
+ name = "titlebot";
phases = [ "installPhase" ];
- buildInputs = [ makeWrapper ];
installPhase = ''
- mkdir -p $out/bin
- install -vm 755 ${src} $out/bin/random-emoji.sh
- wrapProgram $out/bin/random-emoji.sh \
- --prefix PATH : ${lib.makeSearchPath "bin" (with pkgs; [
- coreutils
- gnused
- gnugrep
- xmlstarlet
- curl])};
+ mkdir -p $out
+ ln -s ${titlebot_cmds}/* $out
'';
- extraConfig = ''
- public_commands.insert(0,{
- 'capname' : "emoji",
- 'pattern' : indirect_pattern.format("emoji"),
- 'argv' : ["random-emoji.sh"])
+ config = ''
+ def titlebot_cmd(cmd):
+ from os import environ
+ return { 'capname': cmd,
+ 'env': { 'TITLEDB':
+ environ['state_dir']+'/suggestions.json' },
+ 'pattern': '^\\.' + cmd + '\\s*(?:\\s+(?P<args>.*))?$$',
+ 'argv': [ '${titlebot_cmds}/bin/' + cmd ] }
+ for i in ['up','help','list','top','new']:
+ public_commands.insert(0,titlebot_cmd(i))
+ commands.insert(0,titlebot_cmd('clear'))
'';
};
}
diff --git a/makefu/2configs/Reaktor/random-issue.sh b/krebs/5pkgs/Reaktor/scripts/random-issue.sh
similarity index 100%
rename from makefu/2configs/Reaktor/random-issue.sh
rename to krebs/5pkgs/Reaktor/scripts/random-issue.sh
diff --git a/makefu/2configs/Reaktor/sed-plugin.py b/krebs/5pkgs/Reaktor/scripts/sed-plugin.py
similarity index 100%
rename from makefu/2configs/Reaktor/sed-plugin.py
rename to krebs/5pkgs/Reaktor/scripts/sed-plugin.py
diff --git a/makefu/2configs/Reaktor/shack-correct.sh b/krebs/5pkgs/Reaktor/scripts/shack-correct.sh
similarity index 100%
rename from makefu/2configs/Reaktor/shack-correct.sh
rename to krebs/5pkgs/Reaktor/scripts/shack-correct.sh
diff --git a/makefu/2configs/Reaktor/full.nix b/makefu/2configs/Reaktor/full.nix
deleted file mode 100644
index 50620890f..000000000
--- a/makefu/2configs/Reaktor/full.nix
+++ /dev/null
@@ -1,18 +0,0 @@
-_:
-{
- # implementation of the complete Reaktor bot
- imports = [
- #./stockholmLentil.nix
- ./simpleExtend.nix
- ./random-emoji.nix
- ./titlebot.nix
- ./shack-correct.nix
- ./sed-plugin.nix
- ];
- krebs.Reaktor.nickname = "Reaktor|bot";
- krebs.Reaktor.enable = true;
-
- krebs.Reaktor.extraEnviron = {
- REAKTOR_CHANNELS = "#krebs,#binaergewitter,#shackspace";
- };
-}
diff --git a/makefu/2configs/Reaktor/random-emoji.nix b/makefu/2configs/Reaktor/random-emoji.nix
deleted file mode 100644
index 3113a826b..000000000
--- a/makefu/2configs/Reaktor/random-emoji.nix
+++ /dev/null
@@ -1,26 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-with pkgs;
-let
- rpkg = pkgs.substituteAll( {
- name="random-emoji";
- dir= "bin";
- isExecutable=true;
- src= ./random-emoji.sh;
- });
- rpkg-path = lib.makeSearchPath "bin" (with pkgs; [
- coreutils
- gnused
- gnugrep
- xmlstarlet
- curl]);
-in {
- # TODO: make origin a variable, <- module is generic enough to handle different origins, not only stockholm
- krebs.Reaktor.extraConfig = ''
- public_commands.insert(0,{
- 'capname' : "emoji",
- 'pattern' : indirect_pattern.format("emoji"),
- 'argv' : ["${rpkg}/bin/random-emoji"],
- 'env' : { 'PATH':'${rpkg-path}' } })
- '';
-}
diff --git a/makefu/2configs/Reaktor/random-emoji.sh b/makefu/2configs/Reaktor/random-emoji.sh
deleted file mode 100644
index 386aa68b9..000000000
--- a/makefu/2configs/Reaktor/random-emoji.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/sh
-curl http://emojicons.com/random -s | \
- grep data-text | \
- sed -n 's/.*>\(.*\)<\/textarea>/\1/p' | \
- head -n 1 | \
- xmlstarlet unesc
diff --git a/makefu/2configs/Reaktor/sed-plugin.nix b/makefu/2configs/Reaktor/sed-plugin.nix
deleted file mode 100644
index a451e0d3e..000000000
--- a/makefu/2configs/Reaktor/sed-plugin.nix
+++ /dev/null
@@ -1,18 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-with pkgs;
-let
- script = ./sed-plugin.py;
-in {
- #TODO: this will eat up the last regex, fix Reaktor
- krebs.Reaktor.extraConfig = ''
- public_commands.append({
- 'capname' : "sed-plugin",
- # only support s///gi
- 'pattern' : '^(?P<args>.*)$$',
- 'argv' : ["${pkgs.python3}/bin/python3","${script}"],
- 'env' : { 'state_dir' : workdir,
- 'PATH':'${lib.makeSearchPath "bin" [pkgs.gnused]}' }})
- '';
-}
-
diff --git a/makefu/2configs/Reaktor/shack-correct.nix b/makefu/2configs/Reaktor/shack-correct.nix
deleted file mode 100644
index 8f30807f1..000000000
--- a/makefu/2configs/Reaktor/shack-correct.nix
+++ /dev/null
@@ -1,20 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-with pkgs;
-let
- script = pkgs.substituteAll ( {
- name="shack-correct";
- isExecutable=true;
- dir = "";
- src = ./shack-correct.sh;
- });
-in {
- krebs.Reaktor.extraConfig = ''
- public_commands.insert(0,{
- 'capname' : "shack-correct",
- 'pattern' : '^(?P<args>.*Shack.*)$$',
- 'argv' : ["${script}"],
- 'env' : { }})
- '';
-}
-
diff --git a/makefu/2configs/Reaktor/simpleExtend.nix b/makefu/2configs/Reaktor/simpleExtend.nix
deleted file mode 100644
index 95175a4e0..000000000
--- a/makefu/2configs/Reaktor/simpleExtend.nix
+++ /dev/null
@@ -1,19 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-with pkgs;
-let
- nixos-version-script = pkgs.writeScript "nix-version" ''
- #! /bin/sh
- . /etc/os-release
- echo "$PRETTY_NAME"
- '';
-in {
- krebs.Reaktor.extraConfig = ''
- public_commands.insert(0,{
- 'capname' : "nixos-version",
- 'pattern' : indirect_pattern.format("nixos-version"),
- 'argv' : ["${nixos-version-script}"],
- 'env' : { 'state_dir': workdir } })
- '';
-}
-
diff --git a/makefu/2configs/Reaktor/stockholmLentil.nix b/makefu/2configs/Reaktor/stockholmLentil.nix
deleted file mode 100644
index 21f0305fb..000000000
--- a/makefu/2configs/Reaktor/stockholmLentil.nix
+++ /dev/null
@@ -1,27 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-with pkgs;
-let
- random-issue = pkgs.substituteAll( {
- name="random-issue";
- dir= "bin";
- isExecutable=true;
- src= ./random-issue.sh;
- });
- random-issue-path = lib.makeSearchPath "bin" (with pkgs; [
- coreutils
- git
- gnused
- lentil]);
-in {
- # TODO: make origin a variable, <- module is generic enough to handle different origins, not only stockholm
- krebs.Reaktor.extraConfig = ''
- public_commands.insert(0,{
- 'capname' : "stockholm-issue",
- 'pattern' : indirect_pattern.format("stockholm-issue"),
- 'argv' : ["${random-issue}/bin/random-issue"],
- 'env' : { 'state_dir': workdir,
- 'PATH':'${random-issue-path}',
- 'origin':'http://cgit.pnp/stockholm' } })
- '';
-}
diff --git a/makefu/2configs/Reaktor/titlebot.nix b/makefu/2configs/Reaktor/titlebot.nix
deleted file mode 100644
index 9ef02548b..000000000
--- a/makefu/2configs/Reaktor/titlebot.nix
+++ /dev/null
@@ -1,38 +0,0 @@
-{ stdenv,config, lib, pkgs, ... }:
-
-with pkgs;
-let
- pypkgs = pkgs.python3Packages;
- titlebot_cmds = pypkgs.buildPythonPackage {
- name = "titlebot_cmds";
- propagatedBuildInputs = with pypkgs; [ setuptools ];
- src = fetchurl {
- # https://github.com/makefu/reaktor-titlebot tag 2.1.0
- url = "https://github.com/makefu/reaktor-titlebot/archive/2.1.0.tar.gz";
- sha256 = "0wvf09wmk8b52f9j65qrw81nwrhs9pfhijwrlkzp5l7l2q8cjkp6";
- };
- };
- pub_cmds = ["up" "help" "list" "top" "highest" "undo" ];
- priv_cmds = [ "clear" ];
-in {
- # TODO: write identify file in
- # {config.users.extraUsers.Reaktor.home}/state/admin.lst
- krebs.Reaktor.extraConfig = ''
- def titlebot_cmd(cmd):
- return {
- 'capname': cmd,
- 'env': {
- 'TITLEDB':
- '${config.users.extraUsers.Reaktor.home}/suggestions.json'
- },
- 'pattern': '^\\.' + cmd + '\\s*(?:\\s+(?P<args>.*))?$$',
- 'argv': [ '${titlebot_cmds}/bin/' + cmd ] }
- # TODO: for each element in ${titlebot_cmds}/bin/*
- public_commands.insert(0,titlebot_cmd('up'))
- public_commands.insert(0,titlebot_cmd('help'))
- public_commands.insert(0,titlebot_cmd('list'))
- public_commands.insert(0,titlebot_cmd('top'))
- public_commands.insert(0,titlebot_cmd('new'))
- commands.insert(0,titlebot_cmd('clear'))
- '';
-}
From a2f5e7e320bb0fbca0a0694d91e4fb20dc4ef329 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Sat, 26 Dec 2015 11:31:09 +0100
Subject: [PATCH 32/65] k 5 ReaktorPlugins: hotfix for env generation
---
krebs/5pkgs/Reaktor/plugins.nix | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/krebs/5pkgs/Reaktor/plugins.nix b/krebs/5pkgs/Reaktor/plugins.nix
index 3b2508862..b1a61d3fa 100644
--- a/krebs/5pkgs/Reaktor/plugins.nix
+++ b/krebs/5pkgs/Reaktor/plugins.nix
@@ -32,7 +32,7 @@ rec {
''indirect_pattern.format("${name}")'' else
''"${pattern}"'' },
'argv' : ["${src_file}"],
- 'env' : ${builtins.toJSON path_env // env})})
+ 'env' : ${builtins.toJSON (path_env // env)})})
'';
config_file = pkgs.writeText "plugin.py" config;
in buildBaseReaktorPlugin (attrs // rec {
From 7bed1761bdbfc3fc7e2df56dcf069511eec2a97d Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Sat, 26 Dec 2015 11:41:41 +0100
Subject: [PATCH 33/65] m 3 Reaktor: now supports plugin infra see
m/1/pornocauster
---
krebs/3modules/Reaktor.nix | 5 ++++-
krebs/5pkgs/Reaktor/plugins.nix | 4 ++--
makefu/1systems/pornocauster.nix | 13 +++++++------
3 files changed, 13 insertions(+), 9 deletions(-)
diff --git a/krebs/3modules/Reaktor.nix b/krebs/3modules/Reaktor.nix
index 59058bffc..607eb7cac 100644
--- a/krebs/3modules/Reaktor.nix
+++ b/krebs/3modules/Reaktor.nix
@@ -9,6 +9,7 @@ let
${cfg.overrideConfig}
'' else ""}
## Extra Config
+ ${concatStringsSep "\n" (map (plug: plug.config) cfg.plugins)}
${cfg.extraConfig}
'';
cfg = config.krebs.Reaktor;
@@ -35,7 +36,6 @@ let
'';
};
-
overrideConfig = mkOption {
default = null;
type = types.nullOr types.str;
@@ -44,6 +44,9 @@ let
Reaktor default cfg can be retrieved via `reaktor get-config`
'';
};
+ plugins = mkOption {
+ default = [pkgs.ReaktorPlugins.nixos-version];
+ };
extraConfig = mkOption {
default = "";
type = types.string;
diff --git a/krebs/5pkgs/Reaktor/plugins.nix b/krebs/5pkgs/Reaktor/plugins.nix
index b1a61d3fa..5c7b89f5c 100644
--- a/krebs/5pkgs/Reaktor/plugins.nix
+++ b/krebs/5pkgs/Reaktor/plugins.nix
@@ -32,7 +32,7 @@ rec {
''indirect_pattern.format("${name}")'' else
''"${pattern}"'' },
'argv' : ["${src_file}"],
- 'env' : ${builtins.toJSON (path_env // env)})})
+ 'env' : ${builtins.toJSON (path_env // env)} })
'';
config_file = pkgs.writeText "plugin.py" config;
in buildBaseReaktorPlugin (attrs // rec {
@@ -81,7 +81,7 @@ rec {
stockholm-issue = buildSimpleReaktorPlugin "stockholm-issue" {
script = ./scripts/random-issue.sh;
path = with pkgs; [ git gnused lentil ];
- env = { "origin"= "http://cgit.gum/stockholm"; };
+ env = { "origin" = "http://cgit.gum/stockholm"; };
};
titlebot =
diff --git a/makefu/1systems/pornocauster.nix b/makefu/1systems/pornocauster.nix
index 28b77d330..690e26b36 100644
--- a/makefu/1systems/pornocauster.nix
+++ b/makefu/1systems/pornocauster.nix
@@ -26,6 +26,7 @@
# services
../2configs/git/brain-retiolum.nix
../2configs/tor.nix
+ # ../2configs/buildbot-standalone.nix
# hardware specifics are in here
../2configs/hw/tp-x220.nix
@@ -36,14 +37,14 @@
];
nixpkgs.config.packageOverrides = pkgs: {
tinc = pkgs.tinc_pre;
- buildbot = let
- pkgs1509 = import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-unstable.tar.gz) {};
- in pkgs1509.buildbot;
};
- makefu.buildbot.master.enable = true;
- #krebs.Reaktor.enable = true;
- #krebs.Reaktor.nickname = "makefu|r";
+ krebs.Reaktor = {
+ enable = true;
+ nickname = "makefu|r";
+ plugins = with pkgs.ReaktorPlugins; [ nixos-version random-emoji ];
+ };
+
# nix.binaryCaches = [ "http://acng.shack/nixos" "https://cache.nixos.org" ];
environment.systemPackages = with pkgs;[
From 70d0cae1d3831cd24b3e2ac68a927132f71d2801 Mon Sep 17 00:00:00 2001
From: lassulus <lass@aidsballs.de>
Date: Sat, 26 Dec 2015 17:30:59 +0100
Subject: [PATCH 34/65] l 2: move pkgs from xserver to baseX
---
lass/2configs/baseX.nix | 9 +++++++--
lass/2configs/xserver/default.nix | 14 --------------
2 files changed, 7 insertions(+), 16 deletions(-)
diff --git a/lass/2configs/baseX.nix b/lass/2configs/baseX.nix
index 4e46c18d2..0596682df 100644
--- a/lass/2configs/baseX.nix
+++ b/lass/2configs/baseX.nix
@@ -31,10 +31,15 @@ in {
environment.systemPackages = with pkgs; [
- powertop
- sxiv
+ gitAndTools.qgit
+ mpv
much
+ pavucontrol
+ powertop
push
+ slock
+ sxiv
+ xsel
zathura
#window manager stuff
diff --git a/lass/2configs/xserver/default.nix b/lass/2configs/xserver/default.nix
index da337f6a7..04d14c7ce 100644
--- a/lass/2configs/xserver/default.nix
+++ b/lass/2configs/xserver/default.nix
@@ -7,7 +7,6 @@ let
user = config.users.users.mainUser;
out = {
-
services.xserver = {
display = 11;
tty = 11;
@@ -41,16 +40,6 @@ let
};
};
- environment.systemPackages = [
- pkgs.gitAndTools.qgit
- pkgs.mpv
- pkgs.pavucontrol
- pkgs.slock
- pkgs.sxiv
- pkgs.xsel
- pkgs.zathura
- ];
-
security.setuidPrograms = [
"slock"
];
@@ -106,9 +95,6 @@ let
set -efu
export PATH; PATH=${makeSearchPath "bin" ([
pkgs.rxvt_unicode
- pkgs.i3lock
- pkgs.haskellPackages.yeganesh
- pkgs.dmenu
] ++ config.environment.systemPackages)}:/var/setuid-wrappers
settle() {(
# Use PATH for a clean journal
From f22fe4e5d97237dbe76bc856909950487634c7be Mon Sep 17 00:00:00 2001
From: lassulus <lass@aidsballs.de>
Date: Sat, 26 Dec 2015 17:31:38 +0100
Subject: [PATCH 35/65] l 5 newsbot-js: rev 6ee4884 -> 802b172
---
lass/5pkgs/newsbot-js/default.nix | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lass/5pkgs/newsbot-js/default.nix b/lass/5pkgs/newsbot-js/default.nix
index 0d194e6f3..cabd7422c 100644
--- a/lass/5pkgs/newsbot-js/default.nix
+++ b/lass/5pkgs/newsbot-js/default.nix
@@ -26,8 +26,8 @@ in nodePackages.buildNodePackage {
src = fetchgit {
url = "http://cgit.echelon/newsbot-js/";
- rev = "6ee488430c6915eeae03f1569084577d39cef51d";
- sha256 = "00xmn7hzcs0mm6hjf5i37d9nna5rcd0gra0ynch7x2id8liazksx";
+ rev = "802b172d0eed6c9625a9cb5db408f5cc8c01784e";
+ sha256 = "794fc7845aca311f7cf7b6bdc109b5a25d0e2299322bc6612edadc477b2536e2";
};
phases = [
From 743842268327b5fd12ba4d19b6260d47535976a3 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Mon, 28 Dec 2015 10:58:13 +0100
Subject: [PATCH 36/65] k 5 default: populate supports infesting arg
by setting infesting for populate, data will be written to /mnt instead of root.
---
krebs/default.nix | 21 +++++++++++++++------
1 file changed, 15 insertions(+), 6 deletions(-)
diff --git a/krebs/default.nix b/krebs/default.nix
index ad0205426..81ddd3ea6 100644
--- a/krebs/default.nix
+++ b/krebs/default.nix
@@ -36,6 +36,7 @@ let out = {
{ system ? current-host-name
, target ? system
}@args: let
+ config = get-config system;
in ''
#! /bin/sh
# ${current-date} ${current-user-name}@${current-host-name}
@@ -47,6 +48,10 @@ let out = {
${builtins.readFile ./4lib/infest/install-nix.sh}
''}
+ # Prepare target source via bind-mounting
+
+ (${populate (args // { infesting = true;}) })
+
(${nixos-install args})
${rootssh target ''
@@ -98,7 +103,6 @@ let out = {
#! /bin/sh
# ${current-date} ${current-user-name}@${current-host-name}
# krebs.nixos-install
- (${populate args})
${rootssh target ''
export PATH; PATH=/root/.nix-profile/bin:$PATH
@@ -205,6 +209,7 @@ let out = {
populate =
{ system ? current-host-name
, target ? system
+ , infesting ? false
}@args:
let out = ''
#! /bin/sh
@@ -217,6 +222,8 @@ let out = {
["dir" "git"])}
'';
+
+ target_prefix=lib.optionalString infesting "/mnt";
config = get-config system;
current-host = config.krebs.hosts.${current-host-name};
@@ -225,17 +232,18 @@ let out = {
methods.dir = config:
let
can-push = config.host.name == current-host.name;
+ target-path = target_prefix + config.target-path;
push-method = ''
rsync \
--exclude .git \
--exclude .graveyard \
--exclude old \
--exclude tmp \
- --rsync-path='mkdir -p ${config.target-path} && rsync' \
+ --rsync-path='mkdir -p ${target-path} && rsync' \
--delete-excluded \
-vrLptgoD \
${config.path}/ \
- root@${target}:${config.target-path}
+ root@${target}:${target-path}
'';
in
if can-push then push-method else
@@ -244,9 +252,10 @@ let out = {
throw "No way to push ${dir} from ${current-host.name} to ${target}";
methods.git = config:
- rootssh target ''
- mkdir -p ${config.target-path}
- cd ${config.target-path}
+ let target-path = target_prefix + config.target-path;
+ in rootssh target ''
+ mkdir -p ${target-path}
+ cd ${target-path}
if ! test -e .git; then
git init
fi
From 95caa8d7fb6f72cbf5064256f71750096d32a6c0 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Mon, 28 Dec 2015 13:23:34 +0100
Subject: [PATCH 37/65] k 5 test/infest-cac-centos7: use defer, loop until we
get a working cac box
---
krebs/5pkgs/test/infest-cac-centos7/notes | 73 +++++++++++++++--------
1 file changed, 49 insertions(+), 24 deletions(-)
diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes
index cfb074423..3f4fcd859 100755
--- a/krebs/5pkgs/test/infest-cac-centos7/notes
+++ b/krebs/5pkgs/test/infest-cac-centos7/notes
@@ -8,6 +8,17 @@ set -eufx
krebs_cred=${krebs_cred-./cac.json}
retiolum_key=${retiolum_key-./retiolum.rsa_key.priv}
+clear_defer(){
+ echo "${trapstr:-exit}"
+ trap - INT TERM EXIT KILL
+}
+defer(){
+ if test -z "${debug:-}"; then
+ trapstr="$1;${trapstr:-exit}"
+ trap "$trapstr" INT TERM EXIT KILL
+ fi
+}
+
# Sanity
if test ! -r "$krebs_cred";then
echo "\$krebs_cred=$krebs_cred must be readable"; exit 1
@@ -24,8 +35,8 @@ export cac_servers_cache=$krebs_secrets/servers_cache.json
export cac_tasks_cache=$krebs_secrets/tasks_cache.json
export cac_templates_cache=$krebs_secrets/templates_cache.json
# we need to receive this key from buildmaster to speed up tinc bootstrap
-TRAP="rm -r $krebs_secrets;trap - INT TERM EXIT"
-trap "$TRAP" INT TERM EXIT
+defer "trap - INT TERM EXIT"
+defer "rm -r $krebs_secrets"
cat > $sec_file <<EOF
cac_login="$(jq -r .email $krebs_cred)"
@@ -39,30 +50,44 @@ cac-cli --config $krebs_cred panel add-api-ip
cac update
cac servers
-# Template 26: CentOS7
-# TODO: use cac templates to determine the real Centos7 template in case it changes
-name=$( cac build cpu=1 ram=512 storage=10 os=26 2>&1\
- | jq -r .servername)
+# preserve old trap
+old_trapstr=$(clear_defer)
+while true;do
+ # Template 26: CentOS7
+ # TODO: use cac templates to determine the real Centos7 template in case it changes
+ name=$( cac build cpu=1 ram=512 storage=10 os=26 2>&1\
+ | jq -r .servername)
+ id=servername:$name
-id=servername:$name
-trap "cac delete $id;$TRAP;exit" INT TERM EXIT
-# TODO: timeout?
+ clear_defer >/dev/null
+ defer "cac delete $id"
-wait_login_cac(){
- # timeout
- for t in `seq 180`;do
- # now we have a working cac server
- if cac ssh $1 -o ConnectTimeout=10 \
- cat /etc/redhat-release | \
- grep CentOS ;then
- return 0
- fi
- sleep 10
- done
- return 1
-}
-# die on timeout
-wait_login_cac $id
+ # TODO: timeout?
+
+ wait_login_cac(){
+ # we wait for 15 minutes
+ for t in `seq 90`;do
+ # now we have a working cac server
+ if cac ssh $1 -o ConnectTimeout=10 \
+ cat /etc/redhat-release | \
+ grep CentOS ;then
+ return 0
+ fi
+ sleep 10
+ done
+ return 1
+ }
+ # die on timeout
+ if ! wait_login_cac $id;then
+ echo "unable to boot a working system within time frame, retrying..." >&2
+ echo "Cleaning up old image,last status: $(cac update;cac getserver $id | jq -r .status)"
+ eval "$(clear_defer)"
+ else
+ echo "got a working system" >&2
+ fi
+done
+clear_defer >/dev/null
+defer "cac delete $id;$old_trapstr"
mkdir -p shared/2configs/temp
cac generatenetworking $id > \
From 246116dabbe849e75612fbdb57b01696913ff27e Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Mon, 28 Dec 2015 13:29:11 +0100
Subject: [PATCH 38/65] m 2 urlwatch: replace url for cvs2svn
---
makefu/2configs/urlwatch.nix | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/makefu/2configs/urlwatch.nix b/makefu/2configs/urlwatch.nix
index eadffa7dd..e4f639d5b 100644
--- a/makefu/2configs/urlwatch.nix
+++ b/makefu/2configs/urlwatch.nix
@@ -12,7 +12,7 @@
http://git.sysphere.org/vicious/log/?qt=grep&q=Next+release
https://pypi.python.org/simple/bepasty/
https://pypi.python.org/simple/xstatic/
- http://cvs2svn.tigris.org/servlets/ProjectDocumentList?folderID=2976
+ http://cvs2svn.tigris.org/svn/cvs2svn/tags/
];
};
}
From 676d0f748138f0e1fa3cb2177b5a08a857f17fac Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Tue, 29 Dec 2015 12:52:35 +0100
Subject: [PATCH 39/65] k Zhosts: init bobby (miefda)
---
krebs/Zhosts/bobby | 11 +++++++++++
1 file changed, 11 insertions(+)
create mode 100644 krebs/Zhosts/bobby
diff --git a/krebs/Zhosts/bobby b/krebs/Zhosts/bobby
new file mode 100644
index 000000000..aac6e377b
--- /dev/null
+++ b/krebs/Zhosts/bobby
@@ -0,0 +1,11 @@
+Subnet = 10.243.111.112/32
+Subnet = 42:0:0:0:0:0:111:112/128
+
+-----BEGIN RSA PUBLIC KEY-----
+MIIBCgKCAQEA+AScnIqFdzGl+iRZTNZ7r91n/r1H4GzDsrAupUvJ4mi7nDN4eP8s
+uLvKtJp22RxfuF3Kf4KhHb8LHQ8bLLN/KDaNDXrCNBc69d7vvLsjoY+wfGLJNu4Y
+Ad/8J4r3rdb83mTA3IHb47T/70MERPBr2gF84YiG6ZoQrPQuTk4lHxaI83SOhjny
+0F0ucS/rBV6Vv9y5/756TKi1cFPSpY4X+qeWc8xWrBGJcJiiqYb8ZX2o/lkAJ5c+
+jI/VdybGFVGY9+bp4Jw5xBIo5KGuFnm8+blRmSDDl3joRneKQSx9FAu7RUwoajBu
+cEbi1529NReQzIFT6Vt22ymbHftxOiuh4QIDAQAB
+-----END RSA PUBLIC KEY-----
From 5fb35e60a8e1b997473e7657d342802d533cd070 Mon Sep 17 00:00:00 2001
From: lassulus <lass@aidsballs.de>
Date: Tue, 29 Dec 2015 19:30:57 +0100
Subject: [PATCH 40/65] k hosts: add kebsco
---
krebs/Zhosts/kebsco | 11 +++++++++++
1 file changed, 11 insertions(+)
create mode 100644 krebs/Zhosts/kebsco
diff --git a/krebs/Zhosts/kebsco b/krebs/Zhosts/kebsco
new file mode 100644
index 000000000..2fd1c5f42
--- /dev/null
+++ b/krebs/Zhosts/kebsco
@@ -0,0 +1,11 @@
+Subnet = 10.243.212.68
+Subnet = 42:9d30:3845:c822:988b:96c5:39ab:90b7
+
+-----BEGIN RSA PUBLIC KEY-----
+MIIBCgKCAQEA0dEwTZh2uzJpP9GL7YRyiLuezJqYiJ8/4Bl4IPshJnuO9IGbEcto
+0cFm9uM9gxxqggfaCi96DsIQNlyqff2vDfEj3mdIu9T3tkRROByQF8y1NWX29NyH
+zZEX8Ri8u4U2KdYTEzPXEFxBEl0GQX9mMtlvwzCq7V4ueCcWB1xDA+DtJjpd894z
+3FOw0rIxYmfYhLAL5B3rzF74bcHFGV30f4JWq11wLBkyR6/Q5gxgZzkKYGwdZ/SN
+C6gg86abKdp65/Wq5P331IbwPBal1ZhGbaAo1y7JpjpLvZytI2jboXeQuPZ8P5hU
+L3zKKceAibPKrw9+y8lb+IKoYLF7I1KYIwIDAQAB
+-----END RSA PUBLIC KEY-----
From 9abd00f6af48676f08b6afdddd03d12410a1d1cd Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Tue, 29 Dec 2015 21:20:11 +0100
Subject: [PATCH 41/65] k 3 makefu: add ssh pubkeys to hosts
---
krebs/3modules/makefu/default.nix | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/krebs/3modules/makefu/default.nix b/krebs/3modules/makefu/default.nix
index 1970a0777..31516d591 100644
--- a/krebs/3modules/makefu/default.nix
+++ b/krebs/3modules/makefu/default.nix
@@ -83,6 +83,9 @@ with lib;
'';
};
};
+ ssh.privkey.path = <secrets/ssh_host_ed25519_key>;
+ ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHDM0E608d/6rGzXqGbNSuMb2RlCojCJSiiz6QcPOC2G root@pornocauster";
+
};
vbob = {
@@ -108,6 +111,8 @@ with lib;
'';
};
};
+ ssh.privkey.path = <secrets/ssh_host_ed25519_key>;
+ ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICPLTMl+thSq77cjYa2XF7lz5fA7JMftrLo8Dy/OBXSg root@nixos";
};
flap = rec {
cores = 1;
@@ -238,6 +243,8 @@ with lib;
'';
};
};
+ ssh.privkey.path = <secrets/ssh_host_ed25519_key>;
+ ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH4Tjx9qK6uWtxT1HCpeC0XvDZKO/kaPygyKatpAqU6I root@wry";
};
filepimp = rec {
cores = 1;
@@ -287,6 +294,8 @@ with lib;
'';
};
};
+ ssh.privkey.path = <secrets/ssh_host_ed25519_key>;
+ ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIujMZ3ZFxKpWeB/cjfKfYRr77+VRZk0Eik+92t03NoA root@servarch";
};
gum = rec {
cores = 1;
@@ -327,6 +336,8 @@ with lib;
'';
};
};
+ ssh.privkey.path = <secrets/ssh_host_ed25519_key>;
+ ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIcxWFEPzke/Sdd9qNX6rSJgXal8NmINYajpFCxXfYdj root@gum";
};
};
users = addNames rec {
From 4b6cd401a85cdc7aab150208cc5310645a7e59e2 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Tue, 29 Dec 2015 21:20:36 +0100
Subject: [PATCH 42/65] m 1 gum: add smart monitor
---
makefu/1systems/gum.nix | 3 ++-
makefu/2configs/smart-monitor.nix | 18 ++++++++++++++++++
2 files changed, 20 insertions(+), 1 deletion(-)
create mode 100644 makefu/2configs/smart-monitor.nix
diff --git a/makefu/1systems/gum.nix b/makefu/1systems/gum.nix
index 93fb3dc3a..1907424ec 100644
--- a/makefu/1systems/gum.nix
+++ b/makefu/1systems/gum.nix
@@ -6,11 +6,11 @@ let
internal-ip = head config.krebs.build.host.nets.retiolum.addrs4;
in {
imports = [
- # TODO: copy this config or move to krebs
../2configs/tinc-basic-retiolum.nix
../2configs/headless.nix
../2configs/fs/simple-swap.nix
../2configs/fs/single-partition-ext4.nix
+ ../2configs/smart-monitor.nix
# ../2configs/iodined.nix
../2configs/git/cgit-retiolum.nix
../2configs/mattermost-docker.nix
@@ -18,6 +18,7 @@ in {
../2configs/exim-retiolum.nix
../2configs/urlwatch.nix
+
];
diff --git a/makefu/2configs/smart-monitor.nix b/makefu/2configs/smart-monitor.nix
new file mode 100644
index 000000000..7086f622b
--- /dev/null
+++ b/makefu/2configs/smart-monitor.nix
@@ -0,0 +1,18 @@
+{ config, ... }:
+{
+ services.smartd = {
+ enable = true;
+ notifications = {
+ mail = {
+ enable = true;
+ recipient = config.krebs.users.makefu.mail;
+ };
+ };
+ # short daily, long weekly, check on boot
+ defaults.monitored = "-a -o on -s (S/../.././02|L/../../7/04)";
+
+ devices = [{
+ device = "/dev/sda";
+ }];
+ };
+}
From 81badfd47ede4cf3860e7006c13586340415ade5 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Tue, 29 Dec 2015 21:21:04 +0100
Subject: [PATCH 43/65] m 2 urlwatch: use py2 instead of py2k urlwatch
---
makefu/2configs/urlwatch.nix | 20 ++++++++++++++++++--
1 file changed, 18 insertions(+), 2 deletions(-)
diff --git a/makefu/2configs/urlwatch.nix b/makefu/2configs/urlwatch.nix
index e4f639d5b..a83279ba2 100644
--- a/makefu/2configs/urlwatch.nix
+++ b/makefu/2configs/urlwatch.nix
@@ -1,6 +1,22 @@
-{ config, ... }:
+{ config, lib, ... }:
{
+ nixpkgs.config.packageOverrides = pkgs: {
+ urlwatch = with pkgs.pythonPackages; buildPythonPackage rec {
+ name = "urlwatch-1.18";
+
+ propagatedBuildInputs = [ futures ];
+
+ src = pkgs.fetchurl {
+ url = "http://thp.io/2008/urlwatch/${name}.tar.gz";
+ sha256 = "090qfgx249ks7103sap6w47f8302ix2k46wxhfssxwsqcqdl25vb";
+ };
+
+ postFixup = ''
+ wrapProgram "$out/bin/urlwatch" --prefix "PYTHONPATH" : "$PYTHONPATH"
+ '';
+ };
+ };
krebs.urlwatch = {
enable = true;
mailto = config.krebs.users.makefu.mail;
@@ -12,7 +28,7 @@
http://git.sysphere.org/vicious/log/?qt=grep&q=Next+release
https://pypi.python.org/simple/bepasty/
https://pypi.python.org/simple/xstatic/
- http://cvs2svn.tigris.org/svn/cvs2svn/tags/
+ http://guest:derpi@cvs2svn.tigris.org/svn/cvs2svn/tags/
];
};
}
From d574c0ef78f7572aec88e484d3ff6256247e878c Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 30 Dec 2015 01:38:33 +0100
Subject: [PATCH 44/65] m 3 buildbot/master: add secrets
---
krebs/3modules/buildbot/master.nix | 16 +++++++++++++---
shared/2configs/buildbot-standalone.nix | 8 +++++---
2 files changed, 18 insertions(+), 6 deletions(-)
diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index 5870c3145..74385a433 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -132,6 +132,16 @@ let
'';
};
+ secrets = mkOption {
+ default = [];
+ type = types.listOf types.str;
+ example = [ "cac.json" ];
+ description = ''
+ List of all the secrets in <secrets> which should be copied into the
+ buildbot master directory.
+ '';
+ };
+
slaves = mkOption {
default = {};
type = types.attrsOf types.str;
@@ -344,10 +354,10 @@ let
fi
# always override the master.cfg
cp ${buildbot-master-config} ${workdir}/master.cfg
+
# copy secrets
- cp ${secretsdir}/cac.json ${workdir}
- cp ${secretsdir}/retiolum-ci.rsa_key.priv \
- ${workdir}/retiolum.rsa_key.priv
+ ${ concatMapStringsSep "\n"
+ (f: "cp ${secretsdir}/${f} ${workdir}/${f}" ) cfg.secrets }
# sanity
${buildbot}/bin/buildbot checkconfig ${workdir}
diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix
index 51c600329..28d1eef2e 100644
--- a/shared/2configs/buildbot-standalone.nix
+++ b/shared/2configs/buildbot-standalone.nix
@@ -8,6 +8,9 @@ in {
};
networking.firewall.allowedTCPPorts = [ 8010 9989 ];
krebs.buildbot.master = {
+ secrets = [
+ "cac.json"
+ ];
slaves = {
testslave = "krebspass";
omo = "krebspass";
@@ -93,9 +96,8 @@ in {
# slave needs 2 files:
# * cac.json
# * retiolum
- for file in ["cac.json", "retiolum.rsa_key.priv"]:
- s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/{}".format(file),
- slavedest=file))
+ s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/cac.json", slavedest="cac.json"))
+ s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/retiolum-ci.rsa_key.priv", slavedest="retiolum.rsa_key.priv"))
addShell(s, name="infest-cac-centos7",env=env,
sigtermTime=60, # SIGTERM 1 minute before SIGKILL
From a8cb9d41bdefe8a5dc72ca76b3ebc5b4047ddf65 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 30 Dec 2015 02:45:47 +0100
Subject: [PATCH 45/65] s 1 test-all-krebs-modules: init
---
krebs/3modules/shared/default.nix | 1 +
shared/1systems/test-all-krebs-modules.nix | 45 ++++++++++++++++++++++
shared/2configs/buildbot-standalone.nix | 17 ++++++--
3 files changed, 59 insertions(+), 4 deletions(-)
create mode 100644 shared/1systems/test-all-krebs-modules.nix
diff --git a/krebs/3modules/shared/default.nix b/krebs/3modules/shared/default.nix
index b332676c6..518e46587 100644
--- a/krebs/3modules/shared/default.nix
+++ b/krebs/3modules/shared/default.nix
@@ -7,6 +7,7 @@ let
"test-arch"
"test-centos6"
"test-centos7"
+ "test-all-krebs-modules"
] (name: {
inherit name;
cores = 1;
diff --git a/shared/1systems/test-all-krebs-modules.nix b/shared/1systems/test-all-krebs-modules.nix
new file mode 100644
index 000000000..b98004dfe
--- /dev/null
+++ b/shared/1systems/test-all-krebs-modules.nix
@@ -0,0 +1,45 @@
+{ config, pkgs, lib, ... }:
+let
+ en = { enable = true;};
+in {
+ krebs = {
+ enable = true;
+ build.user = config.krebs.users.shared;
+ build.host = config.krebs.hosts.test-all-krebs-modules;
+ Reaktor.enable = true;
+ apt-cacher-ng.enable = true;
+ backup.enable = true;
+ bepasty.enable = true;
+ buildbot.master.enable = true;
+ buildbot.slave = {
+ enable = true;
+ username = "lol";
+ password = "wut";
+ };
+ exim-retiolum.enable = true;
+ exim-smarthost = {
+ enable = true;
+ system-aliases = [ { from = "dick"; to = "butt"; } ];
+ };
+ go.enable = true;
+ iptables = {
+ enable = true;
+ tables = {};
+ };
+ nginx.enable = true;
+ realwallpaper.enable = true;
+ retiolum.enable = true;
+ retiolum-bootstrap.enable = true;
+ tinc_graphs.enable = true;
+ urlwatch.enable = true;
+ fetchWallpaper = {
+ enable = true;
+ url ="localhost";
+ };
+ };
+ # just get the system running
+ boot.loader.grub.devices = ["/dev/sda"];
+ fileSystems."/" = {
+ device = "/dev/lol";
+ };
+}
diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix
index 28d1eef2e..22e9861cc 100644
--- a/shared/2configs/buildbot-standalone.nix
+++ b/shared/2configs/buildbot-standalone.nix
@@ -8,12 +8,9 @@ in {
};
networking.firewall.allowedTCPPorts = [ 8010 9989 ];
krebs.buildbot.master = {
- secrets = [
- "cac.json"
- ];
+ secrets = [ "retiolum-ci.rsa_key.priv" "cac.json" ];
slaves = {
testslave = "krebspass";
- omo = "krebspass";
};
change_source.stockholm = ''
stockholm_repo = 'http://cgit.gum/stockholm'
@@ -85,6 +82,18 @@ in {
addShell(f,name="eval-cross-check",env=env,
command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"])
+ addShell(f,name="instaniate-test-all-modules",env=env,
+ command=nixshell + \
+ ["touch retiolum.rsa_key.priv; \
+ nix-instantiate --eval -A \
+ users.shared.test-all-krebs-modules.system \
+ -I stockholm=. \
+ -I secrets=. '<stockholm>' \
+ --argstr current-date lol \
+ --argstr current-user-name shared \
+ --argstr current-host-name lol \
+ --strict --json"])
+
bu.append(util.BuilderConfig(name="fast-tests",
slavenames=slavenames,
factory=f))
From 2bee6a0f6d71fbb1e1f02bd0df396bc38d3b6f67 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 30 Dec 2015 04:43:16 +0100
Subject: [PATCH 46/65] s 2 buildbot-standalone: cosmetics
---
shared/2configs/buildbot-standalone.nix | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix
index 22e9861cc..c62f8920c 100644
--- a/shared/2configs/buildbot-standalone.nix
+++ b/shared/2configs/buildbot-standalone.nix
@@ -73,16 +73,16 @@ in {
fast-tests = ''
f = util.BuildFactory()
f.addStep(grab_repo)
- addShell(f,name="centos7-eval",env=env,
+ addShell(f,name="deploy-eval-centos7",env=env,
command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"])
- addShell(f,name="wolf-eval",env=env,
+ addShell(f,name="deploy-eval-wolf",env=env,
command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"])
- addShell(f,name="eval-cross-check",env=env,
+ addShell(f,name="deploy-eval-cross-check",env=env,
command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"])
- addShell(f,name="instaniate-test-all-modules",env=env,
+ addShell(f,name="instantiate-test-all-modules",env=env,
command=nixshell + \
["touch retiolum.rsa_key.priv; \
nix-instantiate --eval -A \
@@ -110,7 +110,7 @@ in {
addShell(s, name="infest-cac-centos7",env=env,
sigtermTime=60, # SIGTERM 1 minute before SIGKILL
- timeout=5400, # 1.5h timeout
+ timeout=7200, # 2h
command=nixshell + ["infest-cac-centos7"])
bu.append(util.BuilderConfig(name="full-tests",
@@ -137,6 +137,7 @@ in {
username = "testslave";
password = "krebspass";
packages = with pkgs;[ git nix ];
+ # all nix commands will need a working nixpkgs installation
extraEnviron = { NIX_PATH="nixpkgs=${toString <nixpkgs>}"; };
};
}
From 1798dbaf47fea7793545be2bc78ac5f1c8e27e18 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 30 Dec 2015 04:56:53 +0100
Subject: [PATCH 47/65] k 5 test: fix endless loop in test
---
krebs/5pkgs/test/infest-cac-centos7/notes | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes
index 3f4fcd859..eee0bfc34 100755
--- a/krebs/5pkgs/test/infest-cac-centos7/notes
+++ b/krebs/5pkgs/test/infest-cac-centos7/notes
@@ -81,9 +81,10 @@ while true;do
if ! wait_login_cac $id;then
echo "unable to boot a working system within time frame, retrying..." >&2
echo "Cleaning up old image,last status: $(cac update;cac getserver $id | jq -r .status)"
- eval "$(clear_defer)"
+ eval "$(clear_defer | sed 's/;exit//')"
else
echo "got a working system" >&2
+ break
fi
done
clear_defer >/dev/null
From c962e8549e968fd15d4f15b4d184e86e1cd7ed04 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 30 Dec 2015 11:29:28 +0100
Subject: [PATCH 48/65] k 3 Reaktor: add channels Option
---
krebs/3modules/Reaktor.nix | 11 +++++++++--
makefu/1systems/wry.nix | 21 ++++++++++++++++-----
2 files changed, 25 insertions(+), 7 deletions(-)
diff --git a/krebs/3modules/Reaktor.nix b/krebs/3modules/Reaktor.nix
index 607eb7cac..92400139c 100644
--- a/krebs/3modules/Reaktor.nix
+++ b/krebs/3modules/Reaktor.nix
@@ -70,12 +70,17 @@ let
REAKTOR_HOST
REAKTOR_PORT
REAKTOR_STATEDIR
- REAKTOR_CHANNELS
debug and nickname can be set separately via the Reaktor api
'';
};
-
+ channels = mkOption {
+ default = [ "#krebs" ];
+ type = types.listOf types.str;
+ description = ''
+ Channels the Reaktor should connect to at startup.
+ '';
+ };
debug = mkOption {
default = false;
description = ''
@@ -112,7 +117,9 @@ let
GIT_SSL_CAINFO = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
REAKTOR_NICKNAME = cfg.nickname;
REAKTOR_DEBUG = (if cfg.debug then "True" else "False");
+ REAKTOR_CHANNELS = lib.concatStringsSep "," cfg.channels;
state_dir = cfg.workdir;
+
} // cfg.extraEnviron;
serviceConfig= {
ExecStartPre = pkgs.writeScript "Reaktor-init" ''
diff --git a/makefu/1systems/wry.nix b/makefu/1systems/wry.nix
index cd2b3f657..3bdf053db 100644
--- a/makefu/1systems/wry.nix
+++ b/makefu/1systems/wry.nix
@@ -18,8 +18,6 @@ in {
../2configs/iodined.nix
- # Reaktor
- ../2configs/Reaktor/simpleExtend.nix
# other nginx
../2configs/nginx/euer.wiki.nix
@@ -29,9 +27,22 @@ in {
# collectd
../2configs/collectd/collectd-base.nix
];
+
krebs.build.host = config.krebs.hosts.wry;
- krebs.Reaktor.enable = true;
+ krebs.Reaktor = {
+ nickname = "Reaktor|bot";
+ channels = [ "#krebs_test" ];
+ enable = true;
+ debug = true;
+ plugins = with pkgs.ReaktorPlugins;[
+ titlebot
+ # stockholm-issue
+ nixos-version
+ shack-correct
+ sed-plugin
+ random-emoji ];
+ };
# bepasty to listen only on the correct interfaces
krebs.bepasty.servers.internal.nginx.listen = [ "${internal-ip}:80" ];
@@ -59,11 +70,11 @@ in {
};
networking = {
- firewall = {
+ firewall = {
allowPing = true;
logRefusedConnections = false;
allowedTCPPorts = [ 53 80 443 ];
- allowedUDPPorts = [ 655 ];
+ allowedUDPPorts = [ 655 53 ];
};
interfaces.enp2s1.ip4 = [{
address = external-ip;
From f0ce9a72a6595f521f68a156aa46b2372a391d38 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 30 Dec 2015 11:52:22 +0100
Subject: [PATCH 49/65] k 5 Reaktor.plugins: fix sed-plugin
---
krebs/5pkgs/Reaktor/plugins.nix | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/krebs/5pkgs/Reaktor/plugins.nix b/krebs/5pkgs/Reaktor/plugins.nix
index 5c7b89f5c..7490be4ca 100644
--- a/krebs/5pkgs/Reaktor/plugins.nix
+++ b/krebs/5pkgs/Reaktor/plugins.nix
@@ -14,6 +14,7 @@ rec {
buildSimpleReaktorPlugin = name: { script
, path ? []
, env ? {}
+ , append_rule ? false # append the rule instead of insert
, pattern ? ""
, ... } @ attrs:
let
@@ -26,7 +27,7 @@ rec {
});
src_file = "${src_dir}/bin/${name}";
config = ''
- public_commands.insert(0,{
+ public_commands.${if append_rule then "append(" else "insert(0," }{
'capname' : "${name}",
'pattern' : ${if pattern == "" then
''indirect_pattern.format("${name}")'' else
@@ -58,9 +59,10 @@ rec {
};
sed-plugin = buildSimpleReaktorPlugin "sed-plugin" {
- path = [ pkgs.gnused ];
+ path = [ pkgs.gnused pkgs.python3 ];
# only support s///gi the plugin needs to see every msg
# TODO: this will eat up the last regex, fix Reaktor to support fallthru
+ append_rule = true;
pattern = "^(?P<args>.*)$$";
script = ./scripts/sed-plugin.py;
};
@@ -105,7 +107,7 @@ rec {
config = ''
def titlebot_cmd(cmd):
from os import environ
- return { 'capname': cmd,
+ return { 'capname': None,
'env': { 'TITLEDB':
environ['state_dir']+'/suggestions.json' },
'pattern': '^\\.' + cmd + '\\s*(?:\\s+(?P<args>.*))?$$',
From f7894c29dbfb8404aeb9f4d387942fd638434a22 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 30 Dec 2015 11:53:48 +0100
Subject: [PATCH 50/65] m 1 wry: update Reaktor config
---
makefu/1systems/wry.nix | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/makefu/1systems/wry.nix b/makefu/1systems/wry.nix
index 3bdf053db..f022311c9 100644
--- a/makefu/1systems/wry.nix
+++ b/makefu/1systems/wry.nix
@@ -32,9 +32,8 @@ in {
krebs.Reaktor = {
nickname = "Reaktor|bot";
- channels = [ "#krebs_test" ];
+ channels = [ "#krebs" "#shackspace" "#binaergewitter" ];
enable = true;
- debug = true;
plugins = with pkgs.ReaktorPlugins;[
titlebot
# stockholm-issue
From ca9e1700ef0deac0b71d4c3e2a6d1ee0a0ccbc42 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 30 Dec 2015 14:47:40 +0100
Subject: [PATCH 51/65] s 1 minimal-deploy: init test
---
shared/1systems/test-minimal-deploy.nix | 13 +++++++++++++
shared/2configs/buildbot-standalone.nix | 11 +++++++++++
2 files changed, 24 insertions(+)
create mode 100644 shared/1systems/test-minimal-deploy.nix
diff --git a/shared/1systems/test-minimal-deploy.nix b/shared/1systems/test-minimal-deploy.nix
new file mode 100644
index 000000000..ddd96f6b5
--- /dev/null
+++ b/shared/1systems/test-minimal-deploy.nix
@@ -0,0 +1,13 @@
+{ config, pkgs, lib, ... }:
+{
+ krebs = {
+ enable = true;
+ build.user = config.krebs.users.shared;
+ build.host = config.krebs.hosts.test-all-krebs-modules;
+ };
+ # just get the system running
+ boot.loader.grub.devices = ["/dev/sda"];
+ fileSystems."/" = {
+ device = "/dev/lol";
+ };
+}
diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix
index c62f8920c..6ffd7fe8a 100644
--- a/shared/2configs/buildbot-standalone.nix
+++ b/shared/2configs/buildbot-standalone.nix
@@ -94,6 +94,17 @@ in {
--argstr current-host-name lol \
--strict --json"])
+ addShell(f,name="instantiate-test-minimal-deploy",env=env,
+ command=nixshell + \
+ ["nix-instantiate --eval -A \
+ users.shared.test-minimal-deploy.system \
+ -I stockholm=. \
+ -I secrets=. '<stockholm>' \
+ --argstr current-date lol \
+ --argstr current-user-name shared \
+ --argstr current-host-name lol \
+ --strict --json"])
+
bu.append(util.BuilderConfig(name="fast-tests",
slavenames=slavenames,
factory=f))
From 5d15b95ac200359392d9a86a68905c2162404904 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 30 Dec 2015 23:37:02 +0100
Subject: [PATCH 52/65] s 2 buildbot: add short tree timeout before trying a
test
---
shared/2configs/buildbot-standalone.nix | 1 +
1 file changed, 1 insertion(+)
diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix
index 6ffd7fe8a..544b54dde 100644
--- a/shared/2configs/buildbot-standalone.nix
+++ b/shared/2configs/buildbot-standalone.nix
@@ -30,6 +30,7 @@ in {
# test the master real quick
sched.append(schedulers.SingleBranchScheduler(
change_filter=util.ChangeFilter(branch="master"),
+ treeStableTimer=10, #only test the latest push
name="fast-master-test",
builderNames=["fast-tests"]))
'';
From f916b84ebd6629d7471f50fbb468161285f5026e Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Sat, 2 Jan 2016 17:31:06 +0100
Subject: [PATCH 53/65] k default: root path for populate
---
krebs/default.nix | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/krebs/default.nix b/krebs/default.nix
index 81ddd3ea6..15d0e8e2e 100644
--- a/krebs/default.nix
+++ b/krebs/default.nix
@@ -50,7 +50,6 @@ let out = {
# Prepare target source via bind-mounting
- (${populate (args // { infesting = true;}) })
(${nixos-install args})
@@ -103,6 +102,7 @@ let out = {
#! /bin/sh
# ${current-date} ${current-user-name}@${current-host-name}
# krebs.nixos-install
+ (${populate (args // { root = "/mnt"; })})
${rootssh target ''
export PATH; PATH=/root/.nix-profile/bin:$PATH
@@ -209,7 +209,7 @@ let out = {
populate =
{ system ? current-host-name
, target ? system
- , infesting ? false
+ , root ? ""
}@args:
let out = ''
#! /bin/sh
@@ -223,7 +223,6 @@ let out = {
'';
- target_prefix=lib.optionalString infesting "/mnt";
config = get-config system;
current-host = config.krebs.hosts.${current-host-name};
@@ -232,7 +231,7 @@ let out = {
methods.dir = config:
let
can-push = config.host.name == current-host.name;
- target-path = target_prefix + config.target-path;
+ target-path = root + config.target-path;
push-method = ''
rsync \
--exclude .git \
@@ -252,7 +251,7 @@ let out = {
throw "No way to push ${dir} from ${current-host.name} to ${target}";
methods.git = config:
- let target-path = target_prefix + config.target-path;
+ let target-path = root + config.target-path;
in rootssh target ''
mkdir -p ${target-path}
cd ${target-path}
From 6fb2bff38742607dda99e24ebb40466839e44a16 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Sat, 2 Jan 2016 21:22:00 +0100
Subject: [PATCH 54/65] ma 1 filepimp: add missing kernel modules
pata_atiixp is required for booting sata
---
makefu/1systems/filepimp.nix | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/makefu/1systems/filepimp.nix b/makefu/1systems/filepimp.nix
index 66ea2ce90..1e9ee5031 100644
--- a/makefu/1systems/filepimp.nix
+++ b/makefu/1systems/filepimp.nix
@@ -17,15 +17,15 @@
loader.grub.device = "/dev/sda";
initrd.availableKernelModules = [
- "usb_storage"
"ahci"
- "xhci_hcd"
- "ata_piix"
- "uhci_hcd"
+ "ohci_pci"
"ehci_pci"
+ "pata_atiixp"
+ "usb_storage"
+ "usbhid"
];
- kernelModules = [ ];
+ kernelModules = [ "kvm-amd" ];
extraModulePackages = [ ];
};
From f0e802d2593ebe7101968deb3593f1c120f552fd Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Sat, 2 Jan 2016 21:36:51 +0100
Subject: [PATCH 55/65] k 5 test/infest-cac-centos7: add timeouts, error
handling
---
krebs/5pkgs/test/infest-cac-centos7/notes | 18 +++++++++++++-----
1 file changed, 13 insertions(+), 5 deletions(-)
diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes
index eee0bfc34..6bfb6906e 100755
--- a/krebs/5pkgs/test/infest-cac-centos7/notes
+++ b/krebs/5pkgs/test/infest-cac-centos7/notes
@@ -55,9 +55,16 @@ old_trapstr=$(clear_defer)
while true;do
# Template 26: CentOS7
# TODO: use cac templates to determine the real Centos7 template in case it changes
- name=$( cac build cpu=1 ram=512 storage=10 os=26 2>&1\
- | jq -r .servername)
- id=servername:$name
+ out=$(cac build cpu=1 ram=512 storage=10 os=26 2>&1)
+ if name=$(echo "$out" | jq -r .servername);then
+ id=servername:$name
+ echo "got a working machine, id=$id"
+ else
+ echo "Unable to build a virtual machine, retrying in 15 seconds" >&2
+ echo "Output of build program: $out" >&2
+ sleep 15
+ continue
+ fi
clear_defer >/dev/null
defer "cac delete $id"
@@ -65,8 +72,8 @@ while true;do
# TODO: timeout?
wait_login_cac(){
- # we wait for 15 minutes
- for t in `seq 90`;do
+ # we wait for 30 minutes
+ for t in `seq 180`;do
# now we have a working cac server
if cac ssh $1 -o ConnectTimeout=10 \
cat /etc/redhat-release | \
@@ -82,6 +89,7 @@ while true;do
echo "unable to boot a working system within time frame, retrying..." >&2
echo "Cleaning up old image,last status: $(cac update;cac getserver $id | jq -r .status)"
eval "$(clear_defer | sed 's/;exit//')"
+ sleep 15
else
echo "got a working system" >&2
break
From 98848a9fffc8f4a2f456770654648f04bf92d5e2 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Sun, 3 Jan 2016 06:07:35 +0100
Subject: [PATCH 56/65] ma 1 omo: actually build the host
---
makefu/1systems/omo.nix | 48 ++++++++++++++++++++++++++++++++++-------
1 file changed, 40 insertions(+), 8 deletions(-)
diff --git a/makefu/1systems/omo.nix b/makefu/1systems/omo.nix
index 6ae79398a..08923d1c2 100644
--- a/makefu/1systems/omo.nix
+++ b/makefu/1systems/omo.nix
@@ -6,32 +6,64 @@
{
imports =
- [ # Include the results of the hardware scan.
+ [
+ # TODO: unlock home partition via ssh
../2configs/fs/single-partition-ext4.nix
../2configs/tinc-basic-retiolum.nix
+ ../2configs/zsh-user.nix
../2configs/exim-retiolum.nix
+ ../2configs/smart-monitor.nix
];
krebs.build.host = config.krebs.hosts.omo;
+ services.smartd.devices = [
+ { device = "/dev/sda"; }
+ { device = "/dev/sdb"; }
+ { device = "/dev/sdc"; }
+ { device = "/dev/sdd"; }
+ { device = "/dev/sde"; }
+ ];
# AMD E350
+ fileSystems."/home" = {
+ device = "/dev/mapper/home";
+ fsType = "ext4";
+ };
+ powerManagement.powerUpCommands = ''
+ for i in a b c d e f g h i;do
+ ${pkgs.hdparm}/sbin/hdparm -S 100 /dev/sd$i
+ ${pkgs.hdparm}/sbin/hdparm -B 127 /dev/sd$i
+ ${pkgs.hdparm}/sbin/hdparm -y /dev/sd$i
+ '';
boot = {
- loader.grub.device = "/dev/sda";
+ initrd.luks = {
+ devices = [
+ { name = "home";
+ device = "/dev/disk/by-uuid/85bff22e-dcbb-4246-b030-faf6c1782995";
+ keyFileSize = 4096;
+ keyFile = "/dev/disk/by-id/usb-Verbatim_STORE_N_GO_070B3CEE0B223954-0:0"; }
+ ];
+ };
+ loader.grub.device = "/dev/disk/by-id/ata-INTEL_SSDSA2M080G2GC_CVPO003402PB080BGN";
initrd.availableKernelModules = [
- "usb_storage"
"ahci"
- "xhci_hcd"
- "ata_piix"
- "uhci_hcd"
+ "ohci_pci"
"ehci_pci"
+ "pata_atiixp"
+ "firewire_ohci"
+ "usb_storage"
+ "usbhid"
];
- kernelModules = [ ];
+ kernelModules = [ "kvm-amd" ];
extraModulePackages = [ ];
};
+ networking.firewall.allowedUDPPorts = [ 655 ];
hardware.enableAllFirmware = true;
hardware.cpu.amd.updateMicrocode = true;
- networking.firewall.allowPing = true;
+ #zramSwap.enable = true;
+ zramSwap.numDevices = 2;
+
}
From 757953e551d157b42c06f50e6592cbb3ee64747e Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Sun, 3 Jan 2016 06:08:01 +0100
Subject: [PATCH 57/65] ma 1 filepimp: prepare raid
---
makefu/1systems/filepimp.nix | 15 +++++++++++----
1 file changed, 11 insertions(+), 4 deletions(-)
diff --git a/makefu/1systems/filepimp.nix b/makefu/1systems/filepimp.nix
index 1e9ee5031..2d008cee6 100644
--- a/makefu/1systems/filepimp.nix
+++ b/makefu/1systems/filepimp.nix
@@ -9,12 +9,19 @@
[ # Include the results of the hardware scan.
../2configs/fs/single-partition-ext4.nix
../2configs/tinc-basic-retiolum.nix
+ ../2configs/smart-monitor.nix
];
krebs.build.host = config.krebs.hosts.filepimp;
-
+ services.smartd.devices = [
+ { device = "/dev/sda"; }
+ { device = "/dev/sdb"; }
+ { device = "/dev/sdc"; }
+ { device = "/dev/sdd"; }
+ { device = "/dev/sde"; }
+ ];
# AMD N54L
boot = {
- loader.grub.device = "/dev/sda";
+ loader.grub.device = "/dev/sde";
initrd.availableKernelModules = [
"ahci"
@@ -28,9 +35,9 @@
kernelModules = [ "kvm-amd" ];
extraModulePackages = [ ];
};
-
hardware.enableAllFirmware = true;
hardware.cpu.amd.updateMicrocode = true;
- networking.firewall.allowPing = true;
+ zramSwap.enable = true;
+ zramSwap.numDevices = 2;
}
From e67393f792d885256456341eee1b9ed21403c01f Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Sun, 3 Jan 2016 06:08:36 +0100
Subject: [PATCH 58/65] ma 2 default: bump nixpkgs revision to unstable
---
makefu/2configs/default.nix | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/makefu/2configs/default.nix b/makefu/2configs/default.nix
index a0b49edaf..7593eaff7 100644
--- a/makefu/2configs/default.nix
+++ b/makefu/2configs/default.nix
@@ -23,8 +23,8 @@ with lib;
source = {
git.nixpkgs = {
#url = https://github.com/NixOS/nixpkgs;
- url = mkDefault https://github.com/makefu/nixpkgs;
- rev = mkDefault "3fd2c24685f604edc925f73ed56600b8c66236b3"; # nixos-15.09 + cherry-picking
+ url = mkDefault https://github.com/nixos/nixpkgs;
+ rev = mkDefault "93d8671e2c6d1d25f126ed30e5e6f16764330119"; # unstable @ 2015-01-03, tested on filepimp
target-path = "/var/src/nixpkgs";
};
From 1ba7e916206ee1d40a62c13a65f68da5968182a9 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Sun, 3 Jan 2016 06:09:12 +0100
Subject: [PATCH 59/65] ma 2 smartd: enable exim-retiolum by default
---
makefu/2configs/smart-monitor.nix | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/makefu/2configs/smart-monitor.nix b/makefu/2configs/smart-monitor.nix
index 7086f622b..9b0290a9b 100644
--- a/makefu/2configs/smart-monitor.nix
+++ b/makefu/2configs/smart-monitor.nix
@@ -1,5 +1,6 @@
-{ config, ... }:
+{ config, lib, ... }:
{
+ krebs.exim-retiolum.enable = lib.mkDefault true;
services.smartd = {
enable = true;
notifications = {
@@ -11,7 +12,7 @@
# short daily, long weekly, check on boot
defaults.monitored = "-a -o on -s (S/../.././02|L/../../7/04)";
- devices = [{
+ devices = lib.mkDefault [{
device = "/dev/sda";
}];
};
From 6cb83cd17413be412836041d8235793ff53e66f5 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Sun, 3 Jan 2016 23:07:55 +0100
Subject: [PATCH 60/65] m 1 omo: act as mail client
---
makefu/1systems/omo.nix | 1 +
1 file changed, 1 insertion(+)
diff --git a/makefu/1systems/omo.nix b/makefu/1systems/omo.nix
index 08923d1c2..d7d3dba00 100644
--- a/makefu/1systems/omo.nix
+++ b/makefu/1systems/omo.nix
@@ -13,6 +13,7 @@
../2configs/zsh-user.nix
../2configs/exim-retiolum.nix
../2configs/smart-monitor.nix
+ ../2configs/mail-client.nix
];
krebs.build.host = config.krebs.hosts.omo;
services.smartd.devices = [
From d73c8df6e4246f34e7a98091bc3c7dab9f90fdde Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Tue, 5 Jan 2016 16:07:13 +0100
Subject: [PATCH 61/65] k 5 snapraid: is part of upstream
---
krebs/5pkgs/snapraid/default.nix | 33 ---------------------
makefu/1systems/omo.nix | 49 ++++++++++++++++++--------------
2 files changed, 28 insertions(+), 54 deletions(-)
delete mode 100644 krebs/5pkgs/snapraid/default.nix
diff --git a/krebs/5pkgs/snapraid/default.nix b/krebs/5pkgs/snapraid/default.nix
deleted file mode 100644
index 41db0f284..000000000
--- a/krebs/5pkgs/snapraid/default.nix
+++ /dev/null
@@ -1,33 +0,0 @@
-{stdenv, fetchurl}:
-let
- s = # Generated upstream information
- rec {
- baseName="jq";
- version="1.5";
- name="${baseName}-${version}";
- url=https://github.com/stedolan/jq/releases/download/jq-1.5/jq-1.5.tar.gz;
- sha256="0g29kyz4ykasdcrb0zmbrp2jqs9kv1wz9swx849i2d1ncknbzln4";
- };
- buildInputs = [
- ];
-in
-stdenv.mkDerivation {
- inherit (s) name version;
- inherit buildInputs;
- src = fetchurl {
- inherit (s) url sha256;
- };
-
- # jq is linked to libjq:
- configureFlags = [
- "LDFLAGS=-Wl,-rpath,\\\${libdir}"
- ];
- meta = {
- inherit (s) version;
- description = ''A lightweight and flexible command-line JSON processor'';
- license = stdenv.lib.licenses.mit ;
- maintainers = [stdenv.lib.maintainers.raskin];
- platforms = stdenv.lib.platforms.linux ++ stdenv.lib.platforms.darwin;
- };
-}
-
diff --git a/makefu/1systems/omo.nix b/makefu/1systems/omo.nix
index d7d3dba00..65a25a2a1 100644
--- a/makefu/1systems/omo.nix
+++ b/makefu/1systems/omo.nix
@@ -2,9 +2,18 @@
# your system. Help is available in the configuration.nix(5) man page
# and in the NixOS manual (accessible by running ‘nixos-help’).
-{ config, pkgs, ... }:
-
-{
+{ config, pkgs, lib, ... }:
+let
+ byid = dev: "/dev/disk/by-id/" + dev;
+ keyFile = "/dev/disk/by-id/usb-Verbatim_STORE_N_GO_070B3CEE0B223954-0:0";
+ rootDisk = byid "ata-INTEL_SSDSA2M080G2GC_CVPO003402PB080BGN";
+ homePartition = byid "ata-INTEL_SSDSA2M080G2GC_CVPO003402PB080BGN-part3";
+ cryptDisk0 = byid "ata-ST2000DM001-1CH164_Z240XTT6";
+ cryptDisk1 = byid "ata-TP02000GB_TPW151006050068";
+ cryptDisk2 = byid "ata-WDC_WD20EARS-00MVWB0_WD-WCAZA5548487";
+ # all physical disks
+ allDisks = [ rootDisk cryptDisk0 cryptDisk1 cryptDisk2 ];
+in {
imports =
[
# TODO: unlock home partition via ssh
@@ -16,35 +25,33 @@
../2configs/mail-client.nix
];
krebs.build.host = config.krebs.hosts.omo;
- services.smartd.devices = [
- { device = "/dev/sda"; }
- { device = "/dev/sdb"; }
- { device = "/dev/sdc"; }
- { device = "/dev/sdd"; }
- { device = "/dev/sde"; }
- ];
+ services.smartd.devices = builtins.map (x: { device = x; }) allDisks;
# AMD E350
fileSystems."/home" = {
device = "/dev/mapper/home";
fsType = "ext4";
};
- powerManagement.powerUpCommands = ''
- for i in a b c d e f g h i;do
- ${pkgs.hdparm}/sbin/hdparm -S 100 /dev/sd$i
- ${pkgs.hdparm}/sbin/hdparm -B 127 /dev/sd$i
- ${pkgs.hdparm}/sbin/hdparm -y /dev/sd$i
- '';
+ powerManagement.powerUpCommands = lib.concatStrings (map (disk: ''
+ ${pkgs.hdparm}/sbin/hdparm -S 100 ${disk}
+ ${pkgs.hdparm}/sbin/hdparm -B 127 ${disk}
+ ${pkgs.hdparm}/sbin/hdparm -y ${disk}
+ '') allDisks);
boot = {
initrd.luks = {
- devices = [
- { name = "home";
- device = "/dev/disk/by-uuid/85bff22e-dcbb-4246-b030-faf6c1782995";
+ devices = let
+ usbkey = name: device: {
+ inherit name device keyFile;
keyFileSize = 4096;
- keyFile = "/dev/disk/by-id/usb-Verbatim_STORE_N_GO_070B3CEE0B223954-0:0"; }
+ };
+ in [
+ (usbkey "home" homePartition)
+ (usbkey "crypt0" cryptDisk0)
+ (usbkey "crypt1" cryptDisk1)
+ (usbkey "crypt2" cryptDisk2)
];
};
- loader.grub.device = "/dev/disk/by-id/ata-INTEL_SSDSA2M080G2GC_CVPO003402PB080BGN";
+ loader.grub.device = rootDisk;
initrd.availableKernelModules = [
"ahci"
From 719b8fb7a8b9b4992200c222b37bd9a6744c25ec Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Tue, 5 Jan 2016 16:21:01 +0100
Subject: [PATCH 62/65] ma 3 snapraid: init, configuration for omo
---
makefu/1systems/omo.nix | 29 ++++++--
makefu/3modules/default.nix | 1 +
makefu/3modules/snapraid.nix | 125 +++++++++++++++++++++++++++++++++++
3 files changed, 150 insertions(+), 5 deletions(-)
create mode 100644 makefu/3modules/snapraid.nix
diff --git a/makefu/1systems/omo.nix b/makefu/1systems/omo.nix
index 65a25a2a1..e19205a95 100644
--- a/makefu/1systems/omo.nix
+++ b/makefu/1systems/omo.nix
@@ -8,6 +8,10 @@ let
keyFile = "/dev/disk/by-id/usb-Verbatim_STORE_N_GO_070B3CEE0B223954-0:0";
rootDisk = byid "ata-INTEL_SSDSA2M080G2GC_CVPO003402PB080BGN";
homePartition = byid "ata-INTEL_SSDSA2M080G2GC_CVPO003402PB080BGN-part3";
+ # cryptsetup luksFormat $dev --cipher aes-xts-plain64 -s 512 -h sha512
+ # cryptsetup luksAddKey $dev tmpkey
+ # cryptsetup luksOpen $dev crypt0
+ # mkfs.xfs /dev/mapper/crypt0 -L crypt0
cryptDisk0 = byid "ata-ST2000DM001-1CH164_Z240XTT6";
cryptDisk1 = byid "ata-TP02000GB_TPW151006050068";
cryptDisk2 = byid "ata-WDC_WD20EARS-00MVWB0_WD-WCAZA5548487";
@@ -23,15 +27,30 @@ in {
../2configs/exim-retiolum.nix
../2configs/smart-monitor.nix
../2configs/mail-client.nix
+ ../3modules
];
krebs.build.host = config.krebs.hosts.omo;
services.smartd.devices = builtins.map (x: { device = x; }) allDisks;
-
- # AMD E350
- fileSystems."/home" = {
- device = "/dev/mapper/home";
- fsType = "ext4";
+ makefu.snapraid = let
+ toMapper = id: "/media/crypt${builtins.toString id}";
+ in {
+ enable = true;
+ disks = map toMapper [ 0 1 ];
+ parity = toMapper 2;
};
+ # AMD E350
+ fileSystems = let
+ cryptMount = name:
+ { "/media/${name}" = { device = "/dev/mapper/${name}"; fsType = "xfs"; };};
+ in {
+ "/home" = {
+ device = "/dev/mapper/home";
+ fsType = "ext4";
+ };
+ } // cryptMount "crypt0"
+ // cryptMount "crypt1"
+ // cryptMount "crypt2";
+
powerManagement.powerUpCommands = lib.concatStrings (map (disk: ''
${pkgs.hdparm}/sbin/hdparm -S 100 ${disk}
${pkgs.hdparm}/sbin/hdparm -B 127 ${disk}
diff --git a/makefu/3modules/default.nix b/makefu/3modules/default.nix
index a8a1f69d0..218c9138e 100644
--- a/makefu/3modules/default.nix
+++ b/makefu/3modules/default.nix
@@ -2,6 +2,7 @@ _:
{
imports = [
+ ./snapraid.nix
];
}
diff --git a/makefu/3modules/snapraid.nix b/makefu/3modules/snapraid.nix
new file mode 100644
index 000000000..fbdf50219
--- /dev/null
+++ b/makefu/3modules/snapraid.nix
@@ -0,0 +1,125 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+ # returns dirname without / , used as disk name
+ dname = dir: replaceChars ["/"] [""] (head (reverseList (splitString "/" dir)));
+ snapraid-conf = ''
+ # Disks
+ ${concatMapStringsSep "\n" (d: "disk ${dname d} ${d}") cfg.disks}
+ # Parity
+ ${optionalString (cfg.parity != "") "parity ${cfg.parity}/snapraid.parity"}
+
+ # content on Disks
+ ${optionalString cfg.contentOnDisks
+ concatMapStringsSep "\n" (d: "content ${d}/snapraid.content") cfg.disks}
+
+ # content on Parity
+ ${optionalString (cfg.contentOnParity && cfg.parity != "")
+ "content ${cfg.parity}/snapraid.content"}
+ # Default content file
+ content ${cfg.defaultContentFile}
+
+ # Extra Configuration
+ ${cfg.extraConfig}
+ '';
+ cfg = config.makefu.snapraid;
+
+ out = {
+ options.makefu.snapraid = api;
+ config = mkIf cfg.enable imp;
+ };
+
+ api = {
+ enable = mkEnableOption "snapraid";
+
+ timerConfig = mkOption {
+ type = types.unspecified;
+ description = ''
+ Start snapraid service
+ '';
+ default = {
+ OnCalendar = "daily";
+ };
+ };
+ disks = mkOption {
+ type = with types;listOf str;
+ description = ''
+ Disks to protect. Each disk is a path to the mounted directory of the
+ disk.
+ '';
+ };
+ parity = mkOption {
+ type = types.str;
+ description = ''
+ Folder to store parity file.
+ Set to empty string if you want to configure the parity yourself in
+ extraConfig.
+
+ All extra parity files (2,3,z, etc...) should be configured via
+ extraConfig.
+ '';
+ };
+ contentOnDisks = mkOption {
+ type = types.bool;
+ default = true;
+ description = ''
+ Store Content file on each Disk to protect.
+ Set this to false if you do not want this behavior to apply.
+ '';
+ };
+ contentOnParity = mkOption {
+ type = types.bool;
+ default = true;
+ description = ''
+ Store Content file on parity Disk.
+ Set this to false if you do not want this behavior to apply.
+ '';
+ };
+ defaultContentFile = mkOption {
+ type = types.str;
+ default = "/var/cache/snapraid.content";
+ description = ''
+ Path to default content file
+ Set to empty string if this content file should be written.
+ '';
+ };
+ extraConfig = mkOption {
+ type = types.string;
+ default = "";
+ description = ''
+ Extra configuration to be appended to the snapraid conf file.
+ You can configure extra Parity files as well as extra content files.
+ See `man snapraid` for additional configuration
+ '';
+ };
+ };
+
+ imp = {
+ environment.systemPackages = [
+ # for scrubbing,fixing
+ pkgs.snapraid
+ ];
+ environment.etc."snapraid.conf".text = snapraid-conf;
+ systemd.timers.snapraid-sync = {
+ description = "snapraid sync timer";
+ wantedBy = [ "timers.target" ];
+ timerConfig = cfg.timerConfig;
+ };
+ systemd.services.snapraid-sync = {
+ description = "Snapraid sync service";
+ after = [ "network.target" "local-fs.target" ];
+
+ serviceConfig = {
+ Type = "simple";
+ ExecStartPre = pkgs.writeScript "Snapraid-sync-init" ''
+ #! /bin/sh
+ ${optionalString (cfg.defaultContentFile != "")
+ "mkdir -p $(dirname ${cfg.defaultContentFile})"}
+ '';
+ ExecStart = "${pkgs.snapraid}/bin/snapraid sync";
+ };
+ };
+ };
+in out
From 1fda893916e1cf8c3cecd43fd861c9d36999b280 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Tue, 5 Jan 2016 16:21:23 +0100
Subject: [PATCH 63/65] ma 2 mail-client: put imapfilter,gnupg into the loop
---
makefu/2configs/mail-client.nix | 2 ++
1 file changed, 2 insertions(+)
diff --git a/makefu/2configs/mail-client.nix b/makefu/2configs/mail-client.nix
index a6ae33d2f..bda21e9d0 100644
--- a/makefu/2configs/mail-client.nix
+++ b/makefu/2configs/mail-client.nix
@@ -7,6 +7,8 @@ with lib;
mutt-kz
notmuch
offlineimap
+ imapfilter
+ gnupg
];
}
From ff945f40b1c3fdb69a5016911ab48462e00cf536 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Thu, 7 Jan 2016 08:05:05 +0100
Subject: [PATCH 64/65] s 2 buildbot: up cac timeout to 3h
---
shared/2configs/buildbot-standalone.nix | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix
index 544b54dde..3275189a5 100644
--- a/shared/2configs/buildbot-standalone.nix
+++ b/shared/2configs/buildbot-standalone.nix
@@ -122,7 +122,7 @@ in {
addShell(s, name="infest-cac-centos7",env=env,
sigtermTime=60, # SIGTERM 1 minute before SIGKILL
- timeout=7200, # 2h
+ timeout=10800, # 3h
command=nixshell + ["infest-cac-centos7"])
bu.append(util.BuilderConfig(name="full-tests",
From 49b6fd9c87678893ed47794b116660700994b1bc Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Thu, 7 Jan 2016 17:34:56 +0100
Subject: [PATCH 65/65] ma 1 pnp: be able to build as vm
---
makefu/1systems/pnp.nix | 64 ++++++++++++++++++-----------------------
1 file changed, 28 insertions(+), 36 deletions(-)
diff --git a/makefu/1systems/pnp.nix b/makefu/1systems/pnp.nix
index a1b73c0c9..51c124bbe 100644
--- a/makefu/1systems/pnp.nix
+++ b/makefu/1systems/pnp.nix
@@ -1,59 +1,51 @@
-# Edit this configuration file to define what should be installed on
-# your system. Help is available in the configuration.nix(5) man page
-# and in the NixOS manual (accessible by running ‘nixos-help’).
-
+# Usage:
+# NIX_PATH=secrets=/home/makefu/secrets/wry:nixpkgs=/var/src/nixpkgs nix-build -A users.makefu.pnp.config.system.build.vm
+# result/bin/run-pnp-vm -virtfs local,path=/home/makefu/secrets/pnp,security_model=none,mount_tag=secrets
{ config, pkgs, ... }:
{
imports =
- [ # Include the results of the hardware scan.
- # Base
+ [
../2configs/tinc-basic-retiolum.nix
../2configs/headless.nix
+ ../../krebs/3modules/Reaktor.nix
- # HW/FS
-
- # enables virtio kernel modules in initrd
+ # these will be overwritten by qemu-vm.nix but will be used if the system
+ # is directly deployed
<nixpkgs/nixos/modules/profiles/qemu-guest.nix>
../2configs/fs/vm-single-partition.nix
- # Services
- ../2configs/git/cgit-retiolum.nix
-
- ## Reaktor
- ## \/ are only plugins, must enable Reaktor explicitly
- ../2configs/Reaktor/stockholmLentil.nix
- ../2configs/Reaktor/simpleExtend.nix
- ../2configs/Reaktor/random-emoji.nix
- ../2configs/Reaktor/titlebot.nix
- ../2configs/Reaktor/shack-correct.nix
-
- # ../2configs/graphite-standalone.nix
+ # config.system.build.vm
+ <nixpkgs/nixos/modules/virtualisation/qemu-vm.nix>
];
- krebs.urlwatch.verbose = true;
- krebs.Reaktor.enable = true;
- krebs.Reaktor.debug = true;
- krebs.Reaktor.nickname = "Reaktor|bot";
- krebs.Reaktor.extraEnviron = {
- REAKTOR_CHANNELS = "#krebs,#binaergewitter,#shackspace";
+ virtualisation.graphics = false;
+ # also export secrets, see Usage above
+ fileSystems = pkgs.lib.mkVMOverride {
+ "${builtins.toString <secrets>}" =
+ { device = "secrets";
+ fsType = "9p";
+ options = "trans=virtio,version=9p2000.L,cache=loose";
+ neededForBoot = true;
+ };
+ };
+
+ krebs.Reaktor = {
+ enable = true;
+ debug = true;
+ extraEnviron = {
+ REAKTOR_HOST = "cd.retiolum";
+ };
+ plugins = with pkgs.ReaktorPlugins; [ stockholm-issue nixos-version sed-plugin ];
+ channels = [ "#retiolum" ];
};
krebs.build.host = config.krebs.hosts.pnp;
nixpkgs.config.packageOverrides = pkgs: { tinc = pkgs.tinc_pre; };
-
networking.firewall.allowedTCPPorts = [
- # nginx runs on 80
- 80
- # graphite-web runs on 8080, carbon cache runs on 2003 tcp and udp
- # 8080 2003
-
- # smtp
25
];
- # networking.firewall.allowedUDPPorts = [ 2003 ];
-
}