summaryrefslogtreecommitdiffstats
path: root/krebs/5pkgs
diff options
context:
space:
mode:
Diffstat (limited to 'krebs/5pkgs')
-rw-r--r--krebs/5pkgs/Reaktor/plugins.nix120
-rw-r--r--krebs/5pkgs/Reaktor/scripts/random-emoji.sh6
-rw-r--r--krebs/5pkgs/Reaktor/scripts/random-issue.sh20
-rw-r--r--krebs/5pkgs/Reaktor/scripts/sed-plugin.py53
-rw-r--r--krebs/5pkgs/Reaktor/scripts/shack-correct.sh6
-rw-r--r--krebs/5pkgs/cacpanel/default.nix4
-rw-r--r--krebs/5pkgs/default.nix6
-rw-r--r--krebs/5pkgs/snapraid/default.nix33
-rw-r--r--krebs/5pkgs/test/infest-cac-centos7/default.nix39
-rwxr-xr-xkrebs/5pkgs/test/infest-cac-centos7/notes150
10 files changed, 402 insertions, 35 deletions
diff --git a/krebs/5pkgs/Reaktor/plugins.nix b/krebs/5pkgs/Reaktor/plugins.nix
new file mode 100644
index 000000000..7490be4ca
--- /dev/null
+++ b/krebs/5pkgs/Reaktor/plugins.nix
@@ -0,0 +1,120 @@
+{ stdenv, lib, pkgs, makeWrapper }:
+
+rec {
+ # Begin API
+ buildBaseReaktorPlugin = { name
+ , config # python extra configuration for plugin
+ , phases ? []
+ , ... } @ attrs:
+ stdenv.mkDerivation (attrs // {
+ name = "Reaktor-plugin-" + name;
+ isReaktorPlugin = true;
+ });
+
+ buildSimpleReaktorPlugin = name: { script
+ , path ? []
+ , env ? {}
+ , append_rule ? false # append the rule instead of insert
+ , pattern ? ""
+ , ... } @ attrs:
+ let
+ path_env = { "PATH" = lib.makeSearchPath "bin" (path ++ [ pkgs.coreutils ]); };
+ src_dir = pkgs.substituteAll ( {
+ inherit name;
+ dir = "bin";
+ isExecutable = true;
+ src = script;
+ });
+ src_file = "${src_dir}/bin/${name}";
+ config = ''
+ public_commands.${if append_rule then "append(" else "insert(0," }{
+ 'capname' : "${name}",
+ 'pattern' : ${if pattern == "" then
+ ''indirect_pattern.format("${name}")'' else
+ ''"${pattern}"'' },
+ 'argv' : ["${src_file}"],
+ 'env' : ${builtins.toJSON (path_env // env)} })
+ '';
+ config_file = pkgs.writeText "plugin.py" config;
+ in buildBaseReaktorPlugin (attrs // rec {
+ inherit name config;
+
+ phases = [ "installPhase" ];
+ buildInputs = [ makeWrapper ];
+ installPhase = ''
+ mkdir -p $out/bin $out/etc/Reaktor
+ ln -s ${src_file} $out/bin
+ wrapProgram $out/bin/${name} \
+ --prefix PATH : ${path_env.PATH}
+ ln -s ${config_file} $out/etc/Reaktor/plugin.py
+ '';
+
+ });
+ # End API
+
+ # Begin Plugins
+ random-emoji = buildSimpleReaktorPlugin "emoji" {
+ path = with pkgs; [ gnused gnugrep xmlstarlet curl ];
+ script = ./scripts/random-emoji.sh;
+ };
+
+ sed-plugin = buildSimpleReaktorPlugin "sed-plugin" {
+ path = [ pkgs.gnused pkgs.python3 ];
+ # only support s///gi the plugin needs to see every msg
+ # TODO: this will eat up the last regex, fix Reaktor to support fallthru
+ append_rule = true;
+ pattern = "^(?P<args>.*)$$";
+ script = ./scripts/sed-plugin.py;
+ };
+
+ shack-correct = buildSimpleReaktorPlugin "shack-correct" {
+ path = [ pkgs.gnused ];
+ pattern = "^(?P<args>.*Shack.*)$$";
+ script = ./scripts/shack-correct.sh;
+ };
+
+ nixos-version = buildSimpleReaktorPlugin "nixos-version" {
+ script = pkgs.writeScript "nixos-version" ''
+ #! /bin/sh
+ . /etc/os-release
+ echo "$PRETTY_NAME"
+ '';
+ };
+ stockholm-issue = buildSimpleReaktorPlugin "stockholm-issue" {
+ script = ./scripts/random-issue.sh;
+ path = with pkgs; [ git gnused lentil ];
+ env = { "origin" = "http://cgit.gum/stockholm"; };
+ };
+
+ titlebot =
+ let
+ pypkgs = pkgs.python3Packages;
+ titlebot_cmds = pypkgs.buildPythonPackage {
+ name = "titlebot_cmds";
+ propagatedBuildInputs = with pypkgs; [ setuptools ];
+ src = pkgs.fetchurl {
+ url = "https://github.com/makefu/reaktor-titlebot/archive/2.1.0.tar.gz";
+ sha256 = "0wvf09wmk8b52f9j65qrw81nwrhs9pfhijwrlkzp5l7l2q8cjkp6";
+ };
+ };
+ in buildBaseReaktorPlugin rec {
+ name = "titlebot";
+ phases = [ "installPhase" ];
+ installPhase = ''
+ mkdir -p $out
+ ln -s ${titlebot_cmds}/* $out
+ '';
+ config = ''
+ def titlebot_cmd(cmd):
+ from os import environ
+ return { 'capname': None,
+ 'env': { 'TITLEDB':
+ environ['state_dir']+'/suggestions.json' },
+ 'pattern': '^\\.' + cmd + '\\s*(?:\\s+(?P<args>.*))?$$',
+ 'argv': [ '${titlebot_cmds}/bin/' + cmd ] }
+ for i in ['up','help','list','top','new']:
+ public_commands.insert(0,titlebot_cmd(i))
+ commands.insert(0,titlebot_cmd('clear'))
+ '';
+ };
+}
diff --git a/krebs/5pkgs/Reaktor/scripts/random-emoji.sh b/krebs/5pkgs/Reaktor/scripts/random-emoji.sh
new file mode 100644
index 000000000..386aa68b9
--- /dev/null
+++ b/krebs/5pkgs/Reaktor/scripts/random-emoji.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+curl http://emojicons.com/random -s | \
+ grep data-text | \
+ sed -n 's/.*>\(.*\)<\/textarea>/\1/p' | \
+ head -n 1 | \
+ xmlstarlet unesc
diff --git a/krebs/5pkgs/Reaktor/scripts/random-issue.sh b/krebs/5pkgs/Reaktor/scripts/random-issue.sh
new file mode 100644
index 000000000..5c47c6156
--- /dev/null
+++ b/krebs/5pkgs/Reaktor/scripts/random-issue.sh
@@ -0,0 +1,20 @@
+#! /bin/sh
+set -eu
+# requires env:
+# $state_dir
+# $origin
+
+# in PATH: git,lentil,coreutils
+subdir=`echo "$1" | tr -dc "[:alnum:]"`
+name=`echo "$origin" | tr -dc "[:alnum:]"`
+track="$state_dir/$name-checkout"
+(if test -e "$track" ;then
+ cd "$track"
+ git fetch origin master
+ git reset --hard origin/master
+else
+ git clone "$origin" "$track"
+fi) >&2
+
+cd "$track"
+lentil "${subdir:-.}" -f csv | sed 1d | shuf | head -1
diff --git a/krebs/5pkgs/Reaktor/scripts/sed-plugin.py b/krebs/5pkgs/Reaktor/scripts/sed-plugin.py
new file mode 100644
index 000000000..8103c9585
--- /dev/null
+++ b/krebs/5pkgs/Reaktor/scripts/sed-plugin.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+
+# Usage:
+# _from=krebs state_dir=. python sed-plugin.py 'dick butt'
+# _from=krebs state_dir=. python sed-plugin.py 's/t/l/g'
+## dick bull
+import shelve
+from os import environ
+from os.path import join
+from sys import argv
+d = shelve.open(join(environ['state_dir'],'sed-plugin.shelve'),writeback=True)
+usr = environ['_from']
+import re
+
+def is_regex(line):
+ myre = re.compile(r'^s/((?:\\/|[^/])+)/((?:\\/|[^/])*)/([ig]*)$')
+ return myre.match(line)
+
+line = argv[1]
+m = is_regex(line)
+
+if m:
+ f,t,flagstr = m.groups()
+ fn = f.replace('\/','/')
+ tn = t.replace('\/','/')
+ flags = 0
+ count = 1
+ if flagstr:
+ if 'i' in flagstr:
+ flags = re.IGNORECASE
+ if 'g' in flagstr:
+ count = 0
+ else:
+ flagstr = ''
+ last = d.get(usr,None)
+ if last:
+ #print(re.sub(fn,tn,last,count=count,flags=flags))
+ from subprocess import Popen,PIPE
+ p = Popen(['sed','s/{}/{}/{}'.format(f,t,flagstr)],stdin=PIPE,stdout=PIPE )
+ so,se = p.communicate(bytes("{}\n".format(last),"UTF-8"))
+ if p.returncode:
+ print("something went wrong when trying to process your regex: {}".format(se.decode()))
+ ret = so.decode()
+ print("\x1b[1m{}\x1b[0m meinte: {}".format(usr,ret.strip()))
+ if ret:
+ d[usr] = ret
+
+ else:
+ print("no last message")
+else:
+ d[usr] = line
+
+d.close()
diff --git a/krebs/5pkgs/Reaktor/scripts/shack-correct.sh b/krebs/5pkgs/Reaktor/scripts/shack-correct.sh
new file mode 100644
index 000000000..3b4d04f80
--- /dev/null
+++ b/krebs/5pkgs/Reaktor/scripts/shack-correct.sh
@@ -0,0 +1,6 @@
+#! /bin/sh
+set -eu
+printf "Sie meinten wohl \""
+echo -n $@ | sed 's/Shack/shack/g'
+echo "\""
+echo "${_from}--"
diff --git a/krebs/5pkgs/cacpanel/default.nix b/krebs/5pkgs/cacpanel/default.nix
index 3e3e2e1fc..3df4dffed 100644
--- a/krebs/5pkgs/cacpanel/default.nix
+++ b/krebs/5pkgs/cacpanel/default.nix
@@ -2,11 +2,11 @@
python3Packages.buildPythonPackage rec {
name = "cacpanel-${version}";
- version = "0.2.1";
+ version = "0.2.3";
src = pkgs.fetchurl {
url = "https://pypi.python.org/packages/source/c/cacpanel/cacpanel-${version}.tar.gz";
- sha256 = "1zaazg5r10kgva32zh4fhpw6l6h51ijkwpa322na0kh4x6f6aqj3";
+ sha256 = "1fib7416qqv8yzrj75kxra7ccpz9abqh58b6gkaavws2fa6m3mm8";
};
propagatedBuildInputs = with python3Packages; [
diff --git a/krebs/5pkgs/default.nix b/krebs/5pkgs/default.nix
index 7df7b7d3c..c4b1dafe4 100644
--- a/krebs/5pkgs/default.nix
+++ b/krebs/5pkgs/default.nix
@@ -26,6 +26,8 @@ subdirs // rec {
inherit (subdirs) get jq;
};
+ ReaktorPlugins = pkgs.callPackage ./Reaktor/plugins.nix {};
+
execve = name: { filename, argv, envp ? {}, destination ? "" }:
writeC name { inherit destination; } ''
#include <unistd.h>
@@ -40,6 +42,10 @@ subdirs // rec {
}
'';
+ test = {
+ infest-cac-centos7 = pkgs.callPackage ./test/infest-cac-centos7 {};
+ };
+
execveBin = name: cfg: execve name (cfg // { destination = "/bin/${name}"; });
writeC = name: { destination ? "" }: src: pkgs.runCommand name {} ''
diff --git a/krebs/5pkgs/snapraid/default.nix b/krebs/5pkgs/snapraid/default.nix
deleted file mode 100644
index 41db0f284..000000000
--- a/krebs/5pkgs/snapraid/default.nix
+++ /dev/null
@@ -1,33 +0,0 @@
-{stdenv, fetchurl}:
-let
- s = # Generated upstream information
- rec {
- baseName="jq";
- version="1.5";
- name="${baseName}-${version}";
- url=https://github.com/stedolan/jq/releases/download/jq-1.5/jq-1.5.tar.gz;
- sha256="0g29kyz4ykasdcrb0zmbrp2jqs9kv1wz9swx849i2d1ncknbzln4";
- };
- buildInputs = [
- ];
-in
-stdenv.mkDerivation {
- inherit (s) name version;
- inherit buildInputs;
- src = fetchurl {
- inherit (s) url sha256;
- };
-
- # jq is linked to libjq:
- configureFlags = [
- "LDFLAGS=-Wl,-rpath,\\\${libdir}"
- ];
- meta = {
- inherit (s) version;
- description = ''A lightweight and flexible command-line JSON processor'';
- license = stdenv.lib.licenses.mit ;
- maintainers = [stdenv.lib.maintainers.raskin];
- platforms = stdenv.lib.platforms.linux ++ stdenv.lib.platforms.darwin;
- };
-}
-
diff --git a/krebs/5pkgs/test/infest-cac-centos7/default.nix b/krebs/5pkgs/test/infest-cac-centos7/default.nix
new file mode 100644
index 000000000..7f2e3f231
--- /dev/null
+++ b/krebs/5pkgs/test/infest-cac-centos7/default.nix
@@ -0,0 +1,39 @@
+{ stdenv, coreutils,makeWrapper, cac, cacpanel, gnumake, gnused, jq, openssh, ... }:
+
+stdenv.mkDerivation rec {
+ name = "${shortname}-${version}";
+ shortname = "infest-cac-centos7";
+ version = "0.2.0";
+
+ src = ./notes;
+
+ phases = [
+ "installPhase"
+ ];
+ buildInputs = [ makeWrapper ];
+
+ path = stdenv.lib.makeSearchPath "bin" [
+ coreutils
+ cac
+ cacpanel
+ gnumake
+ gnused
+ jq
+ openssh
+ ];
+
+ installPhase =
+ ''
+ mkdir -p $out/bin
+ cp ${src} $out/bin/${shortname}
+ chmod +x $out/bin/${shortname}
+ wrapProgram $out/bin/${shortname} \
+ --prefix PATH : ${path}
+ '';
+ meta = with stdenv.lib; {
+ homepage = http://krebsco.de;
+ description = "Krebs CI Scripts";
+ license = licenses.wtfpl;
+ maintainers = [ maintainers.makefu ];
+ };
+}
diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes
new file mode 100755
index 000000000..6bfb6906e
--- /dev/null
+++ b/krebs/5pkgs/test/infest-cac-centos7/notes
@@ -0,0 +1,150 @@
+#! /bin/sh
+
+# nix-shell -p gnumake jq openssh cac cacpanel
+set -eufx
+
+# 2 secrets are required:
+
+krebs_cred=${krebs_cred-./cac.json}
+retiolum_key=${retiolum_key-./retiolum.rsa_key.priv}
+
+clear_defer(){
+ echo "${trapstr:-exit}"
+ trap - INT TERM EXIT KILL
+}
+defer(){
+ if test -z "${debug:-}"; then
+ trapstr="$1;${trapstr:-exit}"
+ trap "$trapstr" INT TERM EXIT KILL
+ fi
+}
+
+# Sanity
+if test ! -r "$krebs_cred";then
+ echo "\$krebs_cred=$krebs_cred must be readable"; exit 1
+fi
+if test ! -r "$retiolum_key";then
+ echo "\$retiolum_key=$retiolum_key must be readable"; exit 1
+fi
+
+krebs_secrets=$(mktemp -d)
+sec_file=$krebs_secrets/cac_config
+krebs_ssh=$krebs_secrets/tempssh
+export cac_resources_cache=$krebs_secrets/res_cache.json
+export cac_servers_cache=$krebs_secrets/servers_cache.json
+export cac_tasks_cache=$krebs_secrets/tasks_cache.json
+export cac_templates_cache=$krebs_secrets/templates_cache.json
+# we need to receive this key from buildmaster to speed up tinc bootstrap
+defer "trap - INT TERM EXIT"
+defer "rm -r $krebs_secrets"
+
+cat > $sec_file <<EOF
+cac_login="$(jq -r .email $krebs_cred)"
+cac_key="$(cac-cli --config $krebs_cred panel settings | jq -r .apicode)"
+EOF
+
+export cac_secrets=$sec_file
+cac-cli --config $krebs_cred panel add-api-ip
+
+# test login:
+cac update
+cac servers
+
+# preserve old trap
+old_trapstr=$(clear_defer)
+while true;do
+ # Template 26: CentOS7
+ # TODO: use cac templates to determine the real Centos7 template in case it changes
+ out=$(cac build cpu=1 ram=512 storage=10 os=26 2>&1)
+ if name=$(echo "$out" | jq -r .servername);then
+ id=servername:$name
+ echo "got a working machine, id=$id"
+ else
+ echo "Unable to build a virtual machine, retrying in 15 seconds" >&2
+ echo "Output of build program: $out" >&2
+ sleep 15
+ continue
+ fi
+
+ clear_defer >/dev/null
+ defer "cac delete $id"
+
+ # TODO: timeout?
+
+ wait_login_cac(){
+ # we wait for 30 minutes
+ for t in `seq 180`;do
+ # now we have a working cac server
+ if cac ssh $1 -o ConnectTimeout=10 \
+ cat /etc/redhat-release | \
+ grep CentOS ;then
+ return 0
+ fi
+ sleep 10
+ done
+ return 1
+ }
+ # die on timeout
+ if ! wait_login_cac $id;then
+ echo "unable to boot a working system within time frame, retrying..." >&2
+ echo "Cleaning up old image,last status: $(cac update;cac getserver $id | jq -r .status)"
+ eval "$(clear_defer | sed 's/;exit//')"
+ sleep 15
+ else
+ echo "got a working system" >&2
+ break
+ fi
+done
+clear_defer >/dev/null
+defer "cac delete $id;$old_trapstr"
+
+mkdir -p shared/2configs/temp
+cac generatenetworking $id > \
+ shared/2configs/temp/networking.nix
+# new temporary ssh key we will use to log in after infest
+ssh-keygen -f $krebs_ssh -N ""
+cp $retiolum_key $krebs_secrets/retiolum.rsa_key.priv
+# we override the directories for secrets and stockholm
+# additionally we set the ssh key we generated
+ip=$(cac getserver $id | jq -r .ip)
+
+cat > shared/2configs/temp/dirs.nix <<EOF
+_: {
+ krebs.build.source.dir = {
+ secrets.path = "$krebs_secrets";
+ stockholm.path = "$(pwd)";
+ };
+ users.extraUsers.root.openssh.authorizedKeys.keys = [
+ "$(cat ${krebs_ssh}.pub)"
+ ];
+ krebs.build.target = "$ip";
+}
+EOF
+
+LOGNAME=shared make eval get=krebs.infest \
+ target=derp system=test-centos7 filter=json \
+ | sed -e "s#^ssh.*<<#cac ssh $id<<#" \
+ -e "/^rsync/a -e 'cac ssh $id' \\\\" \
+ -e "s#root.derp:#:#" > $krebs_secrets/infest
+sh -x $krebs_secrets/infest
+
+# TODO: generate secrets directory $krebs_secrets for nix import
+cac powerop $id reset
+
+wait_login(){
+ # timeout
+ for t in `seq 90`;do
+ # now we have a working cac server
+ if ssh -o StrictHostKeyChecking=no \
+ -o UserKnownHostsFile=/dev/null \
+ -i $krebs_ssh \
+ -o ConnectTimeout=10 \
+ -o BatchMode=yes \
+ root@$1 nixos-version ;then
+ return 0
+ fi
+ sleep 10
+ done
+ return 1
+}
+wait_login $ip
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-02 09:45:50 +0000