diff options
62 files changed, 772 insertions, 730 deletions
diff --git a/.rsync-filter b/.rsync-filter new file mode 100644 index 000000000..d7657cd00 --- /dev/null +++ b/.rsync-filter @@ -0,0 +1,2 @@ +- /.git +- /.graveyard @@ -1,53 +1,51 @@ -# -# usage: -# make infest system=foo [target=bar] -# make [deploy] system=foo [target=bar] -# make [deploy] systems='foo bar' -# make eval get=users.tv.wu.config.time.timeZone [filter=json] -# - .ONESHELL: .SHELLFLAGS := -eufc -ifdef systems -$(systems): - @ - unset target - parallel \ - --line-buffer \ - -j0 \ - --no-notice \ - --tagstring {} \ - -q make -s systems= system={} ::: $(systems) -else ifdef system -.PHONY: deploy infest -deploy infest:;@ - export get=krebs.$@ - export filter=json - make -s eval | sh +ifndef system +$(error unbound variable: system) +endif + +export target_host ?= $(system) +export target_user ?= root +export target_path ?= /var/src + +# usage: make deploy system=foo [target_host=bar] +.PHONY: deploy +deploy: populate ;@set -x + ssh "$$target_user@$$target_host" nixos-rebuild switch -I "$$target_path" +# usage: make populate system=foo [target_host=bar] +.PHONY: populate +populate:;@ + result=$$(make -s eval get=config.krebs.build.populate filter=json) + echo "$$result" | sh + +# usage: make eval system=foo get=config.krebs.build [LOGNAME=tv] [filter=json] .PHONY: eval -eval: - @ +eval:;@ ifeq ($(filter),json) extraArgs='--json --strict' - filter() { jq -r .; } + filter() { echo "$$1" | jq -r .; } else - filter() { cat; } + filter() { echo "$$1"; } endif result=$$(nix-instantiate \ $${extraArgs-} \ + --show-trace \ + --readonly-mode \ --eval \ -A "$$get" \ - -I stockholm="$$PWD" \ - '<stockholm>' \ - --argstr current-date "$$(date -Is)" \ - --argstr current-host-name "$$HOSTNAME" \ - --argstr current-user-name "$$LOGNAME" \ - $${system+--argstr system "$$system"} \ - $${target+--argstr target "$$target"}) - echo "$$result" | filter + --arg configuration "./$$LOGNAME/1systems/$$system.nix") + filter "$$result" -else -$(error unbound variable: system[s]) -endif +## usage: make install system=foo target= +#.PHONY: install +#install: ssh = ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null +#install:;@set -x +# $(ssh) "$$target_user@$$target_host" \ +# env target_path=/var/src \ +# sh -s prepare < krebs/4lib/infest/prepare.sh +# make -s populate target_path=/mnt"$$target_path" +# $(ssh) "$$target_user@$$target_host" \ +# env NIXOS_CONFIG=/var/src/nixos-config \ +# nixos-install diff --git a/default.nix b/default.nix index 1637aa464..278f1d14d 100644 --- a/default.nix +++ b/default.nix @@ -1,6 +1,15 @@ -{ current-date ? abort "current-date not defined" -, current-host-name ? abort "current-host-name not defined" -, current-user-name ? builtins.getEnv "LOGNAME" +{ configuration ? import (nixpkgs-path + "/nixos/lib/from-env.nix") "NIXOS_CONFIG" <nixos-config> +, system ? builtins.currentSystem +, current-host-name ? + let v = builtins.getEnv "HOSTNAME"; in + if v != "" then v else builtins.readFile /proc/sys/kernel/hostname +, current-user-name ? + let v = builtins.getEnv "LOGNAME"; in + if v != "" then v else abort "undefined variable: LOGNAME" +, nixpkgs-path ? + if (builtins.tryEval <nixpkgs/krebs>).success + then <upstream-nixpkgs> + else <nixpkgs> , StrictHostKeyChecking ? "yes" }@args: @@ -8,26 +17,26 @@ let stockholm = { inherit krebs; inherit users; inherit lib; - inherit pkgs; + inherit config options pkgs; + system = config.system.build.toplevel; }; krebs = import ./krebs (args // { inherit lib stockholm; }); lib = let - nlib = import <nixpkgs/lib>; + nlib = import (slib.npath "lib"); klib = import (slib.kpath "4lib") { lib = nlib; }; slib = rec { - stockholm-path = ./.; - nspath = ns: p: stockholm-path + "/${ns}/${p}"; - kpath = nspath "krebs"; - upath = nspath current-user-name; + npath = p: nixpkgs-path + "/${p}"; + kpath = p: ./. + "/krebs/${p}"; + upath = p: ./. + "/${current-user-name}/${p}"; }; ulib = let p = slib.upath "4lib"; in nlib.optionalAttrs (klib.dir.has-default-nix p) (import p { lib = nlib // klib; }); in nlib // klib // slib // ulib // builtins; - inherit (eval {}) pkgs; + inherit (eval configuration) config options pkgs; base-module = { config, ... }: { imports = builtins.filter lib.dir.has-default-nix (lib.concatLists [ @@ -45,7 +54,8 @@ let stockholm = { in kpkgs // upkgs; }; - eval = config: import <nixpkgs/nixos/lib/eval-config.nix> { + eval = config: import (lib.npath "nixos/lib/eval-config.nix") { + inherit system; specialArgs = { inherit lib; }; diff --git a/krebs/3modules/backup.nix b/krebs/3modules/backup.nix index 01bb16a2b..a1f335905 100644 --- a/krebs/3modules/backup.nix +++ b/krebs/3modules/backup.nix @@ -28,9 +28,17 @@ let type = types.krebs.file-location; }; startAt = mkOption { - type = types.str; + default = "hourly"; + type = types.str; # TODO systemd.time(7)'s calendar event }; snapshots = mkOption { + default = { + hourly = { format = "%Y-%m-%dT%H"; retain = 4; }; + daily = { format = "%Y-%m-%d"; retain = 7; }; + weekly = { format = "%YW%W"; retain = 4; }; + monthly = { format = "%Y-%m"; retain = 12; }; + yearly = { format = "%Y"; }; + }; type = types.attrsOf (types.submodule { options = { format = mkOption { @@ -284,3 +292,15 @@ let }; in out +# TODO ionice +# TODO mail on failed push, pull +# TODO mail on missing push +# TODO don't cancel plans on activation +# also, don't hang while deploying at: +# starting the following units: backup.wu-home-xu.push.service, backup.wu-home-xu.push.timer +# TODO make sure /bku is properly mounted +# TODO make sure that secure hosts cannot backup to insecure ones +# TODO optionally only backup when src and dst are near enough :) +# TODO try using btrfs for snapshots (configurable) +# TODO warn if partial snapshots are found +# TODO warn if unknown stuff is found in dst path diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix index 7f004cd81..00142acdd 100644 --- a/krebs/3modules/build.nix +++ b/krebs/3modules/build.nix @@ -28,48 +28,159 @@ let type = types.user; }; - options.krebs.build.source.dir = mkOption { - type = let - default-host = config.krebs.current.host; - in types.attrsOf (types.submodule ({ config, ... }: { + options.krebs.build.source = let + raw = types.either types.str types.path; + url = types.submodule { options = { - host = mkOption { - type = types.host; - default = default-host; - }; - path = mkOption { - type = types.str; - }; - target-path = mkOption { - type = types.str; - default = "/root/${config._module.args.name}"; - }; url = mkOption { type = types.str; - default = "file://${config.host.name}${config.path}"; - }; - }; - })); - default = {}; - }; - - options.krebs.build.source.git = mkOption { - type = with types; attrsOf (submodule ({ config, ... }: { - options = { - url = mkOption { - type = types.str; # TODO must be shell safe }; rev = mkOption { type = types.str; }; - target-path = mkOption { + dev = mkOption { type = types.str; - default = "/root/${config._module.args.name}"; }; }; - })); + }; + in mkOption { + type = types.attrsOf (types.either types.str url); + apply = let f = mapAttrs (_: value: { + string = value; + path = toString value; + set = f value; + }.${typeOf value}); in f; default = {}; }; + + options.krebs.build.populate = mkOption { + type = types.str; + default = let + source = config.krebs.build.source; + target-user = maybeEnv "target_user" "root"; + target-host = maybeEnv "target_host" config.krebs.build.host.name; + target-path = maybeEnv "target_path" "/var/src"; + out = '' + #! /bin/sh + set -eu + + verbose() { + printf '+%s\n' "$(printf ' %q' "$@")" >&2 + "$@" + } + + echo ${shell.escape git-script} \ + | ssh ${shell.escape "${target-user}@${target-host}"} -T + + unset tmpdir + trap ' + rm "$tmpdir"/* + rmdir "$tmpdir" + trap - EXIT INT QUIT + ' EXIT INT QUIT + tmpdir=$(mktemp -dt stockholm.XXXXXXXX) + chmod 0755 "$tmpdir" + + ${concatStringsSep "\n" + (mapAttrsToList + (name: spec: let dst = removePrefix "symlink:" (get-url spec); in + "verbose ln -s ${shell.escape dst} $tmpdir/${shell.escape name}") + symlink-specs)} + + verbose proot \ + -b $tmpdir:${shell.escape target-path} \ + ${concatStringsSep " \\\n " + (mapAttrsToList + (name: spec: + "-b ${shell.escape "${get-url spec}:${target-path}/${name}"}") + file-specs)} \ + rsync \ + -f ${shell.escape "P /*"} \ + ${concatMapStringsSep " \\\n " + (name: "-f ${shell.escape "R /${name}"}") + (attrNames file-specs)} \ + --delete \ + -vFrlptD \ + ${shell.escape target-path}/ \ + ${shell.escape "${target-user}@${target-host}:${target-path}"} + ''; + + get-schema = uri: + if substring 0 1 uri == "/" + then "file" + else head (splitString ":" uri); + + has-schema = schema: uri: get-schema uri == schema; + + get-url = spec: { + string = spec; + path = toString spec; + set = get-url spec.url; + }.${typeOf spec}; + + git-specs = + filterAttrs (_: spec: has-schema "https" (get-url spec)) source // + filterAttrs (_: spec: has-schema "http" (get-url spec)) source // + filterAttrs (_: spec: has-schema "git" (get-url spec)) source; + + file-specs = + filterAttrs (_: spec: has-schema "file" (get-url spec)) source; + + symlink-specs = + filterAttrs (_: spec: has-schema "symlink" (get-url spec)) source; + + git-script = '' + #! /bin/sh + set -efu + + verbose() { + printf '+%s\n' "$(printf ' %q' "$@")" >&2 + "$@" + } + + fetch_git() {( + dst_dir=$1 + src_url=$2 + src_ref=$3 + + if ! test -e "$dst_dir"; then + git clone "$src_url" "$dst_dir" + fi + + cd "$dst_dir" + + if ! url=$(git config remote.origin.url); then + git remote add origin "$src_url" + elif test "$url" != "$src_url"; then + git remote set-url origin "$src_url" + fi + + # TODO resolve src_ref to commit hash + hash=$src_ref + + if ! test "$(git log --format=%H -1)" = "$hash"; then + git fetch origin + git checkout "$hash" -- "$dst_dir" + git checkout "$hash" + fi + + git clean -dxf + )} + + ${concatStringsSep "\n" + (mapAttrsToList + (name: spec: toString (map shell.escape [ + "verbose" + "fetch_git" + "${target-path}/${name}" + spec.url + spec.rev + ])) + git-specs)} + ''; + in out; + }; + }; in out diff --git a/krebs/3modules/git.nix b/krebs/3modules/git.nix index e6267d7e6..7b28ffca8 100644 --- a/krebs/3modules/git.nix +++ b/krebs/3modules/git.nix @@ -27,7 +27,7 @@ let description = '' Enable cgit. Cgit is an attempt to create a fast web interface for the git version - control system, using a built in cache to decrease pressure on the + control system, using a built in cache to decrease pressure on the git server. cgit in this module is being served via fastcgi nginx.This module deploys a http://cgit.<hostname> nginx configuration and enables nginx @@ -44,48 +44,8 @@ let default = "/etc/git"; }; repos = mkOption { - type = types.attrsOf (types.submodule ({ - options = { - desc = mkOption { - type = types.nullOr types.str; - default = null; - description = '' - Repository description. - ''; - }; - section = mkOption { - type = types.nullOr types.str; - default = null; - description = '' - Repository section. - ''; - }; - name = mkOption { - type = types.str; - description = '' - Repository name. - ''; - }; - hooks = mkOption { - type = types.attrsOf types.str; - default = {}; - description = '' - Repository-specific hooks. - ''; - }; - public = mkOption { - type = types.bool; - default = false; - description = '' - Allow everybody to read the repository via HTTP if cgit enabled. - ''; - # TODO allow every configured user to fetch the repository via SSH. - }; - }; - })); - + type = types.attrsOf subtypes.repo; default = {}; - example = literalExample '' { testing = { @@ -99,7 +59,6 @@ let testing2 = { name = "testing2"; }; } ''; - description = '' Repositories. ''; @@ -121,30 +80,158 @@ let ''; }; rules = mkOption { - type = types.unspecified; + type = types.listOf subtypes.rule; + default = []; + example = literalExample '' + singleton { + user = [ config.krebs.users.tv ]; + repo = [ testing ]; # see literal example of repos + perm = push "refs/*" (with lib.git; [ + non-fast-forward create delete merge + ]); + } + ''; + description = '' + Rules. + ''; }; }; + # TODO put into krebs/4lib/types.nix? + subtypes = { + repo = types.submodule ({ + options = { + collaborators = mkOption { + type = types.listOf types.user; + default = []; + description = '' + List of users that should be able to fetch from this repo. + + This option is currently not used by krebs.git but instead can be + used to create rules. See e.g. <stockholm/tv/2configs/git.nix> for + an example. + ''; + }; + desc = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + Repository description. + ''; + }; + section = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + Repository section. + ''; + }; + name = mkOption { + type = types.str; + description = '' + Repository name. + ''; + }; + hooks = mkOption { + type = types.attrsOf types.str; + default = {}; + description = '' + Repository-specific hooks. + ''; + }; + public = mkOption { + type = types.bool; + default = false; + description = '' + Allow everybody to read the repository via HTTP if cgit enabled. + ''; + # TODO allow every configured user to fetch the repository via SSH. + }; + }; + }); + rule = types.submodule ({ config, ... }: { + options = { + user = mkOption { + type = types.listOf types.user; + description = '' + List of users this rule should apply to. + Checked by authorize-command. + ''; + }; + repo = mkOption { + type = types.listOf subtypes.repo; + description = '' + List of repos this rule should apply to. + Checked by authorize-command. + ''; + }; + perm = mkOption { + type = types.submodule { + # TODO generate enum argument from krebs/4lib/git.nix + options = { + allow-commands = mkOption { + type = types.listOf (types.enum (with git; [ + git-receive-pack + git-upload-pack + ])); + default = []; + description = '' + List of commands the rule's users are allowed to execute. + Checked by authorize-command. + ''; + }; + allow-receive-ref = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + Ref that can receive objects. + Checked by authorize-push. + ''; + }; + allow-receive-modes = mkOption { + type = types.listOf (types.enum (with git; [ + fast-forward + non-fast-forward + create + delete + merge + ])); + default = []; + description = '' + List of allowed receive modes. + Checked by pre-receive hook. + ''; + }; + }; + }; + description = '' + Permissions granted. + ''; + }; + }; + }); + }; + git-imp = { system.activationScripts.git-init = "${init-script}"; - + # TODO maybe put all scripts here and then use PATH? environment.etc."${etc-base}".source = scriptFarm "git-ssh-authorizers" { - authorize-command = makeAuthorizeScript (map ({ repo, user, perm }: [ - (map getName (ensureList user)) - (map getName (ensureList repo)) - (map getName perm.allow-commands) + authorize-command = makeAuthorizeScript (map (rule: [ + (map getName (ensureList rule.user)) + (map getName (ensureList rule.repo)) + (map getName rule.perm.allow-commands) ]) cfg.rules); - - authorize-push = makeAuthorizeScript (map ({ repo, user, perm }: [ - (map getName (ensureList user)) - (map getName (ensureList repo)) - (ensureList perm.allow-receive-ref) - (map getName perm.allow-receive-modes) - ]) (filter (x: hasAttr "allow-receive-ref" x.perm) cfg.rules)); + + authorize-push = makeAuthorizeScript (map (rule: [ + (map getName (ensureList rule.user)) + (map getName (ensureList rule.repo)) + (ensureList rule.perm.allow-receive-ref) + (map getName rule.perm.allow-receive-modes) + ]) (filter (rule: rule.perm.allow-receive-ref != null) cfg.rules)); }; - + users.extraUsers = singleton rec { description = "Git repository hosting user"; name = "git"; diff --git a/krebs/3modules/tv/default.nix b/krebs/3modules/tv/default.nix index 6fd1c4224..31c1a375a 100644 --- a/krebs/3modules/tv/default.nix +++ b/krebs/3modules/tv/default.nix @@ -247,6 +247,7 @@ with lib; }; }; secure = true; + ssh.privkey.path = <secrets/ssh.id_ed25519>; ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIcJvu8JDVzObLUtlAQg9qVugthKSfitwCljuJ5liyHa"; }; xu = { diff --git a/krebs/3modules/urlwatch.nix b/krebs/3modules/urlwatch.nix index 31cbfcf6e..7a9fb55fd 100644 --- a/krebs/3modules/urlwatch.nix +++ b/krebs/3modules/urlwatch.nix @@ -54,6 +54,10 @@ let example = [ https://nixos.org/channels/nixos-unstable/git-revision ]; + apply = map (x: getAttr (typeOf x) { + set = x; + string.url = x; + }); }; verbose = mkOption { type = types.bool; @@ -64,7 +68,40 @@ let }; }; - urlsFile = toFile "urls" (concatStringsSep "\n" cfg.urls); + urlsFile = toFile "urls" (concatMapStringsSep "\n---\n" toJSON cfg.urls); + + configFile = toFile "urlwatch.yaml" (toJSON { + display = { + error = true; + new = true; + unchanged = false; + }; + report = { + email = { + enabled = false; + from = ""; + html = false; + smtp = { + host = "localhost"; + keyring = true; + port = 25; + starttls = true; + }; + subject = "{count} changes: {jobs}"; + to = ""; + }; + html.diff = "unified"; + stdout = { + color = true; + enabled = true; + }; + text = { + details = true; + footer = true; + line_length = 75; + }; + }; + }); imp = { systemd.timers.urlwatch = { @@ -109,10 +146,15 @@ let from=${escapeShellArg cfg.from} mailto=${escapeShellArg cfg.mailto} urlsFile=${escapeShellArg urlsFile} + configFile=${escapeShellArg configFile} cd /tmp - urlwatch -e ${optionalString cfg.verbose "-v"} --urls="$urlsFile" > changes || : + urlwatch \ + ${optionalString cfg.verbose "-v"} \ + --urls="$urlsFile" \ + --config="$configFile" \ + > changes || : if test -s changes; then date=$(date -R) diff --git a/krebs/4lib/default.nix b/krebs/4lib/default.nix index dfc51bbe4..4d7e0b549 100644 --- a/krebs/4lib/default.nix +++ b/krebs/4lib/default.nix @@ -6,6 +6,7 @@ with lib; let out = rec { eq = x: y: x == y; + ne = x: y: x != y; mod = x: y: x - y * (x / y); diff --git a/krebs/4lib/types.nix b/krebs/4lib/types.nix index 81ce659bd..c596d0f9d 100644 --- a/krebs/4lib/types.nix +++ b/krebs/4lib/types.nix @@ -164,10 +164,6 @@ types // rec { pubkey = mkOption { type = str; }; - pubkeys = mkOption { - type = attrsOf str; - default = {}; - }; }; }; diff --git a/krebs/5pkgs/Reaktor/plugins.nix b/krebs/5pkgs/Reaktor/plugins.nix index 7490be4ca..0f61688e3 100644 --- a/krebs/5pkgs/Reaktor/plugins.nix +++ b/krebs/5pkgs/Reaktor/plugins.nix @@ -82,7 +82,7 @@ rec { }; stockholm-issue = buildSimpleReaktorPlugin "stockholm-issue" { script = ./scripts/random-issue.sh; - path = with pkgs; [ git gnused lentil ]; + path = with pkgs; [ git gnused haskellPackages.lentil ]; env = { "origin" = "http://cgit.gum/stockholm"; }; }; diff --git a/krebs/5pkgs/cac-api/default.nix b/krebs/5pkgs/cac-api/default.nix new file mode 100644 index 000000000..9ab6ac8b2 --- /dev/null +++ b/krebs/5pkgs/cac-api/default.nix @@ -0,0 +1,42 @@ +{ stdenv, fetchgit, bc, cac-cert, coreutils, curl, dash, gnused, inotifyTools, jq, ncurses, openssh, sshpass, ... }: + +stdenv.mkDerivation { + name = "cac-api-1.1.0"; + + src = fetchgit { + url = http://cgit.cd.krebsco.de/cac-api; + rev = "0809fae379239687ed1170e04311dc2880ef0aba"; + sha256 = "357ced27c9ed88028967c934178a1d230bf38617a7494cd4632fabdd2a04fcdd"; + }; + + phases = [ + "unpackPhase" + "installPhase" + ]; + + installPhase = '' + mkdir -p $out/bin + { cat <<\EOF + #! ${dash}/bin/dash + export PATH=${stdenv.lib.makeSearchPath "bin" [ + bc + coreutils + curl + gnused + inotifyTools + jq + ncurses + openssh + sshpass + ]} + EOF + # [1]: Disable fetching tasks; listtasks is currently broken: + # Unknown column 'iod.apitask.cid' in 'field list' + sed ' + /^\s*tasks \\$/d; # [1] + s|\<_cac_exec curl|<${cac-cert} & --cacert /dev/stdin| + ' cac-api + } > $out/bin/cac-api + chmod +x $out/bin/cac-api + ''; +} diff --git a/krebs/5pkgs/test/infest-cac-centos7/panel.cloudatcost.com.crt b/krebs/5pkgs/cac-cert/cac.pem index 9d02b6bcf..9d02b6bcf 100644 --- a/krebs/5pkgs/test/infest-cac-centos7/panel.cloudatcost.com.crt +++ b/krebs/5pkgs/cac-cert/cac.pem diff --git a/krebs/5pkgs/cac-cert/default.nix b/krebs/5pkgs/cac-cert/default.nix new file mode 100644 index 000000000..d99019dca --- /dev/null +++ b/krebs/5pkgs/cac-cert/default.nix @@ -0,0 +1,2 @@ +{ writeText, ... }: +writeText "cac.pem" (builtins.readFile ./cac.pem) diff --git a/krebs/5pkgs/cac/default.nix b/krebs/5pkgs/cac/default.nix deleted file mode 100644 index 4d39ce2fb..000000000 --- a/krebs/5pkgs/cac/default.nix +++ /dev/null @@ -1,39 +0,0 @@ -{ stdenv, fetchgit, bc, coreutils, curl, gnused, inotifyTools, jq, ncurses, sshpass, ... }: - -stdenv.mkDerivation { - name = "cac-1.0.3"; - - src = fetchgit { - url = http://cgit.cd.retiolum/cac; - rev = "22acc1b990ac7d97c16344fbcbc2621e24cdf915"; - sha256 = "135b740617c983b3f46a1983d4744be17340d5146a0a0de0dff4bb7a53688f2f"; - }; - - phases = [ - "unpackPhase" - "installPhase" - ]; - - installPhase = - let - path = stdenv.lib.makeSearchPath "bin" [ - bc - coreutils - curl - gnused - inotifyTools - jq - ncurses - sshpass - ]; - in - '' - mkdir -p $out/bin - - sed < ./cac > $out/bin/cac ' - s;^_cac_main .*;PATH=${path}''${PATH+:$PATH} &; - ' - - chmod +x $out/bin/cac - ''; -} diff --git a/krebs/5pkgs/get/default.nix b/krebs/5pkgs/get/default.nix index 9a0192aab..13cdeca96 100644 --- a/krebs/5pkgs/get/default.nix +++ b/krebs/5pkgs/get/default.nix @@ -1,12 +1,12 @@ { coreutils, gnugrep, gnused, fetchgit, jq, nix, stdenv, ... }: stdenv.mkDerivation { - name = "get-1.3.1"; + name = "get-1.4.0"; src = fetchgit { url = http://cgit.cd.krebsco.de/get; - rev = "64c97edd3f9952cd5e703208c46748a035a515bf"; - sha256 = "32ca83f4fd86fd3285bef9dcfd0917308086d239189858daceca175de49ff97c"; + rev = "08757d47c480c130d69270855c6c0371f6b7d385"; + sha256 = "7c609e2cde7a071bbf62241a7bea60313fdbf076b9f7b3d97226417e13e5ba9d"; }; phases = [ diff --git a/krebs/5pkgs/lentil/default.nix b/krebs/5pkgs/lentil/default.nix deleted file mode 100644 index 8a57a77fe..000000000 --- a/krebs/5pkgs/lentil/default.nix +++ /dev/null @@ -1,15 +0,0 @@ -{ pkgs, ... }: - -(pkgs.haskellPackages.override { - overrides = self: super: { - lentil = super.lentil.override { - mkDerivation = (attrs: self.mkDerivation (attrs // { - version = "0.1.3.0"; - sha256 = "0xa59avh0bvfg69xh9p5b8dppfhx29mvfq8v41sk9j7qbcnzjivg"; - patches = [ - ./syntaxes.patch - ]; - })); - }; - }; -}).lentil diff --git a/krebs/5pkgs/lentil/syntaxes.patch b/krebs/5pkgs/lentil/syntaxes.patch deleted file mode 100644 index a9390ae51..000000000 --- a/krebs/5pkgs/lentil/syntaxes.patch +++ /dev/null @@ -1,11 +0,0 @@ -diff -rN -u old-lentil/src/Lentil/Parse/Syntaxes.hs new-lentil/src/Lentil/Parse/Syntaxes.hs ---- old-lentil/src/Lentil/Parse/Syntaxes.hs 2015-07-20 23:15:38.600539779 +0200 -+++ new-lentil/src/Lentil/Parse/Syntaxes.hs 2015-07-20 23:15:38.600539779 +0200 -@@ -30,6 +30,7 @@ - | ext `elem` [".pas", ".pp", ".inc"] = Just pascal - | ext `elem` [".py"] = Just python - | ext `elem` [".rb"] = Just ruby -+ | ext `elem` [".nix"] = Just perl -- Nix - | ext `elem` [".pl", ".pm", ".t"] = Just perl - | ext `elem` [".sh"] = Just perl -- shell - | ext `elem` [".txt"] = Just text diff --git a/krebs/5pkgs/much/default.nix b/krebs/5pkgs/much/default.nix index 2a9376139..efe84fbf0 100644 --- a/krebs/5pkgs/much/default.nix +++ b/krebs/5pkgs/much/default.nix @@ -1,6 +1,6 @@ { pkgs, ... }: -pkgs.haskellngPackages.callPackage ( +pkgs.haskellPackages.callPackage ( { mkDerivation, aeson, attoparsec, base, base64-bytestring , blaze-builder, blessings, bytestring, case-insensitive, containers, deepseq , directory, docopt, email-header, fetchgit, filepath diff --git a/krebs/5pkgs/test/infest-cac-centos7/default.nix b/krebs/5pkgs/test/infest-cac-centos7/default.nix index 7adb09ca9..3be4b1c41 100644 --- a/krebs/5pkgs/test/infest-cac-centos7/default.nix +++ b/krebs/5pkgs/test/infest-cac-centos7/default.nix @@ -1,4 +1,4 @@ -{ stdenv, coreutils,makeWrapper, cac-api, cac-panel, gnumake, gnused, jq, openssh, ... }: +{ stdenv, coreutils,makeWrapper, cac-api, cac-cert, cac-panel, gnumake, gnused, jq, openssh, ... }: stdenv.mkDerivation rec { name = "${shortname}-${version}"; @@ -10,6 +10,7 @@ stdenv.mkDerivation rec { phases = [ "installPhase" ]; + buildInputs = [ makeWrapper ]; path = stdenv.lib.makeSearchPath "bin" [ @@ -22,16 +23,15 @@ stdenv.mkDerivation rec { openssh ]; - installPhase = - '' - mkdir -p $out/bin - cp ${src} $out/bin/${shortname} - chmod +x $out/bin/${shortname} - wrapProgram $out/bin/${shortname} \ - --prefix PATH : ${path} \ - --set SSL_CERT_FILE ${./panel.cloudatcost.com.crt} \ - --set REQUESTS_CA_BUNDLE ${./panel.cloudatcost.com.crt} - ''; + installPhase = '' + mkdir -p $out/bin + cp ${src} $out/bin/${shortname} + chmod +x $out/bin/${shortname} + wrapProgram $out/bin/${shortname} \ + --prefix PATH : ${path} \ + --set REQUESTS_CA_BUNDLE ${cac-cert} \ + --set SSL_CERT_FILE ${cac-cert} + ''; meta = with stdenv.lib; { homepage = http://krebsco.de; description = "Krebs CI Scripts"; diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes index 793ef3560..b3beb392f 100755 --- a/krebs/5pkgs/test/infest-cac-centos7/notes +++ b/krebs/5pkgs/test/infest-cac-centos7/notes @@ -1,5 +1,3 @@ -#! /bin/sh - # nix-shell -p gnumake jq openssh cac-api cac-panel set -eufx @@ -54,7 +52,7 @@ cac-api servers old_trapstr=$(clear_defer) while true;do # Template 26: CentOS7 - # TODO: use cac templates to determine the real Centos7 template in case it changes + # TODO: use cac-api templates to determine the real Centos7 template in case it changes out=$(cac-api build cpu=1 ram=512 storage=10 os=26 2>&1) if name=$(echo "$out" | jq -r .servername);then id=servername:$name @@ -67,15 +65,15 @@ while true;do fi clear_defer >/dev/null - defer "cac delete $id" + defer "cac-api delete $id" # TODO: timeout? wait_login_cac(){ # we wait for 30 minutes for t in `seq 180`;do - # now we have a working cac server - if cac ssh $1 -o ConnectTimeout=10 \ + # now we have a working cac-api server + if cac-api ssh $1 -o ConnectTimeout=10 \ cat /etc/redhat-release | \ grep CentOS ;then return 0 @@ -134,7 +132,7 @@ cac-api powerop $id reset wait_login(){ # timeout for t in `seq 90`;do - # now we have a working cac server + # now we have a working cac-api server if ssh -o StrictHostKeyChecking=no \ -o UserKnownHostsFile=/dev/null \ -i $krebs_ssh \ diff --git a/krebs/5pkgs/urlwatch/default.nix b/krebs/5pkgs/urlwatch/default.nix new file mode 100644 index 000000000..d9b595314 --- /dev/null +++ b/krebs/5pkgs/urlwatch/default.nix @@ -0,0 +1,39 @@ +{ stdenv, fetchurl, python3Packages }: + +python3Packages.buildPythonPackage rec { + name = "urlwatch-2.0"; + + src = fetchurl { + url = "https://thp.io/2008/urlwatch/${name}.tar.gz"; + sha256 = "0j38qzw4jxw41vnnpi6j851hqpv8d6p1cbni6cv8r2vqf5307s3b"; + }; + + propagatedBuildInputs = with python3Packages; [ + pyyaml + keyring + (python3Packages.buildPythonPackage rec { + name = "minidb-2.0.1"; + src = fetchurl { + url = "https://thp.io/2010/minidb/${name}.tar.gz"; + sha256 = "1x958zr9jc26vaqij451qb9m2l7apcpz34ir9fwfjg4fwv24z2dy"; + }; + meta = { + description = "A simple SQLite3-based store for Python objects"; + homepage = https://thp.io/2010/minidb/; + license = stdenv.lib.licenses.isc; + maintainers = [ stdenv.lib.maintainers.tv ]; + }; + }) + ]; + + postFixup = '' + wrapProgram "$out/bin/urlwatch" --prefix "PYTHONPATH" : "$PYTHONPATH" + ''; + + meta = { + description = "A tool for monitoring webpages for updates"; + homepage = https://thp.io/2008/urlwatch/; + license = stdenv.lib.licenses.bsd3; + maintainers = [ stdenv.lib.maintainers.tv ]; + }; +}# diff --git a/krebs/5pkgs/with-tmpdir/default.nix b/krebs/5pkgs/with-tmpdir/default.nix new file mode 100644 index 000000000..517e46310 --- /dev/null +++ b/krebs/5pkgs/with-tmpdir/default.nix @@ -0,0 +1,29 @@ +{ stdenv, fetchgit, coreutils, dash, ... }: + +stdenv.mkDerivation { + name = "with-tmpdir-1"; + + src = fetchgit { + url = http://cgit.cd.krebsco.de/with-tmpdir; + rev = "3243c02ed8cd27a04c080bd39560204980f6c16a"; + sha256 = "80ee6cafb2c337999ddcd1e41747d6256b7cfcea605358c2046eb7e3729555c6"; + }; + + phases = [ + "unpackPhase" + "installPhase" + ]; + + installPhase = '' + mkdir -p $out/bin + + { echo '#! ${dash}/bin/dash' + echo 'OLDPATH=$PATH' + echo 'PATH=${coreutils}/bin' + sed '$s/^/#/' ./with-tmpdir + echo '(PATH=$OLDPATH; exec "$@")' + } > $out/bin/with-tmpdir + + chmod +x $out/bin/with-tmpdir + ''; +} diff --git a/krebs/default.nix b/krebs/default.nix index 15d0e8e2e..17c035896 100644 --- a/krebs/default.nix +++ b/krebs/default.nix @@ -1,5 +1,6 @@ -{ current-date -, current-host-name +assert false; + +{ current-host-name , current-user-name , lib , stockholm @@ -7,31 +8,11 @@ }: let out = { - inherit deploy; inherit infest; inherit init; inherit nixos-install; - inherit populate; }; - deploy = - { system ? current-host-name - , target ? system - }@args: let - config = get-config system; - in '' - #! /bin/sh - # ${current-date} ${current-user-name}@${current-host-name} - # krebs.deploy - set -efu - (${populate args}) - ${rootssh target '' - ${nix-install args} - ${config.krebs.build.profile}/bin/switch-to-configuration switch - ''} - echo OK - ''; - infest = { system ? current-host-name , target ? system @@ -39,7 +20,6 @@ let out = { config = get-config system; in '' #! /bin/sh - # ${current-date} ${current-user-name}@${current-host-name} # krebs.infest set -efu @@ -48,9 +28,6 @@ let out = { ${builtins.readFile ./4lib/infest/install-nix.sh} ''} - # Prepare target source via bind-mounting - - (${nixos-install args}) ${rootssh target '' @@ -64,7 +41,6 @@ let out = { config = get-config system; in '' #! /bin/sh - # ${current-date} ${current-user-name}@${current-host-name} # krebs.init set -efu @@ -100,7 +76,6 @@ let out = { }@args: let in '' #! /bin/sh - # ${current-date} ${current-user-name}@${current-host-name} # krebs.nixos-install (${populate (args // { root = "/mnt"; })}) @@ -171,9 +146,10 @@ let out = { ${b} ''; - get-config = system: - stockholm.users.${current-user-name}.${system}.config + get-config = system: let + config = stockholm.users.${current-user-name}.${system}.config or (abort "unknown system: ${system}, user: ${current-user-name}"); + in config; nix-install = { system ? current-host-name @@ -193,7 +169,6 @@ let out = { nix-env \ --show-trace \ -f '<stockholm>' \ - --argstr current-date ${lib.shell.escape current-date} \ --argstr current-host-name ${lib.shell.escape current-host-name} \ --argstr current-user-name ${lib.shell.escape current-user-name} \ --profile ${lib.shell.escape config.krebs.build.profile} \ @@ -206,74 +181,6 @@ let out = { ])} ''; - populate = - { system ? current-host-name - , target ? system - , root ? "" - }@args: - let out = '' - #! /bin/sh - # ${current-date} ${current-user-name}@${current-host-name} - set -efu - ${lib.concatStringsSep "\n" - (lib.concatMap - (type: lib.mapAttrsToList (_: methods.${type}) - config.krebs.build.source.${type}) - ["dir" "git"])} - ''; - - - config = get-config system; - - current-host = config.krebs.hosts.${current-host-name}; - current-user = config.krebs.users.${current-user-name}; - - methods.dir = config: - let - can-push = config.host.name == current-host.name; - target-path = root + config.target-path; - push-method = '' - rsync \ - --exclude .git \ - --exclude .graveyard \ - --exclude old \ - --exclude tmp \ - --rsync-path='mkdir -p ${target-path} && rsync' \ - --delete-excluded \ - -vrLptgoD \ - ${config.path}/ \ - root@${target}:${target-path} - ''; - in - if can-push then push-method else - let dir = "file://${config.host.name}${config.path}"; in - # /!\ revise this message when using more than just push-method - throw "No way to push ${dir} from ${current-host.name} to ${target}"; - - methods.git = config: - let target-path = root + config.target-path; - in rootssh target '' - mkdir -p ${target-path} - cd ${target-path} - if ! test -e .git; then - git init - fi - if ! cur_url=$(git config remote.origin.url 2>/dev/null); then - git remote add origin ${config.url} - elif test "$cur_url" != ${config.url}; then - git remote set-url origin ${config.url} - fi - if test "$(git rev-parse --verify HEAD 2>/dev/null)" != ${config.rev}; then - git fetch origin - git checkout ${config.rev} -- . - git checkout -q ${config.rev} - git submodule init - git submodule update - fi - git clean -dxf - ''; - in out; - rootssh = target: script: let flags = "-o StrictHostKeyChecking=${StrictHostKeyChecking}"; diff --git a/makefu/1systems/omo.nix b/makefu/1systems/omo.nix index e9c51f485..d15cc2779 100644 --- a/makefu/1systems/omo.nix +++ b/makefu/1systems/omo.nix @@ -40,7 +40,7 @@ in { networking.firewall.allowedTCPPorts = [ 80 655 8080 ]; # services.openssh.allowSFTP = false; - krebs.build.source.git.nixpkgs.rev = "d0e3cca04edd5d1b3d61f188b4a5f61f35cdf1ce"; + krebs.build.source.nixpkgs.rev = "d0e3cca04edd5d1b3d61f188b4a5f61f35cdf1ce"; # copy config from <secrets/sabnzbd.ini> to /var/lib/sabnzbd/ services.sabnzbd.enable = true; diff --git a/makefu/1systems/vbob.nix b/makefu/1systems/vbob.nix index d95362919..6c8f5ca26 100644 --- a/makefu/1systems/vbob.nix +++ b/makefu/1systems/vbob.nix @@ -37,8 +37,8 @@ extraEnviron = { NIX_PATH="nixpkgs=${toString <nixpkgs>}"; }; }; - krebs.build.source.git.nixpkgs = { - #url = https://github.com/nixos/nixpkgs; + krebs.build.source.nixpkgs = { + # url = https://github.com/nixos/nixpkgs; # HTTP Everywhere + libredir rev = "8239ac6"; }; diff --git a/makefu/2configs/backup.nix b/makefu/2configs/backup.nix new file mode 100644 index 000000000..ed6d1f4a7 --- /dev/null +++ b/makefu/2configs/backup.nix @@ -0,0 +1,30 @@ +{ config, lib, ... }: +with lib; +let + startAt = "0,6,12,18:00"; + defaultBackupServer = config.krebs.hosts.omo; + defaultBackupDir = "/home/backup"; + defaultPull = host: src: { + method = "pull"; + src = { + inherit host; + path = src; + }; + dst = { + host = defaultBackupServer; + path = defaultBackupDir + src; + }; + startAt = "0,6,12,18:00"; + snapshots = { + hourly = { format = "%Y-%m-%dT%H"; retain = 4; }; + daily = { format = "%Y-%m-%d"; retain = 7; }; + weekly = { format = "%YW%W"; retain = 4; }; + monthly = { format = "%Y-%m"; retain = 12; }; + yearly = { format = "%Y"; }; + }; + }; +in { + krebs.backup.plans = addNames { + wry-to-omo_var-www = defaultPull wry "/var/www"; + }; +} diff --git a/makefu/2configs/default.nix b/makefu/2configs/default.nix index ec1100582..a98393e2b 100644 --- a/makefu/2configs/default.nix +++ b/makefu/2configs/default.nix @@ -20,24 +20,18 @@ with lib; build = { target = mkDefault "root@${config.krebs.build.host.name}"; user = config.krebs.users.makefu; - source = { - git.nixpkgs = { - #url = https://github.com/NixOS/nixpkgs; + source = { + upstream-nixpkgs = { url = mkDefault https://github.com/nixos/nixpkgs; rev = mkDefault "93d8671e2c6d1d25f126ed30e5e6f16764330119"; # unstable @ 2015-01-03, tested on filepimp - target-path = "/var/src/nixpkgs"; }; + secrets = "/home/makefu/secrets/${config.krebs.build.host.name}/"; + stockholm = "/home/makefu/stockholm"; - dir.secrets = { - host = config.krebs.hosts.pornocauster; - path = "/home/makefu/secrets/${config.krebs.build.host.name}/"; - }; - - dir.stockholm = { - host = config.krebs.hosts.pornocauster; - path = "/home/makefu/stockholm" ; - target-path = "/var/src/stockholm"; - }; + # Defaults for all stockholm users? + nixos-config = "symlink:stockholm/${config.krebs.build.user.name}/1systems/${config.krebs.build.host.name}.nix"; + nixpkgs = symlink:stockholm/nixpkgs; + stockholm-user = "symlink:stockholm/${config.krebs.build.user.name}"; }; }; }; diff --git a/makefu/2configs/unstable-sources.nix b/makefu/2configs/unstable-sources.nix index 7a9a8a81c..a34377683 100644 --- a/makefu/2configs/unstable-sources.nix +++ b/makefu/2configs/unstable-sources.nix @@ -1,7 +1,7 @@ _: { - krebs.build.source.git.nixpkgs = { + krebs.build.source.nixpkgs = { url = https://github.com/makefu/nixpkgs; rev = "15b5bbfbd1c8a55e7d9e05dd9058dc102fac04fe"; # cherry-picked collectd }; diff --git a/nixpkgs/default.nix b/nixpkgs/default.nix new file mode 120000 index 000000000..74e9d7633 --- /dev/null +++ b/nixpkgs/default.nix @@ -0,0 +1 @@ +../upstream-nixpkgs/default.nix
\ No newline at end of file diff --git a/nixpkgs/krebs b/nixpkgs/krebs new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/nixpkgs/krebs diff --git a/nixpkgs/lib b/nixpkgs/lib new file mode 120000 index 000000000..2284ef489 --- /dev/null +++ b/nixpkgs/lib @@ -0,0 +1 @@ +../upstream-nixpkgs/lib
\ No newline at end of file diff --git a/nixpkgs/nixos/default.nix b/nixpkgs/nixos/default.nix new file mode 100644 index 000000000..4fe08efd2 --- /dev/null +++ b/nixpkgs/nixos/default.nix @@ -0,0 +1 @@ +import <stockholm> diff --git a/nixpkgs/nixos/lib b/nixpkgs/nixos/lib new file mode 120000 index 000000000..9e69d1a67 --- /dev/null +++ b/nixpkgs/nixos/lib @@ -0,0 +1 @@ +../../../upstream-nixpkgs/nixos/lib
\ No newline at end of file diff --git a/nixpkgs/nixos/modules b/nixpkgs/nixos/modules new file mode 120000 index 000000000..8aa24885c --- /dev/null +++ b/nixpkgs/nixos/modules @@ -0,0 +1 @@ +../../../upstream-nixpkgs/nixos/modules
\ No newline at end of file diff --git a/nixpkgs/pkgs b/nixpkgs/pkgs new file mode 120000 index 000000000..ce5f5448b --- /dev/null +++ b/nixpkgs/pkgs @@ -0,0 +1 @@ +../upstream-nixpkgs/pkgs
\ No newline at end of file @@ -0,0 +1 @@ +../stockholm-user
\ No newline at end of file diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix index c614bd3c1..9982dd915 100644 --- a/shared/2configs/buildbot-standalone.nix +++ b/shared/2configs/buildbot-standalone.nix @@ -86,7 +86,6 @@ -I stockholm=. \ --show-trace \ -I secrets=. '<stockholm>' \ - --argstr current-date lol \ --argstr current-user-name shared \ --argstr current-host-name lol \ --strict --json"]) @@ -98,7 +97,6 @@ -I stockholm=. \ -I secrets=. '<stockholm>' \ --show-trace \ - --argstr current-date lol \ --argstr current-user-name shared \ --argstr current-host-name lol \ --strict --json"]) diff --git a/tv/1systems/cd.nix b/tv/1systems/cd.nix index 8c2a9ae43..e42d5750a 100644 --- a/tv/1systems/cd.nix +++ b/tv/1systems/cd.nix @@ -6,12 +6,17 @@ with lib; krebs.build.host = config.krebs.hosts.cd; krebs.build.target = "root@cd.internet"; + krebs.build.source.upstream-nixpkgs = { + url = https://github.com/NixOS/nixpkgs; + rev = "b7ff030"; + }; + imports = [ ../2configs/hw/CAC-Developer-2.nix ../2configs/fs/CAC-CentOS-7-64bit.nix - #../2configs/consul-server.nix ../2configs/exim-smarthost.nix ../2configs/git.nix + ../2configs/retiolum.nix ../2configs/urlwatch.nix { imports = [ ../2configs/charybdis.nix ]; @@ -25,6 +30,10 @@ with lib; enable = true; hosts = [ "jabber.viljetic.de" ]; }; + tv.iptables.input-internet-accept-new-tcp = [ + "xmpp-client" + "xmpp-server" + ]; } { krebs.github-hosts-sync.enable = true; @@ -32,38 +41,17 @@ with lib; singleton config.krebs.github-hosts-sync.port; } { - tv.iptables = { - enable = true; - input-internet-accept-new-tcp = [ - "ssh" - "tinc" - "smtp" - "xmpp-client" - "xmpp-server" - ]; - input-retiolum-accept-new-tcp = [ - "http" - ]; - }; - } - { - tv.iptables.input-internet-accept-new-tcp = singleton "http"; krebs.nginx.servers.cgit.server-names = [ "cgit.cd.krebsco.de" "cgit.cd.viljetic.de" ]; - } - { # TODO make public_html also available to cd, cd.retiolum (AKA default) - tv.iptables.input-internet-accept-new-tcp = singleton "http"; krebs.nginx.servers.public_html = { server-names = singleton "cd.viljetic.de"; locations = singleton (nameValuePair "~ ^/~(.+?)(/.*)?\$" '' alias /home/$1/public_html$2; ''); }; - } - { krebs.nginx.servers.viljetic = { server-names = singleton "viljetic.de"; # TODO directly set root (instead via location) @@ -71,16 +59,7 @@ with lib; root ${pkgs.viljetic-pages}; ''); }; - } - { - krebs.retiolum = { - enable = true; - connectTo = [ - "fastpoke" - "pigstarter" - "ire" - ]; - }; + tv.iptables.input-internet-accept-new-tcp = singleton "http"; } ]; diff --git a/tv/1systems/mkdir.nix b/tv/1systems/mkdir.nix index 9d8a0bcfa..79e5f73b9 100644 --- a/tv/1systems/mkdir.nix +++ b/tv/1systems/mkdir.nix @@ -22,7 +22,6 @@ in imports = [ ../2configs/hw/CAC-Developer-1.nix ../2configs/fs/CAC-CentOS-7-64bit.nix - ../2configs/consul-server.nix ../2configs/exim-smarthost.nix ../2configs/git.nix { diff --git a/tv/1systems/nomic.nix b/tv/1systems/nomic.nix index 0c6c935a3..37ef204c7 100644 --- a/tv/1systems/nomic.nix +++ b/tv/1systems/nomic.nix @@ -5,45 +5,14 @@ with lib; { krebs.build.host = config.krebs.hosts.nomic; - krebs.build.target = "root@nomic.gg23"; - imports = [ ../2configs/hw/AO753.nix - #../2configs/consul-server.nix + ../2configs/exim-retiolum.nix ../2configs/git.nix - { - tv.iptables = { - enable = true; - input-internet-accept-new-tcp = [ - "ssh" - "http" - "tinc" - "smtp" - ]; - }; - } - { - krebs.exim-retiolum.enable = true; - } - { - krebs.nginx = { - enable = true; - servers.default.locations = [ - (nameValuePair "~ ^/~(.+?)(/.*)?\$" '' - alias /home/$1/public_html$2; - '') - ]; - }; - } - { - krebs.retiolum = { - enable = true; - connectTo = [ - "gum" - "pigstarter" - ]; - }; - } + ../2configs/nginx-public_html.nix + ../2configs/pulse.nix + ../2configs/retiolum.nix + ../2configs/xserver ]; boot.initrd.luks = { diff --git a/tv/1systems/rmdir.nix b/tv/1systems/rmdir.nix index 1f1d975c9..6fd79c596 100644 --- a/tv/1systems/rmdir.nix +++ b/tv/1systems/rmdir.nix @@ -23,7 +23,6 @@ in imports = [ ../2configs/hw/CAC-Developer-1.nix ../2configs/fs/CAC-CentOS-7-64bit.nix - ../2configs/consul-server.nix ../2configs/exim-smarthost.nix ../2configs/git.nix { diff --git a/tv/1systems/wu.nix b/tv/1systems/wu.nix index 54ceb7783..29e6de08b 100644 --- a/tv/1systems/wu.nix +++ b/tv/1systems/wu.nix @@ -7,10 +7,12 @@ with lib; imports = [ ../2configs/hw/w110er.nix - #../2configs/consul-client.nix + ../2configs/exim-retiolum.nix ../2configs/git.nix ../2configs/mail-client.nix + ../2configs/nginx-public_html.nix ../2configs/pulse.nix + ../2configs/retiolum.nix ../2configs/xserver { environment.systemPackages = with pkgs; [ @@ -18,7 +20,7 @@ with lib; # stockholm gnumake hashPassword - lentil + haskellPackages.lentil parallel (pkgs.writeScriptBin "im" '' #! ${pkgs.bash}/bin/bash @@ -41,7 +43,7 @@ with lib; # tv bc bind # dig - cac + cac-api dic file get @@ -123,39 +125,6 @@ with lib; unison ]; } - { - tv.iptables = { - enable = true; - input-internet-accept-new-tcp = [ - "ssh" - "http" - "tinc" - "smtp" - ]; - }; - } - { - krebs.exim-retiolum.enable = true; - } - { - krebs.nginx = { - enable = true; - servers.default.locations = [ - (nameValuePair "~ ^/~(.+?)(/.*)?\$" '' - alias /home/$1/public_html$2; - '') - ]; - }; - } - { - krebs.retiolum = { - enable = true; - connectTo = [ - "gum" - "pigstarter" - ]; - }; - } ]; boot.initrd.luks = { @@ -188,7 +157,7 @@ with lib; nixpkgs.config.chromium.enablePepperFlash = true; - nixpkgs.config.allowUnfree = true; + nixpkgs.config.allowUnfreePredicate = pkg: hasPrefix "nvidia-x11-" pkg.name; hardware.bumblebee.enable = true; hardware.bumblebee.group = "video"; hardware.enableAllFirmware = true; diff --git a/tv/1systems/xu.nix b/tv/1systems/xu.nix index 1f3e010a4..8c4af2bd3 100644 --- a/tv/1systems/xu.nix +++ b/tv/1systems/xu.nix @@ -5,15 +5,14 @@ with lib; { krebs.build.host = config.krebs.hosts.xu; - krebs.build.source.git.nixpkgs.rev = - "7ae05edcdd14f6ace83ead9bf0d114e97c89a83a"; - imports = [ ../2configs/hw/x220.nix - #../2configs/consul-client.nix + ../2configs/exim-retiolum.nix ../2configs/git.nix ../2configs/mail-client.nix + ../2configs/nginx-public_html.nix ../2configs/pulse.nix + ../2configs/retiolum.nix ../2configs/xserver { environment.systemPackages = with pkgs; [ @@ -21,7 +20,7 @@ with lib; # stockholm gnumake hashPassword - lentil + haskellPackages.lentil parallel (pkgs.writeScriptBin "im" '' #! ${pkgs.bash}/bin/bash @@ -124,40 +123,6 @@ with lib; unison ]; } - { - tv.iptables = { - enable = true; - input-internet-accept-new-tcp = [ - "ssh" - "http" - "tinc" - "smtp" - ]; - }; - } - { - krebs.exim-retiolum.enable = true; - } - { - krebs.nginx = { - enable = true; - servers.default.locations = [ - (nameValuePair "~ ^/~(.+?)(/.*)?\$" '' - alias /home/$1/public_html$2; - '') - ]; - }; - } - { - krebs.retiolum = { - enable = true; - connectTo = [ - "cd" - "gum" - "pigstarter" - ]; - }; - } ]; boot.initrd.luks = { @@ -190,7 +155,6 @@ with lib; nixpkgs.config.chromium.enablePepperFlash = true; - nixpkgs.config.allowUnfree = true; #hardware.bumblebee.enable = true; #hardware.bumblebee.group = "video"; hardware.enableAllFirmware = true; diff --git a/tv/2configs/backup.nix b/tv/2configs/backup.nix index 51d3bb8a7..ce937a744 100644 --- a/tv/2configs/backup.nix +++ b/tv/2configs/backup.nix @@ -2,41 +2,17 @@ with lib; { krebs.backup.plans = addNames { - xu-test-cd = { + wu-home-xu = { method = "push"; - - src = { host = config.krebs.hosts.xu; path = "/tmp/xu-test"; }; - dst = { host = config.krebs.hosts.cd; path = "/tmp/backups/xu-test"; }; - - #startAt = "0,6,12,18:00"; - startAt = "minutely"; + src = { host = config.krebs.hosts.wu; path = "/home"; }; + dst = { host = config.krebs.hosts.xu; path = "/bku/wu-home"; }; + startAt = "05:00"; snapshots = { - minutely = { format = "%Y-%m-%dT%H:%M"; retain = 5; }; - hourly = { format = "%Y-%m-%dT%H"; retain = 4; }; - daily = { format = "%Y-%m-%d"; retain = 7; }; - weekly = { format = "%YW%W"; retain = 4; }; - monthly = { format = "%Y-%m"; retain = 12; }; - yearly = { format = "%Y"; }; + daily = { format = "%Y-%m-%d"; retain = 7; }; + weekly = { format = "%YW%W"; retain = 4; }; + monthly = { format = "%Y-%m"; retain = 12; }; + yearly = { format = "%Y"; }; }; }; - #xu-test-wu = { - # method = "push"; - # dst = { user = tv; host = wu; path = "/krebs/backup/xu-test"; }; - #}; - cd-test-xu = { - method = "pull"; - src = { host = config.krebs.hosts.cd; path = "/tmp/cd-test"; }; - dst = { host = config.krebs.hosts.xu; path = "/tmp/backups/cd-test"; }; - startAt = "minutely"; - snapshots = { - minutely = { format = "%Y-%m-%dT%H:%M"; retain = 5; }; - hourly = { format = "%Y-%m-%dT%H"; retain = 4; }; - daily = { format = "%Y-%m-%d"; retain = 7; }; - weekly = { format = "%YW%W"; retain = 4; }; - monthly = { format = "%Y-%m"; retain = 12; }; - yearly = { format = "%Y"; }; - }; - }; - }; } diff --git a/tv/2configs/consul-client.nix b/tv/2configs/consul-client.nix deleted file mode 100644 index 0a8bf4d75..000000000 --- a/tv/2configs/consul-client.nix +++ /dev/null @@ -1,9 +0,0 @@ -{ pkgs, ... }: - -{ - imports = [ ./consul-server.nix ]; - - tv.consul = { - server = pkgs.lib.mkForce false; - }; -} diff --git a/tv/2configs/consul-server.nix b/tv/2configs/consul-server.nix deleted file mode 100644 index d10f9ea75..000000000 --- a/tv/2configs/consul-server.nix +++ /dev/null @@ -1,21 +0,0 @@ -{ config, ... }: - -{ - tv.consul = rec { - enable = true; - - self = config.krebs.build.host; - inherit (self) dc; - - server = true; - - hosts = with config.krebs.hosts; [ - # TODO get this list automatically from each host where tv.consul.enable is true - cd - mkdir - nomic - rmdir - #wu - ]; - }; -} diff --git a/tv/2configs/default.nix b/tv/2configs/default.nix index c300633bb..57c4620c4 100644 --- a/tv/2configs/default.nix +++ b/tv/2configs/default.nix @@ -8,20 +8,21 @@ with lib; krebs.build = { user = config.krebs.users.tv; target = mkDefault "root@${config.krebs.build.host.name}"; - source = { - git.nixpkgs = { - url = mkDefault https://github.com/NixOS/nixpkgs; - rev = mkDefault "c44a593aa43bba6a0708f6f36065a514a5110613"; - target-path = mkDefault "/var/src/nixpkgs"; + source = mapAttrs (_: mkDefault) ({ + nixos-config = "symlink:stockholm/tv/1systems/${config.krebs.build.host.name}.nix"; + nixpkgs = symlink:stockholm/nixpkgs; + secrets = "/home/tv/secrets/${config.krebs.build.host.name}"; + secrets-common = "/home/tv/secrets/common"; + stockholm = "/home/tv/stockholm"; + stockholm-user = "symlink:stockholm/tv"; + upstream-nixpkgs = { + url = https://github.com/NixOS/nixpkgs; + rev = "77f8f35d57618c1ba456d968524f2fb2c3448295"; + dev = "/home/tv/nixpkgs"; }; - dir.secrets = { - path = mkDefault "/home/tv/secrets/${config.krebs.build.host.name}"; - }; - dir.stockholm = { - path = mkDefault "/home/tv/stockholm"; - target-path = mkDefault "/var/src/stockholm"; - }; - }; + } // optionalAttrs config.krebs.build.host.secure { + secrets-master = "/home/tv/secrets/master"; + }); }; networking.hostName = config.krebs.build.host.name; @@ -67,6 +68,9 @@ with lib; nix.useChroot = true; } { + nixpkgs.config.allowUnfree = false; + } + { environment.profileRelativeEnvVars.PATH = mkForce [ "/bin" ]; environment.systemPackages = with pkgs; [ @@ -98,12 +102,7 @@ with lib; }; environment.variables = { - NIX_PATH = - with config.krebs.build.source; with dir; with git; - mkForce (concatStringsSep ":" [ - "nixpkgs=${nixpkgs.target-path}" - "secrets=${stockholm.target-path}/null" - ]); + NIX_PATH = mkForce "secrets=/var/src/stockholm/null:/var/src"; }; programs.bash = { @@ -142,7 +141,12 @@ with lib; ''; }; - programs.ssh.startAgent = false; + programs.ssh = { + extraConfig = '' + UseRoaming no + ''; + startAgent = false; + }; } { @@ -160,12 +164,17 @@ with lib; } { + tv.iptables.enable = true; + } + + { services.openssh = { enable = true; hostKeys = [ { type = "ed25519"; path = "/etc/ssh/ssh_host_ed25519_key"; } ]; }; + tv.iptables.input-internet-accept-new-tcp = singleton "ssh"; } { @@ -177,7 +186,8 @@ with lib; { systemd.tmpfiles.rules = let forUsers = flip map users; - isUser = { group, ... }: hasSuffix "users" group; + isUser = { name, group, ... }: + name == "root" || hasSuffix "users" group; users = filter isUser (mapAttrsToList (_: id) config.users.users); in forUsers (u: "d /run/xdg/${u.name} 0700 ${u.name} ${u.group} -"); environment.variables.XDG_RUNTIME_DIR = "/run/xdg/$LOGNAME"; diff --git a/tv/2configs/exim-retiolum.nix b/tv/2configs/exim-retiolum.nix new file mode 100644 index 000000000..aedf25823 --- /dev/null +++ b/tv/2configs/exim-retiolum.nix @@ -0,0 +1,8 @@ +{ lib, ... }: + +with lib; + +{ + krebs.exim-retiolum.enable = true; + tv.iptables.input-retiolum-accept-new-tcp = singleton "smtp"; +} diff --git a/tv/2configs/exim-smarthost.nix b/tv/2configs/exim-smarthost.nix index f5f63d284..bcfea7821 100644 --- a/tv/2configs/exim-smarthost.nix +++ b/tv/2configs/exim-smarthost.nix @@ -1,4 +1,6 @@ -{ config, pkgs, ... }: +{ config, lib, pkgs, ... }: + +with lib; { krebs.exim-smarthost = { @@ -34,4 +36,5 @@ { from = "mirko"; to = "mv"; } ]; }; + tv.iptables.input-internet-accept-new-tcp = singleton "smtp"; } diff --git a/tv/2configs/git.nix b/tv/2configs/git.nix index 800deff1b..b8180085f 100644 --- a/tv/2configs/git.nix +++ b/tv/2configs/git.nix @@ -9,7 +9,7 @@ let enable = true; root-title = "public repositories at ${config.krebs.build.host.name}"; root-desc = "keep calm and engage"; - repos = mapAttrs (_: s: removeAttrs s ["collaborators"]) repos; + repos = repos; rules = rules; }; }; @@ -22,8 +22,8 @@ let public-repos = mapAttrs make-public-repo ({ } // mapAttrValues (setAttr "section" "1. Miscellaneous") { - cac = { - desc = "CloudAtCost command line interface"; + cac-api = { + desc = "CloudAtCost API command line interface"; }; get = {}; hack = {}; @@ -39,6 +39,7 @@ let stockholm = { desc = "take all the computers hostage, they'll love you!"; }; + with-tmpdir = {}; } // mapAttrValues (setAttr "section" "2. Haskell libraries") { blessings = {}; mime = {}; @@ -98,7 +99,7 @@ let repo = [ repo ]; perm = fetch; } ++ - optional (length (repo.collaborators or []) > 0) { + optional (repo.collaborators or [] != []) { user = repo.collaborators; repo = [ repo ]; perm = fetch; diff --git a/tv/2configs/hw/AO753.nix b/tv/2configs/hw/AO753.nix index acd9ee32b..72a40819f 100644 --- a/tv/2configs/hw/AO753.nix +++ b/tv/2configs/hw/AO753.nix @@ -1,4 +1,6 @@ -{ config, pkgs, ... }: +{ config, lib, pkgs, ... }: + +with lib; { imports = [ @@ -39,8 +41,5 @@ HandleSuspendKey=ignore ''; - nixpkgs.config = { - allowUnfree = false; - allowUnfreePredicate = (x: pkgs.lib.hasPrefix "broadcom-sta-" x.name); - }; + nixpkgs.config.allowUnfreePredicate = pkg: hasPrefix "broadcom-sta-" pkg.name; } diff --git a/tv/2configs/hw/x220.nix b/tv/2configs/hw/x220.nix index 8549311e7..7cec670fa 100644 --- a/tv/2configs/hw/x220.nix +++ b/tv/2configs/hw/x220.nix @@ -14,7 +14,6 @@ networking.wireless.enable = true; #hardware.enableAllFirmware = true; - #nixpkgs.config.allowUnfree = true; #zramSwap.enable = true; #zramSwap.numDevices = 2; diff --git a/tv/2configs/nginx-public_html.nix b/tv/2configs/nginx-public_html.nix new file mode 100644 index 000000000..dc74f7f8d --- /dev/null +++ b/tv/2configs/nginx-public_html.nix @@ -0,0 +1,15 @@ +{ lib, ... }: + +with lib; + +{ + krebs.nginx = { + enable = true; + servers.default.locations = [ + (nameValuePair "~ ^/~(.+?)(/.*)?\$" '' + alias /home/$1/public_html$2; + '') + ]; + }; + tv.iptables.input-internet-accept-new-tcp = singleton "http"; +} diff --git a/tv/2configs/pulse.nix b/tv/2configs/pulse.nix index 3db3532d5..e1894ca64 100644 --- a/tv/2configs/pulse.nix +++ b/tv/2configs/pulse.nix @@ -1,5 +1,6 @@ { config, lib, pkgs, ... }: +with lib; let pkg = pkgs.pulseaudioLight; runDir = "/run/pulse"; @@ -35,36 +36,43 @@ let in { - systemd.tmpfiles.rules = [ - "d ${runDir} 0750 pulse pulse - -" - "d ${runDir}/home 0700 pulse pulse - -" - ]; - - system.activationScripts.pulseaudio-hack = '' - ln -fns ${clientConf} /etc/pulse/client.conf - ''; - environment = { etc = { "asound.conf".source = alsaConf; - #"pulse/client.conf" = lib.mkForce { source = clientConf; }; + # XXX mkForce is not strong enough (and neither is mkOverride) to create + # /etc/pulse/client.conf, see pulseaudio-hack below for a solution. + #"pulse/client.conf" = mkForce { source = clientConf; }; + #"pulse/client.conf".source = mkForce clientConf; "pulse/default.pa".source = configFile; }; - systemPackages = [ pkg ]; + systemPackages = [ + pkg + ] ++ optionals config.services.xserver.enable [ + pkgs.pavucontrol + ]; }; # Allow PulseAudio to get realtime priority using rtkit. security.rtkit.enable = true; + system.activationScripts.pulseaudio-hack = '' + ln -fns ${clientConf} /etc/pulse/client.conf + ''; + systemd.services.pulse = { wantedBy = [ "sound.target" ]; before = [ "sound.target" ]; environment = { PULSE_RUNTIME_PATH = "${runDir}/home"; - #DISPLAY = ":${toString config.services.xserver.display}"; }; serviceConfig = { ExecStart = "${pkg}/bin/pulseaudio"; + ExecStartPre = pkgs.writeScript "pulse-start" '' + #! /bin/sh + install -o pulse -g pulse -m 0750 -d ${runDir} + install -o pulse -g pulse -m 0700 -d ${runDir}/home + ''; + PermissionsStartOnly = "true"; User = "pulse"; }; }; diff --git a/tv/2configs/retiolum.nix b/tv/2configs/retiolum.nix new file mode 100644 index 000000000..d2bb9e6cf --- /dev/null +++ b/tv/2configs/retiolum.nix @@ -0,0 +1,17 @@ +{ config, lib, ... }: + +with lib; + +{ + krebs.retiolum = { + enable = true; + connectTo = filter (ne config.krebs.build.host.name) [ + "gum" + "prism" + "echelon" + "cd" + "ire" + ]; + }; + tv.iptables.input-internet-accept-new-tcp = singleton "tinc"; +} diff --git a/tv/2configs/vim.nix b/tv/2configs/vim.nix index 0537fa7d8..23f90af05 100644 --- a/tv/2configs/vim.nix +++ b/tv/2configs/vim.nix @@ -7,11 +7,6 @@ let vim ]; - # Nano really is just a stupid name for Vim. - nixpkgs.config.packageOverrides = pkgs: { - nano = pkgs.vim; - }; - environment.etc.vimrc.source = vimrc; environment.variables.EDITOR = mkForce "vim"; @@ -89,7 +84,7 @@ let \ | hi Normal ctermfg=White au BufRead,BufNewFile *.hs so ${pkgs.writeText "hs.vim" '' - syn region String start=+\[[^|]*|+ end=+|]+ + syn region String start=+\[[[:alnum:]]*|+ end=+|]+ ''} au BufRead,BufNewFile *.nix so ${pkgs.writeText "nix.vim" '' @@ -110,6 +105,8 @@ let syn match String /"\([^\\"]\|\\.\)*"/ syn match Comment /\(^\|\s\)#.*/ + + let b:current_syntax = "nix" ''} au BufRead,BufNewFile /dev/shm/* set nobackup nowritebackup noswapfile diff --git a/tv/2configs/xserver/default.nix b/tv/2configs/xserver/default.nix index facde4e76..5cd17aa1d 100644 --- a/tv/2configs/xserver/default.nix +++ b/tv/2configs/xserver/default.nix @@ -37,7 +37,6 @@ let pkgs.ff pkgs.gitAndTools.qgit pkgs.mpv - pkgs.pavucontrol pkgs.slock pkgs.sxiv pkgs.xsel diff --git a/tv/3modules/consul.nix b/tv/3modules/consul.nix deleted file mode 100644 index 5c955fdb5..000000000 --- a/tv/3modules/consul.nix +++ /dev/null @@ -1,118 +0,0 @@ -{ config, lib, pkgs, ... }: - -# if quorum gets lost, then start any node with a config that doesn't contain bootstrap_expect -# but -bootstrap -# TODO consul-bootstrap HOST that actually does is -# TODO tools to inspect state of a cluster in outage state - -with lib; -let - cfg = config.tv.consul; - - out = { - options.tv.consul = api; - config = mkIf cfg.enable (mkMerge [ - imp - { tv.iptables.input-retiolum-accept-new-tcp = [ "8300" "8301" ]; } - # TODO udp for 8301 - ]); - }; - - api = { - enable = mkEnableOption "tv.consul"; - - dc = mkOption { - type = types.label; - }; - hosts = mkOption { - type = with types; listOf host; - }; - encrypt-file = mkOption { - type = types.str; # TODO path (but not just into store) - default = toString <secrets/consul-encrypt.json>; - }; - data-dir = mkOption { - type = types.str; # TODO path (but not just into store) - default = "/var/lib/consul"; - }; - self = mkOption { - type = types.host; - }; - server = mkOption { - type = types.bool; - default = false; - }; - GOMAXPROCS = mkOption { - type = types.int; - default = cfg.self.cores; - }; - }; - - consul-config = { - datacenter = cfg.dc; - data_dir = cfg.data-dir; - log_level = "INFO"; - #node_name = - server = cfg.server; - enable_syslog = true; - retry_join = - # TODO allow consul in other nets than retiolum [maybe] - concatMap (host: host.nets.retiolum.addrs) - (filter (host: host.name != cfg.self.name) cfg.hosts); - leave_on_terminate = true; - } // optionalAttrs cfg.server { - bootstrap_expect = length cfg.hosts; - leave_on_terminate = false; - }; - - imp = { - environment.systemPackages = with pkgs; [ - consul - ]; - - systemd.services.consul = { - after = [ "network.target" ]; - wantedBy = [ "multi-user.target" ]; - path = with pkgs; [ - consul - ]; - environment = { - GOMAXPROCS = toString cfg.GOMAXPROCS; - }; - serviceConfig = { - PermissionsStartOnly = "true"; - SyslogIdentifier = "consul"; - User = user.name; - PrivateTmp = "true"; - Restart = "always"; - ExecStartPre = pkgs.writeScript "consul-init" '' - #! /bin/sh - mkdir -p ${cfg.data-dir} - chown ${user.name}: ${cfg.data-dir} - install -o ${user.name} -m 0400 ${cfg.encrypt-file} /tmp/encrypt.json - ''; - ExecStart = pkgs.writeScript "consul-service" '' - #! /bin/sh - set -euf - exec >/dev/null - exec consul agent \ - -config-file=${toFile "consul.json" (toJSON consul-config)} \ - -config-file=/tmp/encrypt.json - ''; - #-node=${cfg.self.fqdn} \ - #ExecStart = "${tinc}/sbin/tincd -c ${confDir} -d 0 -U ${user} -D"; - }; - }; - - users.extraUsers = singleton { - inherit (user) name uid; - }; - }; - - user = rec { - name = "consul"; - uid = genid name; - }; - -in -out diff --git a/tv/3modules/default.nix b/tv/3modules/default.nix index bb10d8261..f7889b245 100644 --- a/tv/3modules/default.nix +++ b/tv/3modules/default.nix @@ -2,7 +2,6 @@ _: { imports = [ - ./consul.nix ./ejabberd.nix ./iptables.nix ]; diff --git a/tv/5pkgs/default.nix b/tv/5pkgs/default.nix index 2108d972c..be10e91eb 100644 --- a/tv/5pkgs/default.nix +++ b/tv/5pkgs/default.nix @@ -11,6 +11,9 @@ --disk-cache-size=50000000 \ "%@" ''; + ejabberd = pkgs.callPackage ./ejabberd { + erlang = pkgs.erlangR16; + }; ff = pkgs.callPackage ./ff {}; viljetic-pages = pkgs.callPackage ./viljetic-pages {}; xmonad-tv = diff --git a/tv/5pkgs/ejabberd/default.nix b/tv/5pkgs/ejabberd/default.nix new file mode 100644 index 000000000..3a77c5cd1 --- /dev/null +++ b/tv/5pkgs/ejabberd/default.nix @@ -0,0 +1,28 @@ +{stdenv, fetchurl, expat, erlang, zlib, openssl, pam, lib}: + +stdenv.mkDerivation rec { + version = "2.1.13"; + name = "ejabberd-${version}"; + src = fetchurl { + url = "http://www.process-one.net/downloads/ejabberd/${version}/${name}.tgz"; + sha256 = "0vf8mfrx7vr3c5h3nfp3qcgwf2kmzq20rjv1h9sk3nimwir1q3d8"; + }; + buildInputs = [ expat erlang zlib openssl pam ]; + patchPhase = '' + sed -i \ + -e "s|erl \\\|${erlang}/bin/erl \\\|" \ + -e 's|EXEC_CMD=\"sh -c\"|EXEC_CMD=\"${stdenv.shell} -c\"|' \ + src/ejabberdctl.template + ''; + preConfigure = '' + cd src + ''; + configureFlags = ["--enable-pam"]; + + meta = { + description = "Open-source XMPP application server written in Erlang"; + license = stdenv.lib.licenses.gpl2; + homepage = http://www.ejabberd.im; + maintainers = [ lib.maintainers.sander ]; + }; +} |