From d2df5375e705e55764b4cacd4ea32dffcb4c6041 Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Fri, 3 Feb 2017 20:45:27 +0100
Subject: [PATCH] m 2 logging: add deadman trigger
---
.../2configs/logging/central-stats-server.nix | 36 +++++++++++++++++++
1 file changed, 36 insertions(+)
diff --git a/makefu/2configs/logging/central-stats-server.nix b/makefu/2configs/logging/central-stats-server.nix
index 69e37e493..d4e73ab79 100644
--- a/makefu/2configs/logging/central-stats-server.nix
+++ b/makefu/2configs/logging/central-stats-server.nix
@@ -6,6 +6,9 @@ let
influx-port = 8086;
grafana-port = 3000; # TODO nginx forward
in {
+ imports = [
+ ../../lass/3modules/kapacitor.nix
+ ];
services.grafana.enable = true;
services.grafana.addr = "0.0.0.0";
@@ -28,6 +31,39 @@ in {
port = collectd-port;
}];
};
+ lass.kapacitor =
+ let
+ echoToIrc = pkgs.writeDash "echo_irc" ''
+ set -euf
+ data="$(${pkgs.jq}/bin/jq -r .message)"
+ export LOGNAME=malarm
+ ${pkgs.irc-announce}/bin/irc-announce \
+ irc.freenode.org 6667 malarm \#krebs-bots "$data" >/dev/null
+ '';
+ in {
+ enable = true;
+ alarms = {
+ cpu_deadman = ''
+ var data = batch
+ |query('''
+ SELECT mean("value") AS mean
+ FROM "collectd_db"."default"."cpu_value"
+ WHERE "type_instance" = 'idle' AND "type" = 'percent' fill(0)
+ ''')
+ .period(10m)
+ .every(1m)
+ .groupBy('host')
+ data |alert()
+ .crit(lambda: "mean" < 50)
+ .stateChangesOnly()
+ .exec('${echoToIrc}')
+ data |deadman(1.0,5m)
+ .stateChangesOnly()
+ .exec('${echoToIrc}')
+ '';
+ };
+
+ };
networking.firewall.extraCommands = ''
iptables -A INPUT -i retiolum -p udp --dport ${toString collectd-port} -j ACCEPT
iptables -A INPUT -i retiolum -p tcp --dport ${toString influx-port} -j ACCEPT