From 1d2c058d78f9cb5d4a51dd2ffb1bfd4b09623055 Mon Sep 17 00:00:00 2001 From: lassulus Date: Mon, 30 Jan 2017 22:56:43 +0100 Subject: l 2 monitoring: introduce {client,server}.nix --- lass/2configs/monitoring/client.nix | 32 ++++++++++++++++++++ lass/2configs/monitoring/server.nix | 59 +++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 lass/2configs/monitoring/client.nix create mode 100644 lass/2configs/monitoring/server.nix (limited to 'lass/2configs/monitoring') diff --git a/lass/2configs/monitoring/client.nix b/lass/2configs/monitoring/client.nix new file mode 100644 index 000000000..eebf2f2e9 --- /dev/null +++ b/lass/2configs/monitoring/client.nix @@ -0,0 +1,32 @@ +{pkgs, config, ...}: +with import ; +{ + lass.telegraf = { + enable = true; + outputs = '' + [outputs.influxdb] + urls = ["http://prism:8086"] + database = "all_data" + user_agent = "telegraf" + ''; + inputs = [ + '' + [cpu] + percpu = false + totalcpu = true + drop = ["cpu_time"] + '' + '' + [[inputs.mem]] + '' + '' + [[inputs.ping]] + urls = ["8.8.8.8"] + '' + ]; + }; + systemd.services.telegraf.path = with pkgs; [ + iputils + lm_sensors + ]; +} diff --git a/lass/2configs/monitoring/server.nix b/lass/2configs/monitoring/server.nix new file mode 100644 index 000000000..335820bc7 --- /dev/null +++ b/lass/2configs/monitoring/server.nix @@ -0,0 +1,59 @@ +{pkgs, config, ...}: +with import ; +{ + services.influxdb = { + enable = true; + }; + + services.influxdb.extraConfig = { + meta.hostname = config.krebs.build.host.name; + # meta.logging-enabled = true; + http.bind-address = ":8086"; + admin.bind-address = ":8083"; + monitoring = { + enabled = false; + # write-interval = "24h"; + }; + }; + + lass.kapacitor = + let + echoToIrc = pkgs.writeDash "echo_irc" '' + set -euf + data="$(${pkgs.jq}/bin/jq -r .message)" + export LOGNAME=prism-alarm + ${pkgs.irc-announce}/bin/irc-announce \ + irc.freenode.org 6667 prism-alarm \#krebs-bots "$data" >/dev/null + ''; + in { + enable = true; + alarms = { + test2 = '' + batch + |query(${"'''"} + SELECT mean("usage_user") AS mean + FROM "${config.lass.kapacitor.check_db}"."default"."cpu" + ${"'''"}) + .every(3m) + .period(1m) + .groupBy('host') + |alert() + .crit(lambda: "mean" > 90) + // Whenever we get an alert write it to a file. + .log('/tmp/alerts.log') + .exec('${echoToIrc}') + ''; + }; + }; + + krebs.iptables.tables.filter.INPUT.rules = [ + { predicate = "-p tcp -i retiolum --dport 8086"; target = "ACCEPT"; } + { predicate = "-p tcp -i retiolum --dport 3000"; target = "ACCEPT"; } + ]; + services.grafana = { + enable = true; + addr = "0.0.0.0"; + auth.anonymous.enable = true; + security = import ; # { AdminUser = ""; adminPassword = ""} + }; +} -- cgit v1.2.3