From 060a8f28fa1fc648bdf66afb31a5d1efac868837 Mon Sep 17 00:00:00 2001 From: makefu Date: Fri, 28 Jul 2023 22:24:15 +0200 Subject: makefu: move out to own repo, add vacation-note --- makefu/2configs/logging/client.nix | 32 ------- makefu/2configs/logging/filter/dnsmasq.conf | 19 ---- makefu/2configs/logging/patterns/dnsmasq | 15 --- makefu/2configs/logging/server.nix | 140 ---------------------------- 4 files changed, 206 deletions(-) delete mode 100644 makefu/2configs/logging/client.nix delete mode 100644 makefu/2configs/logging/filter/dnsmasq.conf delete mode 100644 makefu/2configs/logging/patterns/dnsmasq delete mode 100644 makefu/2configs/logging/server.nix (limited to 'makefu/2configs/logging') diff --git a/makefu/2configs/logging/client.nix b/makefu/2configs/logging/client.nix deleted file mode 100644 index 04d2de0d0..000000000 --- a/makefu/2configs/logging/client.nix +++ /dev/null @@ -1,32 +0,0 @@ -{pkgs, buil, config, ...}: -let - log-server = config.makefu.log-server; - log-port = 9200; -in { - services.journalbeat = { - enable = true; - # TODO: filter for certain journal fields, not all - extraConfig = '' - journalbeat: - name: logs-${config.krebs.build.host.name} - seek_position: cursor - cursor_seek_fallback: tail - write_cursor_state: true - cursor_flush_period: 5s - clean_field_names: true - convert_to_numbers: false - move_metadata_to_field: journal - default_type: journal - output.elasticsearch: - enabled: true - hosts: ["${log-server}:${builtins.toString log-port}"] - template.enabled: false - #output.console: - # enabled: true - logging.level: info - logging.to_syslog: true - logging.selectors: ["*"] - - ''; - }; -} diff --git a/makefu/2configs/logging/filter/dnsmasq.conf b/makefu/2configs/logging/filter/dnsmasq.conf deleted file mode 100644 index 1570b1c60..000000000 --- a/makefu/2configs/logging/filter/dnsmasq.conf +++ /dev/null @@ -1,19 +0,0 @@ - -if ( [program] == "dnsmasq") { - grok { - patterns_dir => ["${./patterns}"] - match => { - "message" => [ - "^%{logdate:LOGDATE} dnsmasq\[[\d]+\]\: query\[[\w]+\] %{domain:DOMAIN} from %{clientip:CLIENTIP}" - , "^%{logdate:LOGDATE} dnsmasq\[[\d]+\]\: reply %{domain:DOMAIN} is %{ip:IP}" - , "^%{logdate:LOGDATE} dnsmasq\[[\d]+\]\: %{blocklist:BLOCKLIST} %{domain:DOMAIN} is %{ip:IP}" - ] - } - } - date { - match => [ "LOGDATE", "MMM dd HH:mm:ss", "MMM d HH:mm:ss", "ISO8601" ] - } - geoip { - source => "IP" - } -} diff --git a/makefu/2configs/logging/patterns/dnsmasq b/makefu/2configs/logging/patterns/dnsmasq deleted file mode 100644 index c1e700d5e..000000000 --- a/makefu/2configs/logging/patterns/dnsmasq +++ /dev/null @@ -1,15 +0,0 @@ -BLOCKLIST [\/\w\.]+ -DOMAIN [\w\.\-]+ -DNSID \d+ -PORT \d+ -DNSRESPONSE cached|reply|forwarded|query -# TODO: there are some strange responses for certain queries like or ... -IPORWORD %{IP}|[<>\.\/\w>]+ - -# TODO use public suffix list by mozilla -TLD [a-z]{2,63} -# matches CCSLD and TLD together (e.g. co.uk ) -CCSLD_TLD [a-z]+\.uk -# actually after a CCTLD this would be the third level domain ... -PUBLIC_SUFFIX (xn--)?%{FUNCTIONAL_SLD}\.(%{CCSLD_TLD}|%{TLD}) -FUNCTIONAL_SLD [a-z0-9-]{1,63} diff --git a/makefu/2configs/logging/server.nix b/makefu/2configs/logging/server.nix deleted file mode 100644 index f2fccec25..000000000 --- a/makefu/2configs/logging/server.nix +++ /dev/null @@ -1,140 +0,0 @@ -{pkgs, config, ...}: - -let - es-port = 9200; - kibana-port = 5601; - primaryName = "log.${config.krebs.build.host.name}"; - serverAliases = [ "${primaryName}.r" "${primaryName}.lan" ]; -in { - - services.nginx.virtualHosts.${primaryName} = { - inherit serverAliases; - locations."/" = { - proxyPass = "http://localhost:5601/"; - extraConfig = '' - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - ''; - }; - }; - services.elasticsearch = { - enable = true; - port = es-port; - }; - services.kibana = { - enable = true; - port = kibana-port; - }; - - networking.firewall.extraCommands = '' - iptables -A INPUT -i retiolum -p tcp --dport ${toString es-port} -j ACCEPT - iptables -A INPUT -i retiolum -p tcp --dport ${toString kibana-port} -j ACCEPT - ''; - - # send logs directly to elasticsearch - services.journalbeat = { - enable = true; - package = pkgs.journalbeat7; - extraConfig = '' - logging: - to_syslog: true - level: info - metrics.enabled: false - template.enabled: false - output.logstash: - hosts: [ "127.0.0.1:5044" ] - template.enabled: false - index: journalbeat - journalbeat.inputs: - - paths: [] - seek: cursor - ''; - }; - - services.logstash = { - enable = true; - # package = pkgs.logstash5; - # plugins = [ pkgs.logstash-contrib ]; - inputConfig = - '' - syslog { - timezone => "Etc/UTC" - } - beats { - port => 5044 - } - ''; - filterConfig = - '' - # Assume Beats - if [syslog] { - mutate { - add_field => { "program" => "%{[syslog][identifier]}" } - } - } - '' + - '' - if ![program] { - mutate { - add_field => { "program" => "unknown" } - } - } - '' + - '' - if ([program] == "logstash") { - drop {} - } - '' + - '' - if ( [program] == "dnsmasq") { - grok { - patterns_dir => ["${./patterns}"] - match => { - "message" => [ - "^%{DNSID:dnsid} %{IP:client}/%{PORT} %{DNSRESPONSE:dnstype}\[[\w]+\] %{DOMAIN:domain} from %{IP}" - , "^%{DNSID:dnsid} %{IP:client}/%{PORT} %{DNSRESPONSE:dnstype} %{DOMAIN:domain} is %{IPORWORD:resolved_ip}" - , "^%{DNSID:dnsid} %{IP:client}/%{PORT} %{DNSRESPONSE:dnstype} %{DOMAIN:domain} to %{IP:upstream_dns}" - ] - } - } - if [resolved_ip] { - geoip { - source => "resolved_ip" - } - } - mutate { - rename => { "host" => "syslog_host" } - } - # Target is to parse the the first and second significant part of the domain - grok { - patterns_dir => ["${./patterns}"] - match => { "domain" => [ "%{PUBLIC_SUFFIX:dns_suffix}$" ] } - } - if [client] { - mutate { copy => { "client" => "clientip" } } - dns { - reverse => [ "client"] - action => "replace" - hostsfile => [ "/etc/hosts" ] - hit_cache_ttl => 1600 - failed_cache_ttl => 60 - } - } - } - '' + '' - if ( [program] == "proftpd") { - kv { - field_split => " " - } - } - ''; - outputConfig = - '' - #stdout { - # codec => rubydebug - #} - elasticsearch { } - ''; - }; -} -- cgit v1.2.3