(define-module (tests tribes-system-node) #:use-module (gnu services) #:use-module (gnu services monitoring) #:use-module (gnu services shepherd) #:use-module (guix gexp) #:use-module (guix packages) #:use-module (srfi srfi-1) #:use-module (srfi srfi-13) #:use-module (srfi srfi-64) #:use-module (tests support) #:use-module (tribes plugins sender) #:use-module (tribes services chrony) #:use-module (tribes services haproxy) #:use-module (tribes services lego) #:use-module (tribes services logging) #:use-module (tribes services tribes) #:use-module (tribes services vinyl) #:use-module (tribes services vinyl-exporter) #:use-module (tribes services victoriametrics) #:use-module (tribes system node) #:export (run-tests)) (define node-module (resolve-module '(tribes system node))) (define chrony-module (resolve-module '(tribes services chrony))) (define haproxy-module (resolve-module '(tribes services haproxy))) (define logging-module (resolve-module '(tribes services logging))) (define tribes-service-module (resolve-module '(tribes services tribes))) (define victoriametrics-module (resolve-module '(tribes services victoriametrics))) (define edge-cache-vcl-text (module-ref node-module 'edge-cache-vcl-text)) (define edge-cache-vcl (module-ref node-module 'edge-cache-vcl)) (define edge-services (module-ref node-module 'edge-services)) (define chrony-shepherd-services (module-ref chrony-module 'chrony-shepherd-services)) (define haproxy-config-file (module-ref haproxy-module 'haproxy-config-file)) (define tribes-system-logging-config-text (module-ref logging-module 'tribes-system-logging-config-text)) (define tribes-root-shepherd-services (module-ref tribes-service-module 'tribes-root-shepherd-services)) (define tribes-profile-packages (module-ref tribes-service-module 'tribes-profile-packages)) (define tribes-sender-ffmpeg-package (module-ref tribes-service-module 'tribes-sender-ffmpeg-package)) (define victoriametrics-shepherd-services (module-ref victoriametrics-module 'victoriametrics-shepherd-services)) (define vmagent-shepherd-services (module-ref victoriametrics-module 'vmagent-shepherd-services)) (define (contains? haystack needle) (and (string-contains haystack needle) #t)) (define (object->string value) (call-with-output-string (lambda (port) (write value port)))) (define (run-tests) (test-begin "tribes-system-node") (let ((vcl (edge-cache-vcl-text (tribes-edge-configuration) (tribes-configuration)))) (test-assert "edge cache backend uses short connect timeout" (contains? vcl ".connect_timeout = 1s;")) (test-assert "edge cache backend uses bounded first-byte timeout" (contains? vcl ".first_byte_timeout = 5s;")) (test-assert "edge cache retries only GET/HEAD 5xx backend responses" (contains? vcl "if ((bereq.method == \"GET\" || bereq.method == \"HEAD\") &&\n beresp.status >= 500 && beresp.status <= 599 &&\n bereq.retries < 5)")) (test-assert "edge cache retries only GET/HEAD backend errors" (contains? vcl "sub vcl_backend_error {\n if ((bereq.method == \"GET\" || bereq.method == \"HEAD\") &&\n bereq.retries < 5)")) (test-assert "edge cache does not cache exhausted 5xx responses" (contains? vcl "if (beresp.status >= 500 && beresp.status <= 599) {\n set beresp.uncacheable = true;\n set beresp.ttl = 0s;")) (test-assert "edge cache keeps unsafe methods as pass-through" (contains? vcl "if (req.method != \"GET\" && req.method != \"HEAD\") {\n return (pass);")) (let ((rendered (edge-cache-vcl (tribes-edge-configuration) (tribes-configuration)))) (test-equal "edge cache renders expected VCL file name" "tribes-edge-cache.vcl" (plain-file-name rendered)) (test-equal "edge cache renders expected VCL file content" vcl (plain-file-content rendered)))) (let* ((config (tribes-node-configuration (tribes (tribes-configuration (host "example.invalid"))))) (services (tribes-node-services config)) (chrony-service (find (lambda (service) (eq? (service-kind service) chrony-service-type)) services)) (node-exporter-service (find (lambda (service) (eq? (service-kind service) prometheus-node-exporter-service-type)) services)) (network-tools-service (find (lambda (service) (let ((value (service-value service))) (and (list? value) (any (lambda (package) (string=? (package-name package) "nftables")) (filter package? value))))) services))) (test-assert "node profile includes nftables for sync partition tests" network-tools-service) (test-assert "node includes Chrony service" chrony-service) (test-assert "node includes Prometheus node exporter service" node-exporter-service) (test-equal "node exporter binds to loopback by default" "127.0.0.1:9100" (prometheus-node-exporter-web-listen-address (service-value node-exporter-service))) (test-assert "node includes VictoriaMetrics storage service" (find (lambda (service) (eq? (service-kind service) victoriametrics-service-type)) services)) (test-assert "node includes vmagent scrape service" (find (lambda (service) (eq? (service-kind service) vmagent-service-type)) services))) (let ((logging-config (tribes-system-logging-configuration))) (test-equal "syslog-ng combined JSONL path is the importer default" "/var/log/tribes-combined.jsonl" (tribes-system-logging-configuration-jsonl-path logging-config)) (let ((text (tribes-system-logging-config-text logging-config))) (test-assert "syslog-ng listens on /dev/log for syslogd compatibility" (contains? text "unix-dgram(\"/dev/log\")")) (test-assert "syslog-ng writes conventional messages log" (contains? text "file(\"/var/log/messages\" template(t_plain))")) (test-assert "syslog-ng writes Tribes combined JSONL" (contains? text "file(\"/var/log/tribes-combined.jsonl\" template(t_json) group(\"tribes\") perm(0640))")) (test-assert "syslog-ng makes combined JSONL readable by the Tribes importer" (contains? text "destination d_combined_jsonl { file(\"/var/log/tribes-combined.jsonl\" template(t_json) group(\"tribes\") perm(0640)); };")) (test-assert "syslog-ng normalizes Shepherd messages copied through kmsg" (and (contains? text "filter f_kmsg_shepherd") (contains? text "set(\"shepherd\", value(\"PROGRAM\"))") (contains? text "set-severity(\"notice\")"))) (test-assert "syslog-ng excludes raw Shepherd kmsg copies from generic system logs" (contains? text "filter(f_not_kmsg_shepherd); destination(d_combined_jsonl)")) (test-assert "syslog-ng tails Tribes service log" (contains? text "file(\"/var/log/tribes.log\" flags(no-parse)")) (test-assert "syslog-ng tails Prometheus node exporter log" (contains? text "file(\"/var/log/prometheus-node-exporter.log\" flags(no-parse) program_override(\"prometheus-node-exporter\")")) (test-assert "syslog-ng writes Chrony native syslog to a per-service log" (contains? text "filter f_native_chronyd { program(\"chronyd\"); };")) (test-assert "syslog-ng does not tail Chrony's native syslog destination" (not (contains? text "file(\"/var/log/chronyd.log\" flags(no-parse)"))))) (let* ((config (chrony-configuration)) (rendered (chrony-config-file config)) (text (plain-file-content rendered)) (services (chrony-shepherd-services config))) (test-equal "Chrony renders expected configuration file name" "chrony.conf" (plain-file-name rendered)) (test-assert "Chrony uses the Guix NTP pool" (contains? text "pool 2.guix.pool.ntp.org iburst")) (test-assert "Chrony allows startup clock steps" (contains? text "makestep 0.1 3")) (test-assert "Chrony stores drift under its state directory" (contains? text "driftfile /var/lib/chrony/chrony.drift")) (test-equal "Chrony renders one shepherd service" 1 (length services)) (test-assert "Chrony shepherd service provides chronyd" (memq 'chronyd (shepherd-service-provision (car services))))) (let ((storage-config (victoriametrics-configuration)) (agent-config (vmagent-configuration)) (scrape-config (default-vmagent-scrape-config-text))) (test-equal "VictoriaMetrics binds to loopback by default" "127.0.0.1:8428" (victoriametrics-configuration-http-listen-address storage-config)) (test-equal "VictoriaMetrics retains node metrics for 90 days" "90d" (victoriametrics-configuration-retention-period storage-config)) (test-equal "vmagent binds to loopback by default" "127.0.0.1:8429" (vmagent-configuration-http-listen-address agent-config)) (test-equal "vmagent remote-writes to local VictoriaMetrics" "http://127.0.0.1:8428/api/v1/write" (vmagent-configuration-remote-write-url agent-config)) (test-assert "default scrape config scrapes node exporter" (contains? scrape-config "targets: [\"127.0.0.1:9100\"]")) (test-assert "default scrape config scrapes VictoriaMetrics" (contains? scrape-config "targets: [\"127.0.0.1:8428\"]")) (test-assert "default scrape config scrapes Tribes metrics" (contains? scrape-config "targets: [\"127.0.0.1:4000\"]")) (test-assert "default scrape config scrapes Vinyl exporter" (contains? scrape-config "targets: [\"127.0.0.1:9131\"]")) (test-equal "VictoriaMetrics renders one shepherd service" 1 (length (victoriametrics-shepherd-services storage-config))) (test-equal "vmagent renders one shepherd service" 1 (length (vmagent-shepherd-services agent-config)))) (let* ((services (tribes-root-shepherd-services (tribes-configuration (host "example.invalid")))) (service-by-provision (lambda (provision) (find (lambda (service) (memq provision (shepherd-service-provision service))) services)))) (for-each (lambda (provision) (test-assert (string-append (symbol->string provision) " waits for explicit post-secret startup") (let ((service (service-by-provision provision))) (and service (not (shepherd-service-auto-start? service)))))) '(tribes tribes-migrations tribes-plugin-rollback-migrations)) (test-assert "tribes-boot-start starts automatically after secrets exist" (let ((service (service-by-provision 'tribes-boot-start))) (and service (shepherd-service-auto-start? service)))) (test-assert "tribes-boot-start uses Shepherd start-service" (let* ((service (service-by-provision 'tribes-boot-start)) (start-sexpr (and service (gexp->approximate-sexp (shepherd-service-start service)))) (start-text (and start-sexpr (object->string start-sexpr)))) (and start-text (contains? start-text "(start-service (lookup-service (quote tribes)))") (not (contains? start-text "perform-service-action"))))) (test-assert "tribes profile includes inotify-tools for file_system" (any (lambda (package) (string=? (package-name package) "inotify-tools")) (tribes-profile-packages (tribes-configuration (host "example.invalid"))))) (test-assert "base tribes config does not force ffmpeg into the launcher" (not (tribes-sender-ffmpeg-package (tribes-configuration (host "example.invalid"))))) (test-assert "sender-enabled tribes config exposes sender ffmpeg for env override" (let ((package (tribes-sender-ffmpeg-package (tribes-configuration (host "example.invalid") (plugins (list (sender-external-plugin))))))) (and package (string=? (package-name package) "sender-ffmpeg"))))) (let* ((config (tribes-node-configuration (tribes (tribes-configuration (host "example.invalid"))) (edge (tribes-edge-configuration (certificate-email "ops@example.invalid"))))) (services (edge-services config)) (vinyl-service (find (lambda (service) (eq? (service-kind service) vinyl-service-type)) services)) (vinyl-exporter-service (find (lambda (service) (eq? (service-kind service) vinyl-exporter-service-type)) services)) (haproxy-service (find (lambda (service) (eq? (service-kind service) haproxy-service-type)) services)) (lego-service (find (lambda (service) (eq? (service-kind service) lego-service-type)) services)) (vinyl-configs (and vinyl-service (service-value vinyl-service))) (vinyl-exporter-config (and vinyl-exporter-service (service-value vinyl-exporter-service))) (lego-config (and lego-service (service-value lego-service))) (edge-vinyl (and vinyl-configs (find (lambda (config) (string=? (vinyl-configuration-name config) "tribes-edge")) vinyl-configs)))) (test-equal "edge uses one vinyl cache process" 1 (and vinyl-configs (length vinyl-configs))) (test-assert "edge includes Vinyl exporter service" vinyl-exporter-service) (test-equal "edge Vinyl exporter uses edge workdir" "/var/vinyl/tribes-edge" (and vinyl-exporter-config (vinyl-exporter-configuration-vinyl-workdir vinyl-exporter-config))) (test-equal "edge Vinyl exporter labels Sender HLS streams by stream id" "/sender/hls/streams/" (and vinyl-exporter-config (vinyl-exporter-configuration-hls-path-prefix vinyl-exporter-config))) (test-equal "edge Vinyl exporter uses one HLS stream label component" 1 (and vinyl-exporter-config (vinyl-exporter-configuration-hls-stream-components vinyl-exporter-config))) (test-equal "edge Vinyl exporter uses vsid identity" '("vsid") (and vinyl-exporter-config (vinyl-exporter-configuration-hls-query-params vinyl-exporter-config))) (test-equal "edge vinyl permits five graceful retries" '((max_retries . 5)) (and edge-vinyl (vinyl-configuration-parameters edge-vinyl))) (test-equal "lego waits for haproxy challenge routing" '(haproxy) (and lego-config (lego-certificate-configuration-requirement (car (lego-configuration-certificates lego-config))))) (test-assert "edge uses haproxy for TLS termination" haproxy-service)) (let* ((config (haproxy-configuration (backend "127.0.0.1:6081") (frontends '("0.0.0.0:443" "[::]:443 v6only")) (http-frontends '("0.0.0.0:80" "[::]:80 v6only")) (acme-backend "127.0.0.1:8080") (pem-files '("/var/lib/lego/tribes/full.pem")))) (rendered (haproxy-config-file config)) (text (plain-file-content rendered))) (test-equal "haproxy renders expected configuration file name" "haproxy.conf" (plain-file-name rendered)) (test-assert "haproxy does not use deprecated master-worker config keyword" (not (contains? text "master-worker\n"))) (test-assert "haproxy does not require OpenSSL QUIC compatibility mode" (not (contains? text "limited-quic"))) (test-assert "haproxy logs to syslog-ng" (contains? text "log /dev/log local0")) (test-assert "haproxy binds configured certificate" (contains? text "bind 0.0.0.0:443 ssl crt /var/lib/lego/tribes/full.pem alpn h2,http/1.1")) (test-assert "haproxy binds QUIC over IPv4" (contains? text "bind quic4@0.0.0.0:443 ssl crt /var/lib/lego/tribes/full.pem alpn h3")) (test-assert "haproxy binds QUIC over IPv6" (contains? text "bind quic6@[::]:443 v6only ssl crt /var/lib/lego/tribes/full.pem alpn h3")) (test-assert "haproxy advertises HTTP/3" (contains? text "http-response set-header alt-svc 'h3=\":443\"; ma=86400'")) (test-assert "haproxy binds public HTTP" (contains? text "bind 0.0.0.0:80")) (test-assert "haproxy routes ACME challenges to lego" (contains? text "use_backend lego_acme if acme_challenge")) (test-assert "haproxy redirects non-ACME HTTP traffic" (contains? text "http-request redirect scheme https code 308 unless acme_challenge")) (test-assert "haproxy forwards ACME requests to lego challenge server" (contains? text "server lego 127.0.0.1:8080")) (test-assert "haproxy forwards to vinyl cache backend" (contains? text "server vinyl 127.0.0.1:6081 check"))) (test-end "tribes-system-node")) (run-tests-when-script "tests/tribes-system-node.scm" run-tests)