You've already forked guix-tribes
self
dfc6f3ebb3
Pinned Docker E2E / pinned-docker-e2e (push) Failing after 32m1s
Give /api/admin/* requests a longer Vinyl backend first-byte and between-bytes timeout while keeping the global edge cache defaults strict. This prevents long-running authenticated admin operations such as cluster_full_resync from being cut off by the 5s public edge timeout.\n\nAdd a system-node VCL assertion for the admin timeout override.
366 lines
18 KiB
Scheme
366 lines
18 KiB
Scheme
(define-module (tests tribes-system-node)
|
|
#:use-module (gnu services)
|
|
#:use-module (gnu services monitoring)
|
|
#:use-module (gnu services shepherd)
|
|
#:use-module (guix gexp)
|
|
#:use-module (guix packages)
|
|
#:use-module (srfi srfi-1)
|
|
#:use-module (srfi srfi-13)
|
|
#:use-module (srfi srfi-64)
|
|
#:use-module (tests support)
|
|
#:use-module (tribes plugins sender)
|
|
#:use-module (tribes services chrony)
|
|
#:use-module (tribes services haproxy)
|
|
#:use-module (tribes services lego)
|
|
#:use-module (tribes services logging)
|
|
#:use-module (tribes services tribes)
|
|
#:use-module (tribes services vinyl)
|
|
#:use-module (tribes services vinyl-exporter)
|
|
#:use-module (tribes services victoriametrics)
|
|
#:use-module (tribes system node)
|
|
#:export (run-tests))
|
|
|
|
(define node-module (resolve-module '(tribes system node)))
|
|
(define chrony-module (resolve-module '(tribes services chrony)))
|
|
(define haproxy-module (resolve-module '(tribes services haproxy)))
|
|
(define logging-module (resolve-module '(tribes services logging)))
|
|
(define tribes-service-module (resolve-module '(tribes services tribes)))
|
|
(define victoriametrics-module (resolve-module '(tribes services victoriametrics)))
|
|
(define edge-cache-vcl-text (module-ref node-module 'edge-cache-vcl-text))
|
|
(define edge-cache-vcl (module-ref node-module 'edge-cache-vcl))
|
|
(define edge-services (module-ref node-module 'edge-services))
|
|
(define chrony-shepherd-services
|
|
(module-ref chrony-module 'chrony-shepherd-services))
|
|
(define haproxy-config-file (module-ref haproxy-module 'haproxy-config-file))
|
|
(define tribes-system-logging-config-text
|
|
(module-ref logging-module 'tribes-system-logging-config-text))
|
|
(define tribes-root-shepherd-services
|
|
(module-ref tribes-service-module 'tribes-root-shepherd-services))
|
|
(define tribes-profile-packages
|
|
(module-ref tribes-service-module 'tribes-profile-packages))
|
|
(define tribes-sender-ffmpeg-package
|
|
(module-ref tribes-service-module 'tribes-sender-ffmpeg-package))
|
|
(define victoriametrics-shepherd-services
|
|
(module-ref victoriametrics-module 'victoriametrics-shepherd-services))
|
|
(define vmagent-shepherd-services
|
|
(module-ref victoriametrics-module 'vmagent-shepherd-services))
|
|
|
|
(define (contains? haystack needle)
|
|
(and (string-contains haystack needle) #t))
|
|
|
|
(define (object->string value)
|
|
(call-with-output-string
|
|
(lambda (port)
|
|
(write value port))))
|
|
|
|
(define (run-tests)
|
|
(test-begin "tribes-system-node")
|
|
|
|
(let ((vcl (edge-cache-vcl-text (tribes-edge-configuration)
|
|
(tribes-configuration))))
|
|
(test-assert "edge cache backend uses short connect timeout"
|
|
(contains? vcl ".connect_timeout = 1s;"))
|
|
(test-assert "edge cache backend uses bounded first-byte timeout"
|
|
(contains? vcl ".first_byte_timeout = 5s;"))
|
|
(test-assert "edge cache gives admin API requests longer backend timeouts"
|
|
(contains? vcl
|
|
"sub vcl_backend_fetch {\n if (bereq.url ~ \"^/api/admin/\") {\n set bereq.connect_timeout = 1s;\n set bereq.first_byte_timeout = 35s;\n set bereq.between_bytes_timeout = 35s;"))
|
|
(test-assert "edge cache retries only GET/HEAD 5xx backend responses"
|
|
(contains? vcl
|
|
"if ((bereq.method == \"GET\" || bereq.method == \"HEAD\") &&\n beresp.status >= 500 && beresp.status <= 599 &&\n bereq.retries < 5)"))
|
|
(test-assert "edge cache retries only GET/HEAD backend errors"
|
|
(contains? vcl
|
|
"sub vcl_backend_error {\n if ((bereq.method == \"GET\" || bereq.method == \"HEAD\") &&\n bereq.retries < 5)"))
|
|
(test-assert "edge cache does not cache exhausted 5xx responses"
|
|
(contains? vcl
|
|
"if (beresp.status >= 500 && beresp.status <= 599) {\n set beresp.uncacheable = true;\n set beresp.ttl = 0s;"))
|
|
(test-assert "edge cache keeps unsafe methods as pass-through"
|
|
(contains? vcl
|
|
"if (req.method != \"GET\" && req.method != \"HEAD\") {\n return (pass);"))
|
|
(let ((rendered (edge-cache-vcl (tribes-edge-configuration)
|
|
(tribes-configuration))))
|
|
(test-equal "edge cache renders expected VCL file name"
|
|
"tribes-edge-cache.vcl"
|
|
(plain-file-name rendered))
|
|
(test-equal "edge cache renders expected VCL file content"
|
|
vcl
|
|
(plain-file-content rendered))))
|
|
|
|
(let* ((config (tribes-node-configuration
|
|
(tribes (tribes-configuration
|
|
(host "example.invalid")))))
|
|
(services (tribes-node-services config))
|
|
(chrony-service
|
|
(find (lambda (service)
|
|
(eq? (service-kind service) chrony-service-type))
|
|
services))
|
|
(node-exporter-service
|
|
(find (lambda (service)
|
|
(eq? (service-kind service)
|
|
prometheus-node-exporter-service-type))
|
|
services))
|
|
(network-tools-service
|
|
(find (lambda (service)
|
|
(let ((value (service-value service)))
|
|
(and (list? value)
|
|
(any (lambda (package)
|
|
(string=? (package-name package) "nftables"))
|
|
(filter package? value)))))
|
|
services)))
|
|
(test-assert "node profile includes nftables for sync partition tests"
|
|
network-tools-service)
|
|
(test-assert "node includes Chrony service"
|
|
chrony-service)
|
|
(test-assert "node includes Prometheus node exporter service"
|
|
node-exporter-service)
|
|
(test-equal "node exporter binds to loopback by default"
|
|
"127.0.0.1:9100"
|
|
(prometheus-node-exporter-web-listen-address
|
|
(service-value node-exporter-service)))
|
|
(test-assert "node includes VictoriaMetrics storage service"
|
|
(find (lambda (service)
|
|
(eq? (service-kind service) victoriametrics-service-type))
|
|
services))
|
|
(test-assert "node includes vmagent scrape service"
|
|
(find (lambda (service)
|
|
(eq? (service-kind service) vmagent-service-type))
|
|
services)))
|
|
|
|
(let ((logging-config (tribes-system-logging-configuration)))
|
|
(test-equal "syslog-ng combined JSONL path is the importer default"
|
|
"/var/log/tribes-combined.jsonl"
|
|
(tribes-system-logging-configuration-jsonl-path logging-config))
|
|
(let ((text (tribes-system-logging-config-text logging-config)))
|
|
(test-assert "syslog-ng listens on /dev/log for syslogd compatibility"
|
|
(contains? text "unix-dgram(\"/dev/log\")"))
|
|
(test-assert "syslog-ng writes conventional messages log"
|
|
(contains? text "file(\"/var/log/messages\" template(t_plain))"))
|
|
(test-assert "syslog-ng writes Tribes combined JSONL"
|
|
(contains? text "file(\"/var/log/tribes-combined.jsonl\" template(t_json) group(\"tribes\") perm(0640))"))
|
|
(test-assert "syslog-ng makes combined JSONL readable by the Tribes importer"
|
|
(contains? text "destination d_combined_jsonl { file(\"/var/log/tribes-combined.jsonl\" template(t_json) group(\"tribes\") perm(0640)); };"))
|
|
(test-assert "syslog-ng normalizes Shepherd messages copied through kmsg"
|
|
(and (contains? text "filter f_kmsg_shepherd")
|
|
(contains? text "set(\"shepherd\", value(\"PROGRAM\"))")
|
|
(contains? text "set-severity(\"notice\")")))
|
|
(test-assert "syslog-ng excludes raw Shepherd kmsg copies from generic system logs"
|
|
(contains? text "filter(f_not_kmsg_shepherd); destination(d_combined_jsonl)"))
|
|
(test-assert "syslog-ng tails Tribes service log"
|
|
(contains? text "file(\"/var/log/tribes.log\" flags(no-parse)"))
|
|
(test-assert "syslog-ng tails Prometheus node exporter log"
|
|
(contains? text "file(\"/var/log/prometheus-node-exporter.log\" flags(no-parse) program_override(\"prometheus-node-exporter\")"))
|
|
(test-assert "syslog-ng writes Chrony native syslog to a per-service log"
|
|
(contains? text "filter f_native_chronyd { program(\"chronyd\"); };"))
|
|
(test-assert "syslog-ng does not tail Chrony's native syslog destination"
|
|
(not (contains? text "file(\"/var/log/chronyd.log\" flags(no-parse)")))))
|
|
|
|
(let* ((config (chrony-configuration))
|
|
(rendered (chrony-config-file config))
|
|
(text (plain-file-content rendered))
|
|
(services (chrony-shepherd-services config)))
|
|
(test-equal "Chrony renders expected configuration file name"
|
|
"chrony.conf"
|
|
(plain-file-name rendered))
|
|
(test-assert "Chrony uses the Guix NTP pool"
|
|
(contains? text "pool 2.guix.pool.ntp.org iburst"))
|
|
(test-assert "Chrony allows startup clock steps"
|
|
(contains? text "makestep 0.1 3"))
|
|
(test-assert "Chrony stores drift under its state directory"
|
|
(contains? text "driftfile /var/lib/chrony/chrony.drift"))
|
|
(test-equal "Chrony renders one shepherd service"
|
|
1
|
|
(length services))
|
|
(test-assert "Chrony shepherd service provides chronyd"
|
|
(memq 'chronyd
|
|
(shepherd-service-provision (car services)))))
|
|
|
|
(let ((storage-config (victoriametrics-configuration))
|
|
(agent-config (vmagent-configuration))
|
|
(scrape-config (default-vmagent-scrape-config-text)))
|
|
(test-equal "VictoriaMetrics binds to loopback by default"
|
|
"127.0.0.1:8428"
|
|
(victoriametrics-configuration-http-listen-address storage-config))
|
|
(test-equal "VictoriaMetrics retains node metrics for 90 days"
|
|
"90d"
|
|
(victoriametrics-configuration-retention-period storage-config))
|
|
(test-equal "vmagent binds to loopback by default"
|
|
"127.0.0.1:8429"
|
|
(vmagent-configuration-http-listen-address agent-config))
|
|
(test-equal "vmagent remote-writes to local VictoriaMetrics"
|
|
"http://127.0.0.1:8428/api/v1/write"
|
|
(vmagent-configuration-remote-write-url agent-config))
|
|
(test-assert "default scrape config scrapes node exporter"
|
|
(contains? scrape-config "targets: [\"127.0.0.1:9100\"]"))
|
|
(test-assert "default scrape config scrapes VictoriaMetrics"
|
|
(contains? scrape-config "targets: [\"127.0.0.1:8428\"]"))
|
|
(test-assert "default scrape config scrapes Tribes metrics"
|
|
(contains? scrape-config "targets: [\"127.0.0.1:4000\"]"))
|
|
(test-assert "default scrape config scrapes Vinyl exporter"
|
|
(contains? scrape-config "targets: [\"127.0.0.1:9131\"]"))
|
|
(test-equal "VictoriaMetrics renders one shepherd service"
|
|
1
|
|
(length (victoriametrics-shepherd-services storage-config)))
|
|
(test-equal "vmagent renders one shepherd service"
|
|
1
|
|
(length (vmagent-shepherd-services agent-config))))
|
|
|
|
(let* ((services (tribes-root-shepherd-services
|
|
(tribes-configuration (host "example.invalid"))))
|
|
(service-by-provision
|
|
(lambda (provision)
|
|
(find (lambda (service)
|
|
(memq provision (shepherd-service-provision service)))
|
|
services))))
|
|
(for-each
|
|
(lambda (provision)
|
|
(test-assert
|
|
(string-append (symbol->string provision)
|
|
" waits for explicit post-secret startup")
|
|
(let ((service (service-by-provision provision)))
|
|
(and service
|
|
(not (shepherd-service-auto-start? service))))))
|
|
'(tribes tribes-migrations tribes-plugin-rollback-migrations))
|
|
(test-assert "tribes-boot-start starts automatically after secrets exist"
|
|
(let ((service (service-by-provision 'tribes-boot-start)))
|
|
(and service
|
|
(shepherd-service-auto-start? service))))
|
|
(test-assert "tribes-boot-start uses Shepherd start-service"
|
|
(let* ((service (service-by-provision 'tribes-boot-start))
|
|
(start-sexpr (and service
|
|
(gexp->approximate-sexp
|
|
(shepherd-service-start service))))
|
|
(start-text (and start-sexpr
|
|
(object->string start-sexpr))))
|
|
(and start-text
|
|
(contains? start-text "(start-service (lookup-service (quote tribes)))")
|
|
(not (contains? start-text "perform-service-action")))))
|
|
(test-assert "tribes profile includes inotify-tools for file_system"
|
|
(any (lambda (package)
|
|
(string=? (package-name package) "inotify-tools"))
|
|
(tribes-profile-packages
|
|
(tribes-configuration (host "example.invalid")))))
|
|
(test-assert "base tribes config does not force ffmpeg into the launcher"
|
|
(not (tribes-sender-ffmpeg-package
|
|
(tribes-configuration (host "example.invalid")))))
|
|
(test-assert "sender-enabled tribes config exposes sender ffmpeg for env override"
|
|
(let ((package (tribes-sender-ffmpeg-package
|
|
(tribes-configuration
|
|
(host "example.invalid")
|
|
(plugins (list (sender-external-plugin)))))))
|
|
(and package
|
|
(string=? (package-name package) "sender-ffmpeg")))))
|
|
|
|
(let* ((config (tribes-node-configuration
|
|
(tribes (tribes-configuration
|
|
(host "example.invalid")))
|
|
(edge (tribes-edge-configuration
|
|
(certificate-email "ops@example.invalid")))))
|
|
(services (edge-services config))
|
|
(vinyl-service (find (lambda (service)
|
|
(eq? (service-kind service) vinyl-service-type))
|
|
services))
|
|
(vinyl-exporter-service
|
|
(find (lambda (service)
|
|
(eq? (service-kind service) vinyl-exporter-service-type))
|
|
services))
|
|
(haproxy-service (find (lambda (service)
|
|
(eq? (service-kind service) haproxy-service-type))
|
|
services))
|
|
(lego-service (find (lambda (service)
|
|
(eq? (service-kind service) lego-service-type))
|
|
services))
|
|
(vinyl-configs (and vinyl-service
|
|
(service-value vinyl-service)))
|
|
(vinyl-exporter-config (and vinyl-exporter-service
|
|
(service-value vinyl-exporter-service)))
|
|
(lego-config (and lego-service
|
|
(service-value lego-service)))
|
|
(edge-vinyl (and vinyl-configs
|
|
(find (lambda (config)
|
|
(string=? (vinyl-configuration-name config)
|
|
"tribes-edge"))
|
|
vinyl-configs))))
|
|
(test-equal "edge uses one vinyl cache process"
|
|
1
|
|
(and vinyl-configs
|
|
(length vinyl-configs)))
|
|
(test-assert "edge includes Vinyl exporter service"
|
|
vinyl-exporter-service)
|
|
(test-equal "edge Vinyl exporter uses edge workdir"
|
|
"/var/vinyl/tribes-edge"
|
|
(and vinyl-exporter-config
|
|
(vinyl-exporter-configuration-vinyl-workdir
|
|
vinyl-exporter-config)))
|
|
(test-equal "edge Vinyl exporter labels Sender HLS streams by stream id"
|
|
"/sender/hls/streams/"
|
|
(and vinyl-exporter-config
|
|
(vinyl-exporter-configuration-hls-path-prefix
|
|
vinyl-exporter-config)))
|
|
(test-equal "edge Vinyl exporter uses one HLS stream label component"
|
|
1
|
|
(and vinyl-exporter-config
|
|
(vinyl-exporter-configuration-hls-stream-components
|
|
vinyl-exporter-config)))
|
|
(test-equal "edge Vinyl exporter uses vsid identity"
|
|
'("vsid")
|
|
(and vinyl-exporter-config
|
|
(vinyl-exporter-configuration-hls-query-params
|
|
vinyl-exporter-config)))
|
|
(test-equal "edge vinyl permits five graceful retries"
|
|
'((max_retries . 5))
|
|
(and edge-vinyl
|
|
(vinyl-configuration-parameters edge-vinyl)))
|
|
(test-equal "lego waits for haproxy challenge routing"
|
|
'(haproxy)
|
|
(and lego-config
|
|
(lego-certificate-configuration-requirement
|
|
(car (lego-configuration-certificates lego-config)))))
|
|
(test-assert "edge uses haproxy for TLS termination"
|
|
haproxy-service))
|
|
|
|
(let* ((config (haproxy-configuration
|
|
(backend "127.0.0.1:6081")
|
|
(frontends '("0.0.0.0:443" "[::]:443 v6only"))
|
|
(http-frontends '("0.0.0.0:80" "[::]:80 v6only"))
|
|
(acme-backend "127.0.0.1:8080")
|
|
(pem-files '("/var/lib/lego/tribes/full.pem"))))
|
|
(rendered (haproxy-config-file config))
|
|
(text (plain-file-content rendered)))
|
|
(test-equal "haproxy renders expected configuration file name"
|
|
"haproxy.conf"
|
|
(plain-file-name rendered))
|
|
(test-assert "haproxy does not use deprecated master-worker config keyword"
|
|
(not (contains? text "master-worker\n")))
|
|
(test-assert "haproxy does not require OpenSSL QUIC compatibility mode"
|
|
(not (contains? text "limited-quic")))
|
|
(test-assert "haproxy logs to syslog-ng"
|
|
(contains? text "log /dev/log local0"))
|
|
(test-assert "haproxy preserves privileged bind capability for QUIC"
|
|
(contains? text "setcap cap_net_bind_service"))
|
|
(test-assert "haproxy binds configured certificate"
|
|
(contains? text
|
|
"bind 0.0.0.0:443 ssl crt /var/lib/lego/tribes/full.pem alpn h2,http/1.1"))
|
|
(test-assert "haproxy binds QUIC over IPv4"
|
|
(contains? text
|
|
"bind quic4@0.0.0.0:443 ssl crt /var/lib/lego/tribes/full.pem alpn h3"))
|
|
(test-assert "haproxy binds QUIC over IPv6"
|
|
(contains? text
|
|
"bind quic6@[::]:443 v6only ssl crt /var/lib/lego/tribes/full.pem alpn h3"))
|
|
(test-assert "haproxy advertises HTTP/3"
|
|
(contains? text "http-response set-header alt-svc 'h3=\":443\"; ma=86400'"))
|
|
(test-assert "haproxy binds public HTTP"
|
|
(contains? text "bind 0.0.0.0:80"))
|
|
(test-assert "haproxy routes ACME challenges to lego"
|
|
(contains? text "use_backend lego_acme if acme_challenge"))
|
|
(test-assert "haproxy redirects non-ACME HTTP traffic"
|
|
(contains? text
|
|
"http-request redirect scheme https code 308 unless acme_challenge"))
|
|
(test-assert "haproxy forwards ACME requests to lego challenge server"
|
|
(contains? text "server lego 127.0.0.1:8080"))
|
|
(test-assert "haproxy forwards to vinyl cache backend"
|
|
(contains? text "server vinyl 127.0.0.1:6081 check")))
|
|
|
|
(test-end "tribes-system-node"))
|
|
|
|
(run-tests-when-script "tests/tribes-system-node.scm" run-tests)
|