From d4170004e7e3278830a659eb518a3e2f4a961685 Mon Sep 17 00:00:00 2001
From: Oskar Kapala <oskar.kapala@gmail.com>
Date: Tue, 30 Jun 2026 16:42:52 +0200
Subject: [PATCH] feat(fleet-prometheus): add liveness alert rules for
 always-on nodes (vps, piha)

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 services/fleet-prometheus/docker-compose.yml |  1 +
 services/fleet-prometheus/prometheus.yml     |  7 ++---
 services/fleet-prometheus/rules/liveness.yml | 28 ++++++++++++++++++++
 3 files changed, 33 insertions(+), 3 deletions(-)
 create mode 100644 services/fleet-prometheus/rules/liveness.yml

diff --git a/services/fleet-prometheus/docker-compose.yml b/services/fleet-prometheus/docker-compose.yml
index 3c5a138..04e3e4e 100644
--- a/services/fleet-prometheus/docker-compose.yml
+++ b/services/fleet-prometheus/docker-compose.yml
@@ -18,6 +18,7 @@ services:
       - '--web.enable-lifecycle'
     volumes:
       - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./rules:/etc/prometheus/rules:ro
       - fleet_prometheus_tsdb:/prometheus
     ports:
       # tailscale-internal: the listen socket is bound ONLY to the VPS Tailscale
diff --git a/services/fleet-prometheus/prometheus.yml b/services/fleet-prometheus/prometheus.yml
index 378d1bc..55d83e7 100644
--- a/services/fleet-prometheus/prometheus.yml
+++ b/services/fleet-prometheus/prometheus.yml
@@ -15,9 +15,10 @@ global:
 #   # Alertmanager wiring is added in a separate step (fleet liveness alerting).
 #   alertmanagers: []
 
-# rule_files:
-#   # Liveness rules (e.g. up == 0 for: 5m) are added in a separate step.
-#   - "rules/*.yml"
+# Liveness rules. Path is relative to /etc/prometheus/ → /etc/prometheus/rules/*.yml,
+# consistent with the ./rules:/etc/prometheus/rules:ro mount in docker-compose.yml.
+rule_files:
+  - "rules/*.yml"
 
 scrape_configs:
   # Self-monitoring — Prometheus scrapes its own /metrics. Bridged container
diff --git a/services/fleet-prometheus/rules/liveness.yml b/services/fleet-prometheus/rules/liveness.yml
new file mode 100644
index 0000000..e8fbbf1
--- /dev/null
+++ b/services/fleet-prometheus/rules/liveness.yml
@@ -0,0 +1,28 @@
+# fleet-prometheus liveness rules.
+#
+# No Alertmanager is wired up by design. These rules only make alerts FIRING,
+# visible at GET /api/v1/alerts on this Prometheus instance. Delivery to
+# Telegram is a SEPARATE future task: brain-watchdog (on PIHA) polls that API
+# over the VPS Tailscale IP and forwards. Do NOT add alerting/Alertmanager
+# config here.
+#
+# SCOPE — alert ONLY for always-on infra nodes: vps, piha. Their up==0 is a
+# real fault.
+#
+# Deliberately EXCLUDED: solaria, lustro. They are intermittent (planned
+# power-off at times), so up==0 is not always an incident. Their liveness will
+# be handled LATER by anomaly detection (metric-history based, separate
+# backlog project), never by a static time-based rule. They stay scraped and
+# visible in prometheus.yml — just not alerted here.
+
+groups:
+  - name: fleet-liveness
+    rules:
+      - alert: NodeDown
+        expr: up{node=~"vps|piha"} == 0
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Node {{ $labels.node }} down"
+          description: "Node {{ $labels.node }} down — up==0 ponad 5m (job {{ $labels.job }}, instance {{ $labels.instance }})."