feat(fleet-prometheus): add liveness alert rules for always-on nodes (vps, piha)
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
831bbedffa
commit
d4170004e7
|
|
@ -18,6 +18,7 @@ services:
|
|||
- '--web.enable-lifecycle'
|
||||
volumes:
|
||||
- ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./rules:/etc/prometheus/rules:ro
|
||||
- fleet_prometheus_tsdb:/prometheus
|
||||
ports:
|
||||
# tailscale-internal: the listen socket is bound ONLY to the VPS Tailscale
|
||||
|
|
|
|||
|
|
@ -15,9 +15,10 @@ global:
|
|||
# # Alertmanager wiring is added in a separate step (fleet liveness alerting).
|
||||
# alertmanagers: []
|
||||
|
||||
# rule_files:
|
||||
# # Liveness rules (e.g. up == 0 for: 5m) are added in a separate step.
|
||||
# - "rules/*.yml"
|
||||
# Liveness rules. Path is relative to /etc/prometheus/ → /etc/prometheus/rules/*.yml,
|
||||
# consistent with the ./rules:/etc/prometheus/rules:ro mount in docker-compose.yml.
|
||||
rule_files:
|
||||
- "rules/*.yml"
|
||||
|
||||
scrape_configs:
|
||||
# Self-monitoring — Prometheus scrapes its own /metrics. Bridged container
|
||||
|
|
|
|||
28
services/fleet-prometheus/rules/liveness.yml
Normal file
28
services/fleet-prometheus/rules/liveness.yml
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
# fleet-prometheus liveness rules.
|
||||
#
|
||||
# No Alertmanager is wired up by design. These rules only make alerts FIRING,
|
||||
# visible at GET /api/v1/alerts on this Prometheus instance. Delivery to
|
||||
# Telegram is a SEPARATE future task: brain-watchdog (on PIHA) polls that API
|
||||
# over the VPS Tailscale IP and forwards. Do NOT add alerting/Alertmanager
|
||||
# config here.
|
||||
#
|
||||
# SCOPE — alert ONLY for always-on infra nodes: vps, piha. Their up==0 is a
|
||||
# real fault.
|
||||
#
|
||||
# Deliberately EXCLUDED: solaria, lustro. They are intermittent (planned
|
||||
# power-off at times), so up==0 is not always an incident. Their liveness will
|
||||
# be handled LATER by anomaly detection (metric-history based, separate
|
||||
# backlog project), never by a static time-based rule. They stay scraped and
|
||||
# visible in prometheus.yml — just not alerted here.
|
||||
|
||||
groups:
|
||||
- name: fleet-liveness
|
||||
rules:
|
||||
- alert: NodeDown
|
||||
expr: up{node=~"vps|piha"} == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Node {{ $labels.node }} down"
|
||||
description: "Node {{ $labels.node }} down — up==0 ponad 5m (job {{ $labels.job }}, instance {{ $labels.instance }})."
|
||||
Loading…
Reference in a new issue