diff --git a/services/brain-watchdog/env.example b/services/brain-watchdog/env.example index f09c999..fc311db 100644 --- a/services/brain-watchdog/env.example +++ b/services/brain-watchdog/env.example @@ -5,3 +5,7 @@ FAILS_BEFORE_ALERT=3 TG_TOKEN= TG_CHAT_ID= HEALTHCHECKS_URL= +# Optional: Prometheus HTTP URL for polling /api/v1/alerts and alerting on firing rules. +# Target: Tailscale IP of Prometheus node, e.g. http://100.95.58.48:9090 +# Leave empty to disable Prometheus polling (watchdog works without it). +PROMETHEUS_URL= diff --git a/services/brain-watchdog/service.yaml b/services/brain-watchdog/service.yaml index 0eec949..59744f2 100644 --- a/services/brain-watchdog/service.yaml +++ b/services/brain-watchdog/service.yaml @@ -32,3 +32,4 @@ service: - TG_TOKEN # Telegram Bot API token (required) - TG_CHAT_ID # Telegram chat/user ID (required) - HEALTHCHECKS_URL # optional healthchecks.io ping URL + - PROMETHEUS_URL # optional Prometheus URL for polling firing alerts diff --git a/services/brain-watchdog/src/brain_watchdog/main.py b/services/brain-watchdog/src/brain_watchdog/main.py index 96ed3b7..951d782 100644 --- a/services/brain-watchdog/src/brain_watchdog/main.py +++ b/services/brain-watchdog/src/brain_watchdog/main.py @@ -21,6 +21,7 @@ FAILS_BEFORE_ALERT = int(os.environ.get("FAILS_BEFORE_ALERT", "3")) TG_TOKEN = os.environ["TG_TOKEN"] TG_CHAT_ID = os.environ["TG_CHAT_ID"] HEALTHCHECKS_URL = os.environ.get("HEALTHCHECKS_URL", "").strip() +PROMETHEUS_URL = os.environ.get("PROMETHEUS_URL", "").strip() STATE_FILE = Path("/data/state.json") @@ -31,7 +32,7 @@ def load_state() -> dict: return json.loads(STATE_FILE.read_text()) except Exception: pass - return {"fail_count": 0, "alerted": False, "last_ok": 0.0} + return {"fail_count": 0, "alerted": False, "last_ok": 0.0, "prom_alerted": {}} def save_state(state: dict) -> None: @@ -74,6 +75,69 @@ def ping_healthchecks() -> None: print(f"[healthchecks] ping failed: {exc}", flush=True) +def check_prometheus_alerts() -> list[dict]: + if not PROMETHEUS_URL: + return [] + status, body = http_get(f"{PROMETHEUS_URL}/api/v1/alerts") + if status is None or status != 200 or not body: + print(f"[prometheus] poll failed (status={status})", flush=True) + return [] + try: + alerts = body.get("data", {}).get("alerts", []) + firing = [] + for alert in alerts: + if alert.get("state") != "firing": + continue + labels = alert.get("labels", {}) + annotations = alert.get("annotations", {}) + alertname = labels.get("alertname", "unknown") + node = labels.get("node", labels.get("instance", "unknown")) + firing.append({ + "key": f"{alertname}:{node}", + "alertname": alertname, + "node": node, + "summary": annotations.get("summary", ""), + "description": annotations.get("description", ""), + }) + return firing + except Exception as exc: + print(f"[prometheus] parse error: {exc}", flush=True) + return [] + + +def handle_prometheus_alerts(state: dict) -> None: + firing = check_prometheus_alerts() + prom_alerted: dict = state.get("prom_alerted", {}) + firing_keys = {a["key"] for a in firing} + + for alert in firing: + key = alert["key"] + if key not in prom_alerted: + msg = ( + f"🚨 Prometheus alert: {alert['alertname']}\n" + f"Node: {alert['node']}\n" + f"Summary: {alert['summary']}" + ) + if alert["description"]: + msg += f"\n{alert['description']}" + sent = send_telegram(msg) + if sent: + prom_alerted[key] = {"alertname": alert["alertname"], "node": alert["node"]} + print(f"[prometheus] sent alert: {key}", flush=True) + + for key in list(prom_alerted.keys()): + if key not in firing_keys: + info = prom_alerted[key] + send_telegram( + f"✅ Prometheus alert resolved: {info['alertname']}\n" + f"Node: {info['node']}" + ) + del prom_alerted[key] + print(f"[prometheus] sent recovery: {key}", flush=True) + + state["prom_alerted"] = prom_alerted + + def check() -> tuple[bool, str]: """Return (ok, human-readable reason). Never reads 'status' field.""" status, body = http_get(f"{CONTROL_PLANE_URL}/summary") @@ -150,6 +214,10 @@ def main() -> None: save_state(state) print("[telegram] sent alert", flush=True) + # --- Prometheus alerts (independent of brain watchdog) --- + handle_prometheus_alerts(state) + save_state(state) + time.sleep(INTERVAL) diff --git a/services/brain-watchdog/tests/test_main.py b/services/brain-watchdog/tests/test_main.py index 5ff588b..f4ba974 100644 --- a/services/brain-watchdog/tests/test_main.py +++ b/services/brain-watchdog/tests/test_main.py @@ -64,3 +64,91 @@ def test_check_fail_unparseable_timestamp(): ok, reason = bwm.check() assert not ok assert "parseable" in reason + + +# ---------- Prometheus alert polling ---------- + +SAMPLE_PROM_RESPONSE = { + "status": "success", + "data": { + "alerts": [ + { + "labels": {"alertname": "NodeDown", "node": "piha"}, + "annotations": {"summary": "Node is down", "description": ""}, + "state": "firing", + "activeAt": "2026-06-30T10:00:00Z", + "value": "0e+00", + }, + { + "labels": {"alertname": "HighLoad", "node": "solaria"}, + "annotations": {"summary": "High CPU", "description": "CPU above 90%"}, + "state": "pending", + "activeAt": "2026-06-30T10:01:00Z", + "value": "0e+00", + }, + ] + }, +} + + +def test_check_prometheus_alerts_disabled(monkeypatch): + monkeypatch.setattr(bwm, "PROMETHEUS_URL", "") + assert bwm.check_prometheus_alerts() == [] + + +def test_check_prometheus_alerts_parses_firing(monkeypatch): + monkeypatch.setattr(bwm, "PROMETHEUS_URL", "http://prom:9090") + with patch.object(bwm, "http_get", return_value=(200, SAMPLE_PROM_RESPONSE)): + result = bwm.check_prometheus_alerts() + assert len(result) == 1 + assert result[0]["alertname"] == "NodeDown" + assert result[0]["node"] == "piha" + assert result[0]["key"] == "NodeDown:piha" + assert result[0]["summary"] == "Node is down" + + +def test_check_prometheus_alerts_unreachable(monkeypatch): + monkeypatch.setattr(bwm, "PROMETHEUS_URL", "http://prom:9090") + with patch.object(bwm, "http_get", return_value=(None, None)): + assert bwm.check_prometheus_alerts() == [] + + +def test_prometheus_debounce_no_duplicate(monkeypatch): + monkeypatch.setattr(bwm, "PROMETHEUS_URL", "http://prom:9090") + firing = [ + { + "key": "NodeDown:piha", + "alertname": "NodeDown", + "node": "piha", + "summary": "down", + "description": "", + } + ] + with patch.object(bwm, "check_prometheus_alerts", return_value=firing): + with patch.object(bwm, "send_telegram", return_value=True) as mock_tg: + state: dict = {} + bwm.handle_prometheus_alerts(state) + assert mock_tg.call_count == 1 + assert "NodeDown:piha" in state["prom_alerted"] + + # Same alert still firing — debounce must suppress second send + bwm.handle_prometheus_alerts(state) + assert mock_tg.call_count == 1 + + +def test_prometheus_debounce_recovery(monkeypatch): + monkeypatch.setattr(bwm, "PROMETHEUS_URL", "http://prom:9090") + state = { + "prom_alerted": { + "NodeDown:piha": {"alertname": "NodeDown", "node": "piha"} + } + } + with patch.object(bwm, "check_prometheus_alerts", return_value=[]): + with patch.object(bwm, "send_telegram", return_value=True) as mock_tg: + bwm.handle_prometheus_alerts(state) + + assert mock_tg.call_count == 1 + call_text = mock_tg.call_args[0][0] + assert "✅" in call_text + assert "NodeDown" in call_text + assert "NodeDown:piha" not in state["prom_alerted"]