feat(brain-watchdog): poll Prometheus /api/v1/alerts as second alert source (Telegram)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

services/brain-watchdog/env.example

@@ -5,3 +5,7 @@ FAILS_BEFORE_ALERT=3
 TG_TOKEN=
 TG_CHAT_ID=
 HEALTHCHECKS_URL=
+# Optional: Prometheus HTTP URL for polling /api/v1/alerts and alerting on firing rules.
+# Target: Tailscale IP of Prometheus node, e.g. http://100.95.58.48:9090
+# Leave empty to disable Prometheus polling (watchdog works without it).
+PROMETHEUS_URL=

services/brain-watchdog/service.yaml

@@ -32,3 +32,4 @@ service:
       - TG_TOKEN              # Telegram Bot API token (required)
       - TG_CHAT_ID            # Telegram chat/user ID (required)
       - HEALTHCHECKS_URL      # optional healthchecks.io ping URL
+      - PROMETHEUS_URL        # optional Prometheus URL for polling firing alerts

services/brain-watchdog/src/brain_watchdog/main.py

@@ -21,6 +21,7 @@ FAILS_BEFORE_ALERT = int(os.environ.get("FAILS_BEFORE_ALERT", "3"))
 TG_TOKEN = os.environ["TG_TOKEN"]
 TG_CHAT_ID = os.environ["TG_CHAT_ID"]
 HEALTHCHECKS_URL = os.environ.get("HEALTHCHECKS_URL", "").strip()
+PROMETHEUS_URL = os.environ.get("PROMETHEUS_URL", "").strip()
 
 STATE_FILE = Path("/data/state.json")
 
@@ -31,7 +32,7 @@ def load_state() -> dict:
             return json.loads(STATE_FILE.read_text())
         except Exception:
             pass
-    return {"fail_count": 0, "alerted": False, "last_ok": 0.0}
+    return {"fail_count": 0, "alerted": False, "last_ok": 0.0, "prom_alerted": {}}
 
 
 def save_state(state: dict) -> None:
@@ -74,6 +75,69 @@ def ping_healthchecks() -> None:
         print(f"[healthchecks] ping failed: {exc}", flush=True)
 
 
+def check_prometheus_alerts() -> list[dict]:
+    if not PROMETHEUS_URL:
+        return []
+    status, body = http_get(f"{PROMETHEUS_URL}/api/v1/alerts")
+    if status is None or status != 200 or not body:
+        print(f"[prometheus] poll failed (status={status})", flush=True)
+        return []
+    try:
+        alerts = body.get("data", {}).get("alerts", [])
+        firing = []
+        for alert in alerts:
+            if alert.get("state") != "firing":
+                continue
+            labels = alert.get("labels", {})
+            annotations = alert.get("annotations", {})
+            alertname = labels.get("alertname", "unknown")
+            node = labels.get("node", labels.get("instance", "unknown"))
+            firing.append({
+                "key": f"{alertname}:{node}",
+                "alertname": alertname,
+                "node": node,
+                "summary": annotations.get("summary", ""),
+                "description": annotations.get("description", ""),
+            })
+        return firing
+    except Exception as exc:
+        print(f"[prometheus] parse error: {exc}", flush=True)
+        return []
+
+
+def handle_prometheus_alerts(state: dict) -> None:
+    firing = check_prometheus_alerts()
+    prom_alerted: dict = state.get("prom_alerted", {})
+    firing_keys = {a["key"] for a in firing}
+
+    for alert in firing:
+        key = alert["key"]
+        if key not in prom_alerted:
+            msg = (
+                f"🚨 <b>Prometheus alert: {alert['alertname']}</b>\n"
+                f"Node: <code>{alert['node']}</code>\n"
+                f"Summary: {alert['summary']}"
+            )
+            if alert["description"]:
+                msg += f"\n{alert['description']}"
+            sent = send_telegram(msg)
+            if sent:
+                prom_alerted[key] = {"alertname": alert["alertname"], "node": alert["node"]}
+                print(f"[prometheus] sent alert: {key}", flush=True)
+
+    for key in list(prom_alerted.keys()):
+        if key not in firing_keys:
+            info = prom_alerted[key]
+            send_telegram(
+                f"✅ <b>Prometheus alert resolved: {info['alertname']}</b>\n"
+                f"Node: <code>{info['node']}</code>"
+            )
+            del prom_alerted[key]
+            print(f"[prometheus] sent recovery: {key}", flush=True)
+
+    state["prom_alerted"] = prom_alerted
+
+
 def check() -> tuple[bool, str]:
     """Return (ok, human-readable reason).  Never reads 'status' field."""
     status, body = http_get(f"{CONTROL_PLANE_URL}/summary")
@@ -150,6 +214,10 @@ def main() -> None:
                     save_state(state)
                     print("[telegram] sent alert", flush=True)
 
+        # --- Prometheus alerts (independent of brain watchdog) ---
+        handle_prometheus_alerts(state)
+        save_state(state)
+
         time.sleep(INTERVAL)
 
 

services/brain-watchdog/tests/test_main.py

@@ -64,3 +64,91 @@ def test_check_fail_unparseable_timestamp():
         ok, reason = bwm.check()
     assert not ok
     assert "parseable" in reason
+
+
+# ---------- Prometheus alert polling ----------
+
+SAMPLE_PROM_RESPONSE = {
+    "status": "success",
+    "data": {
+        "alerts": [
+            {
+                "labels": {"alertname": "NodeDown", "node": "piha"},
+                "annotations": {"summary": "Node is down", "description": ""},
+                "state": "firing",
+                "activeAt": "2026-06-30T10:00:00Z",
+                "value": "0e+00",
+            },
+            {
+                "labels": {"alertname": "HighLoad", "node": "solaria"},
+                "annotations": {"summary": "High CPU", "description": "CPU above 90%"},
+                "state": "pending",
+                "activeAt": "2026-06-30T10:01:00Z",
+                "value": "0e+00",
+            },
+        ]
+    },
+}
+
+
+def test_check_prometheus_alerts_disabled(monkeypatch):
+    monkeypatch.setattr(bwm, "PROMETHEUS_URL", "")
+    assert bwm.check_prometheus_alerts() == []
+
+
+def test_check_prometheus_alerts_parses_firing(monkeypatch):
+    monkeypatch.setattr(bwm, "PROMETHEUS_URL", "http://prom:9090")
+    with patch.object(bwm, "http_get", return_value=(200, SAMPLE_PROM_RESPONSE)):
+        result = bwm.check_prometheus_alerts()
+    assert len(result) == 1
+    assert result[0]["alertname"] == "NodeDown"
+    assert result[0]["node"] == "piha"
+    assert result[0]["key"] == "NodeDown:piha"
+    assert result[0]["summary"] == "Node is down"
+
+
+def test_check_prometheus_alerts_unreachable(monkeypatch):
+    monkeypatch.setattr(bwm, "PROMETHEUS_URL", "http://prom:9090")
+    with patch.object(bwm, "http_get", return_value=(None, None)):
+        assert bwm.check_prometheus_alerts() == []
+
+
+def test_prometheus_debounce_no_duplicate(monkeypatch):
+    monkeypatch.setattr(bwm, "PROMETHEUS_URL", "http://prom:9090")
+    firing = [
+        {
+            "key": "NodeDown:piha",
+            "alertname": "NodeDown",
+            "node": "piha",
+            "summary": "down",
+            "description": "",
+        }
+    ]
+    with patch.object(bwm, "check_prometheus_alerts", return_value=firing):
+        with patch.object(bwm, "send_telegram", return_value=True) as mock_tg:
+            state: dict = {}
+            bwm.handle_prometheus_alerts(state)
+            assert mock_tg.call_count == 1
+            assert "NodeDown:piha" in state["prom_alerted"]
+
+            # Same alert still firing — debounce must suppress second send
+            bwm.handle_prometheus_alerts(state)
+            assert mock_tg.call_count == 1
+
+
+def test_prometheus_debounce_recovery(monkeypatch):
+    monkeypatch.setattr(bwm, "PROMETHEUS_URL", "http://prom:9090")
+    state = {
+        "prom_alerted": {
+            "NodeDown:piha": {"alertname": "NodeDown", "node": "piha"}
+        }
+    }
+    with patch.object(bwm, "check_prometheus_alerts", return_value=[]):
+        with patch.object(bwm, "send_telegram", return_value=True) as mock_tg:
+            bwm.handle_prometheus_alerts(state)
+
+    assert mock_tg.call_count == 1
+    call_text = mock_tg.call_args[0][0]
+    assert "✅" in call_text
+    assert "NodeDown" in call_text
+    assert "NodeDown:piha" not in state["prom_alerted"]