homelab-codex-ws/services/fleet-prometheus/rules/kb-ingest.yml

# fleet-prometheus kb-ingest rules — module 5 phase 3 step 5
# (kb/phases/kb-m5-faza3.md §7.2).
#
# Same delivery convention as liveness.yml: no Alertmanager, these rules only make alerts
# FIRING (visible at GET /api/v1/alerts on this Prometheus instance); brain-watchdog@PIHA
# polls that API and forwards to Telegram. Do NOT add Alertmanager config here.
#
# Metrics come from documents-ingest-cyclic's Prometheus textfile-collector output
# (jobs/documents-ingest/systemd, `/opt/homelab/state/node-exporter/kb-ingest.prom`),
# scraped via node_exporter's textfile collector on PIHA — every series from that target
# already carries `node="piha"` (fleet-node job's static_configs label), same as liveness.yml.

groups:
  - name: kb-ingest
    rules:
      - alert: KbIngestStale
        # 2 missed daily runs (OnCalendar=*-*-* 03:30) — the timer itself down, or every
        # run since has hard-failed (Ollama-down alone never blocks this: the wrapper only
        # skips its two embed stages then, still updating last_success_timestamp).
        expr: time() - kb_ingest_last_success_timestamp{node="piha"} > 172800
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "kb-ingest stale on {{ $labels.node }}"
          description: "kb-ingest.timer has not completed a successful run in over 48h on {{ $labels.node }} — check `journalctl -u kb-ingest.service` and the latest /opt/homelab/logs/kb-ingest/run-*.log."

      - alert: KbEmbedBacklogGrowing
        # SOLARIA/Ollama has been unreachable (or embedding has been failing) for 3 straight
        # days of ticks — the backlog itself isn't an incident (Ollama sleeps by design,
        # plan §1.3), sustained non-zero for this long is.
        expr: kb_ingest_embed_backlog{node="piha"} > 0
        for: 72h
        labels:
          severity: warning
        annotations:
          summary: "kb-ingest embed backlog growing on {{ $labels.node }}"
          description: "kb_ingest_embed_backlog on {{ $labels.node }} has been > 0 for 72h straight — Ollama@SOLARIA may not have come back up in that window, or chunk/summary embedding has been failing on live Ollama."