39 lines
2.2 KiB
YAML
39 lines
2.2 KiB
YAML
|
# fleet-prometheus kb-ingest rules — module 5 phase 3 step 5
|
||
|
|
# (docs/kb/modules/05-faza3-plan.md §7.2).
|
||
|
|
#
|
||
|
|
# Same delivery convention as liveness.yml: no Alertmanager, these rules only make alerts
|
||
|
|
# FIRING (visible at GET /api/v1/alerts on this Prometheus instance); brain-watchdog@PIHA
|
||
|
|
# polls that API and forwards to Telegram. Do NOT add Alertmanager config here.
|
||
|
|
#
|
||
|
|
# Metrics come from documents-ingest-cyclic's Prometheus textfile-collector output
|
||
|
|
# (jobs/documents-ingest/systemd, `/opt/homelab/state/node-exporter/kb-ingest.prom`),
|
||
|
|
# scraped via node_exporter's textfile collector on PIHA — every series from that target
|
||
|
|
# already carries `node="piha"` (fleet-node job's static_configs label), same as liveness.yml.
|
||
|
|
|
||
|
|
groups:
|
||
|
|
- name: kb-ingest
|
||
|
|
rules:
|
||
|
|
- alert: KbIngestStale
|
||
|
|
# 2 missed daily runs (OnCalendar=*-*-* 03:30) — the timer itself down, or every
|
||
|
|
# run since has hard-failed (Ollama-down alone never blocks this: the wrapper only
|
||
|
|
# skips its two embed stages then, still updating last_success_timestamp).
|
||
|
|
expr: time() - kb_ingest_last_success_timestamp{node="piha"} > 172800
|
||
|
|
for: 5m
|
||
|
|
labels:
|
||
|
|
severity: critical
|
||
|
|
annotations:
|
||
|
|
summary: "kb-ingest stale on {{ $labels.node }}"
|
||
|
|
description: "kb-ingest.timer has not completed a successful run in over 48h on {{ $labels.node }} — check `journalctl -u kb-ingest.service` and the latest /opt/homelab/logs/kb-ingest/run-*.log."
|
||
|
|
|
||
|
|
- alert: KbEmbedBacklogGrowing
|
||
|
|
# SOLARIA/Ollama has been unreachable (or embedding has been failing) for 3 straight
|
||
|
|
# days of ticks — the backlog itself isn't an incident (Ollama sleeps by design,
|
||
|
|
# plan §1.3), sustained non-zero for this long is.
|
||
|
|
expr: kb_ingest_embed_backlog{node="piha"} > 0
|
||
|
|
for: 72h
|
||
|
|
labels:
|
||
|
|
severity: warning
|
||
|
|
annotations:
|
||
|
|
summary: "kb-ingest embed backlog growing on {{ $labels.node }}"
|
||
|
|
description: "kb_ingest_embed_backlog on {{ $labels.node }} has been > 0 for 72h straight — Ollama@SOLARIA may not have come back up in that window, or chunk/summary embedding has been failing on live Ollama."
|