"""Tests for the observer SHADOW-READ of Prometheus up{} liveness (cutover etap 1). The whole point of etap 1 is a PARALLEL-RUN: the observer additionally queries Prometheus and LOGS disagreements, but authoritative liveness stays 100% event-driven (compute_liveness). These tests prove exactly that: shadow-read never mutates node_info["liveness"]/["status"], is fail-open on any Prometheus error, and logs a grep-able SHADOW_LIVENESS_MISMATCH line on disagreement. """ from __future__ import annotations import logging import sys import time from pathlib import Path import pytest # Observer lives outside the control-plane package; add scripts/ to path. sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent / "scripts")) import observer.observer as obs_mod from observer.observer import Observer from liveness import FRESH, DEAD # --------------------------------------------------------------------------- # Fixtures / helpers # --------------------------------------------------------------------------- @pytest.fixture(autouse=True) def _redirect_observer_paths(tmp_path, monkeypatch): """Redirect every observer runtime path into a per-test tmp_path. Mirrors the fixture in test_incident_lifecycle.py so these tests never touch the real /opt/homelab state. Also defaults PROM_SHADOW_URL to "" (disabled) so a test that forgets to set it exercises the graceful no-op path. """ world = tmp_path / "world" state = tmp_path / "state" events = tmp_path / "events" logs = tmp_path / "logs" repo = tmp_path / "repo" for d in (world, state, events, logs, repo / "inventory", repo / "hosts"): d.mkdir(parents=True, exist_ok=True) (repo / "inventory" / "topology.yaml").write_text( "nodes:\n" " vps:\n roles: [control-plane]\n connectivity: {}\n" " piha:\n roles: [infra]\n connectivity: {}\n" ) monkeypatch.setattr(obs_mod, "WORLD_DIR", world) monkeypatch.setattr(obs_mod, "STATE_DIR", state) monkeypatch.setattr(obs_mod, "EVENTS_DIR", events) monkeypatch.setattr(obs_mod, "LOGS_DIR", logs) monkeypatch.setattr(obs_mod, "INVENTORY_TOPOLOGY", repo / "inventory" / "topology.yaml") monkeypatch.setattr(obs_mod, "REPO_ROOT", repo) monkeypatch.setattr(obs_mod, "FAILED_EVENTS_DIR", state / "observer_failed_events") monkeypatch.setattr(obs_mod, "OBSERVER_STATE_FILE", state / "observer_checkpoint.json") monkeypatch.setattr(obs_mod, "PROM_SHADOW_URL", "") class _FakeResp: """Minimal context-manager stand-in for urllib.request.urlopen()'s return.""" def __init__(self, body: str): self._body = body def __enter__(self): return self def __exit__(self, *exc): return False def read(self): return self._body.encode("utf-8") def _mock_prom(monkeypatch, body): """Make urllib.request.urlopen return `body` (str) or raise (if Exception).""" def _fake_urlopen(url, timeout=None): if isinstance(body, Exception): raise body return _FakeResp(body) monkeypatch.setattr(obs_mod.urllib.request, "urlopen", _fake_urlopen) _SAMPLE_UP_JSON = ( '{"status":"success","data":{"resultType":"vector","result":[' '{"metric":{"__name__":"up","job":"fleet-node","node":"vps"},"value":[1720000000,"1"]},' '{"metric":{"__name__":"up","job":"fleet-node","node":"piha"},"value":[1720000000,"0"]},' '{"metric":{"__name__":"up","job":"prometheus"},"value":[1720000000,"1"]}' ']}}' ) # --------------------------------------------------------------------------- # 1. _query_prometheus_liveness — disabled / parsing / fail-open # --------------------------------------------------------------------------- def test_query_returns_empty_when_url_unset(monkeypatch): """PROM_SHADOW_URL empty → {} (shadow disabled), no HTTP attempted.""" monkeypatch.setattr(obs_mod, "PROM_SHADOW_URL", "") def _boom(*a, **k): # pragma: no cover - must never be called raise AssertionError("urlopen must not be called when shadow is disabled") monkeypatch.setattr(obs_mod.urllib.request, "urlopen", _boom) assert Observer()._query_prometheus_liveness() == {} def test_query_parses_up_series_by_node_label(monkeypatch): """Parses up{} into {node: bool}; series without a node label are ignored.""" monkeypatch.setattr(obs_mod, "PROM_SHADOW_URL", "http://prom:9090") _mock_prom(monkeypatch, _SAMPLE_UP_JSON) result = Observer()._query_prometheus_liveness() assert result == {"vps": True, "piha": False} assert "prometheus" not in result # up{job="prometheus"} has no node label def test_query_fail_open_on_unreachable(monkeypatch): """Prometheus unreachable → {} and NO exception propagates (fail-open).""" monkeypatch.setattr(obs_mod, "PROM_SHADOW_URL", "http://prom:9090") _mock_prom(monkeypatch, ConnectionRefusedError("nope")) assert Observer()._query_prometheus_liveness() == {} def test_query_fail_open_on_bad_json(monkeypatch): """Malformed JSON body → {} and no raise.""" monkeypatch.setattr(obs_mod, "PROM_SHADOW_URL", "http://prom:9090") _mock_prom(monkeypatch, "this is not json") assert Observer()._query_prometheus_liveness() == {} # --------------------------------------------------------------------------- # 2. The load-bearing invariant: shadow-read NEVER changes liveness # --------------------------------------------------------------------------- def _fresh_node_world(obs: Observer): """Seed one node whose event-driven liveness computes to FRESH.""" obs.world_state["nodes"] = { "vps": {"status": "unknown", "last_seen": int(time.time()), "roles": ["control-plane"]}, } def test_shadow_does_not_change_liveness(monkeypatch): """Event liveness after a cycle WITH shadow (mismatching prom) is byte-for-byte identical to a cycle WITHOUT shadow. Proof that shadow switches nothing.""" # Baseline: shadow disabled. baseline = Observer() _fresh_node_world(baseline) baseline._prune_stale_world() base_liveness = baseline.world_state["nodes"]["vps"]["liveness"] base_status = baseline.world_state["nodes"]["vps"]["status"] assert base_liveness == FRESH assert base_status == "online" # Shadow enabled, and Prometheus DISAGREES (says vps is down). monkeypatch.setattr(obs_mod, "PROM_SHADOW_URL", "http://prom:9090") _mock_prom( monkeypatch, '{"data":{"result":[' '{"metric":{"node":"vps"},"value":[1720000000,"0"]}]}}', ) shadow = Observer() _fresh_node_world(shadow) shadow._prune_stale_world() # Authoritative liveness/status must be IDENTICAL to the no-shadow baseline. assert shadow.world_state["nodes"]["vps"]["liveness"] == base_liveness assert shadow.world_state["nodes"]["vps"]["status"] == base_status # And no prom-derived field leaked into the node record. assert "prom_up" not in shadow.world_state["nodes"]["vps"] # --------------------------------------------------------------------------- # 3. Mismatch logging # --------------------------------------------------------------------------- def test_mismatch_is_logged(monkeypatch, caplog): """event=fresh but prom=down → grep-able SHADOW_LIVENESS_MISMATCH at INFO.""" monkeypatch.setattr(obs_mod, "PROM_SHADOW_URL", "http://prom:9090") _mock_prom( monkeypatch, '{"data":{"result":[' '{"metric":{"node":"vps"},"value":[1720000000,"0"]}]}}', ) obs = Observer() _fresh_node_world(obs) with caplog.at_level(logging.INFO, logger="observer"): obs._prune_stale_world() assert "SHADOW_LIVENESS_MISMATCH" in caplog.text assert "node=vps" in caplog.text assert "event=fresh" in caplog.text assert "prom=down" in caplog.text def test_agreement_does_not_log_mismatch(monkeypatch, caplog): """event=fresh and prom=up → agreement, no mismatch line at INFO.""" monkeypatch.setattr(obs_mod, "PROM_SHADOW_URL", "http://prom:9090") _mock_prom( monkeypatch, '{"data":{"result":[' '{"metric":{"node":"vps"},"value":[1720000000,"1"]}]}}', ) obs = Observer() _fresh_node_world(obs) with caplog.at_level(logging.INFO, logger="observer"): obs._prune_stale_world() assert "SHADOW_LIVENESS_MISMATCH" not in caplog.text def test_dead_node_agrees_with_prom_down(monkeypatch, caplog): """event=dead and prom=down → agreement (DEAD maps to down).""" monkeypatch.setattr(obs_mod, "PROM_SHADOW_URL", "http://prom:9090") _mock_prom( monkeypatch, '{"data":{"result":[' '{"metric":{"node":"vps"},"value":[1720000000,"0"]}]}}', ) obs = Observer() obs.world_state["nodes"] = { "vps": {"status": "unknown", "last_seen": int(time.time()) - 700, "roles": ["control-plane"]}, } with caplog.at_level(logging.INFO, logger="observer"): obs._prune_stale_world() assert obs.world_state["nodes"]["vps"]["liveness"] == DEAD assert "SHADOW_LIVENESS_MISMATCH" not in caplog.text def test_node_unknown_to_prometheus_is_not_mismatch(monkeypatch, caplog): """A node with no up{} series (e.g. chelsty-infra) is not a disagreement.""" monkeypatch.setattr(obs_mod, "PROM_SHADOW_URL", "http://prom:9090") _mock_prom(monkeypatch, '{"data":{"result":[]}}') # empty — knows no node obs = Observer() _fresh_node_world(obs) with caplog.at_level(logging.INFO, logger="observer"): obs._prune_stale_world() assert "SHADOW_LIVENESS_MISMATCH" not in caplog.text # And liveness still computed from events. assert obs.world_state["nodes"]["vps"]["liveness"] == FRESH