homelab-codex-ws/services/control-plane/tests/test_observer_shadow_liveness.py

"""Tests for the observer SHADOW-READ of Prometheus up{} liveness (cutover etap 1).

The whole point of etap 1 is a PARALLEL-RUN: the observer additionally queries
Prometheus and LOGS disagreements, but authoritative liveness stays 100%
event-driven (compute_liveness).  These tests prove exactly that: shadow-read
never mutates node_info["liveness"]/["status"], is fail-open on any Prometheus
error, and logs a grep-able SHADOW_LIVENESS_MISMATCH line on disagreement.
"""
from __future__ import annotations

import logging
import sys
import time
from pathlib import Path

import pytest

# Observer lives outside the control-plane package; add scripts/ to path.
sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent / "scripts"))
import observer.observer as obs_mod
from observer.observer import Observer
from liveness import FRESH, DEAD


# ---------------------------------------------------------------------------
# Fixtures / helpers
# ---------------------------------------------------------------------------

@pytest.fixture(autouse=True)
def _redirect_observer_paths(tmp_path, monkeypatch):
    """Redirect every observer runtime path into a per-test tmp_path.

    Mirrors the fixture in test_incident_lifecycle.py so these tests never touch
    the real /opt/homelab state.  Also defaults PROM_SHADOW_URL to "" (disabled)
    so a test that forgets to set it exercises the graceful no-op path.
    """
    world = tmp_path / "world"
    state = tmp_path / "state"
    events = tmp_path / "events"
    logs = tmp_path / "logs"
    repo = tmp_path / "repo"

    for d in (world, state, events, logs, repo / "inventory", repo / "hosts"):
        d.mkdir(parents=True, exist_ok=True)

    (repo / "inventory" / "topology.yaml").write_text(
        "nodes:\n"
        "  vps:\n    roles: [control-plane]\n    connectivity: {}\n"
        "  piha:\n    roles: [infra]\n    connectivity: {}\n"
    )

    monkeypatch.setattr(obs_mod, "WORLD_DIR", world)
    monkeypatch.setattr(obs_mod, "STATE_DIR", state)
    monkeypatch.setattr(obs_mod, "EVENTS_DIR", events)
    monkeypatch.setattr(obs_mod, "LOGS_DIR", logs)
    monkeypatch.setattr(obs_mod, "INVENTORY_TOPOLOGY", repo / "inventory" / "topology.yaml")
    monkeypatch.setattr(obs_mod, "REPO_ROOT", repo)
    monkeypatch.setattr(obs_mod, "FAILED_EVENTS_DIR", state / "observer_failed_events")
    monkeypatch.setattr(obs_mod, "OBSERVER_STATE_FILE", state / "observer_checkpoint.json")
    monkeypatch.setattr(obs_mod, "PROM_SHADOW_URL", "")
    # Persistent SHADOW_LIVENESS_MISMATCH log → per-test tmp dir (never /opt or /var).
    monkeypatch.setattr(obs_mod, "SHADOW_LOG_DIR", logs / "observer")


class _FakeResp:
    """Minimal context-manager stand-in for urllib.request.urlopen()'s return."""

    def __init__(self, body: str):
        self._body = body

    def __enter__(self):
        return self

    def __exit__(self, *exc):
        return False

    def read(self):
        return self._body.encode("utf-8")


def _mock_prom(monkeypatch, body):
    """Make urllib.request.urlopen return `body` (str) or raise (if Exception)."""
    def _fake_urlopen(url, timeout=None):
        if isinstance(body, Exception):
            raise body
        return _FakeResp(body)
    monkeypatch.setattr(obs_mod.urllib.request, "urlopen", _fake_urlopen)


_SAMPLE_UP_JSON = (
    '{"status":"success","data":{"resultType":"vector","result":['
    '{"metric":{"__name__":"up","job":"fleet-node","node":"vps"},"value":[1720000000,"1"]},'
    '{"metric":{"__name__":"up","job":"fleet-node","node":"piha"},"value":[1720000000,"0"]},'
    '{"metric":{"__name__":"up","job":"prometheus"},"value":[1720000000,"1"]}'
    ']}}'
)


# ---------------------------------------------------------------------------
# 1. _query_prometheus_liveness — disabled / parsing / fail-open
# ---------------------------------------------------------------------------

def test_query_returns_empty_when_url_unset(monkeypatch):
    """PROM_SHADOW_URL empty → {} (shadow disabled), no HTTP attempted."""
    monkeypatch.setattr(obs_mod, "PROM_SHADOW_URL", "")

    def _boom(*a, **k):  # pragma: no cover - must never be called
        raise AssertionError("urlopen must not be called when shadow is disabled")
    monkeypatch.setattr(obs_mod.urllib.request, "urlopen", _boom)

    assert Observer()._query_prometheus_liveness() == {}


def test_query_parses_up_series_by_node_label(monkeypatch):
    """Parses up{} into {node: bool}; series without a node label are ignored."""
    monkeypatch.setattr(obs_mod, "PROM_SHADOW_URL", "http://prom:9090")
    _mock_prom(monkeypatch, _SAMPLE_UP_JSON)

    result = Observer()._query_prometheus_liveness()

    assert result == {"vps": True, "piha": False}
    assert "prometheus" not in result  # up{job="prometheus"} has no node label


def test_query_fail_open_on_unreachable(monkeypatch):
    """Prometheus unreachable → {} and NO exception propagates (fail-open)."""
    monkeypatch.setattr(obs_mod, "PROM_SHADOW_URL", "http://prom:9090")
    _mock_prom(monkeypatch, ConnectionRefusedError("nope"))

    assert Observer()._query_prometheus_liveness() == {}


def test_query_fail_open_on_bad_json(monkeypatch):
    """Malformed JSON body → {} and no raise."""
    monkeypatch.setattr(obs_mod, "PROM_SHADOW_URL", "http://prom:9090")
    _mock_prom(monkeypatch, "this is not json")

    assert Observer()._query_prometheus_liveness() == {}


# ---------------------------------------------------------------------------
# 2. The load-bearing invariant: shadow-read NEVER changes liveness
# ---------------------------------------------------------------------------

def _fresh_node_world(obs: Observer):
    """Seed one node whose event-driven liveness computes to FRESH."""
    obs.world_state["nodes"] = {
        "vps": {"status": "unknown", "last_seen": int(time.time()), "roles": ["control-plane"]},
    }


def test_shadow_does_not_change_liveness(monkeypatch):
    """Event liveness after a cycle WITH shadow (mismatching prom) is byte-for-byte
    identical to a cycle WITHOUT shadow. Proof that shadow switches nothing."""
    # Baseline: shadow disabled.
    baseline = Observer()
    _fresh_node_world(baseline)
    baseline._prune_stale_world()
    base_liveness = baseline.world_state["nodes"]["vps"]["liveness"]
    base_status = baseline.world_state["nodes"]["vps"]["status"]

    assert base_liveness == FRESH
    assert base_status == "online"

    # Shadow enabled, and Prometheus DISAGREES (says vps is down).
    monkeypatch.setattr(obs_mod, "PROM_SHADOW_URL", "http://prom:9090")
    _mock_prom(
        monkeypatch,
        '{"data":{"result":['
        '{"metric":{"node":"vps"},"value":[1720000000,"0"]}]}}',
    )
    shadow = Observer()
    _fresh_node_world(shadow)
    shadow._prune_stale_world()

    # Authoritative liveness/status must be IDENTICAL to the no-shadow baseline.
    assert shadow.world_state["nodes"]["vps"]["liveness"] == base_liveness
    assert shadow.world_state["nodes"]["vps"]["status"] == base_status
    # And no prom-derived field leaked into the node record.
    assert "prom_up" not in shadow.world_state["nodes"]["vps"]


# ---------------------------------------------------------------------------
# 3. Mismatch logging
# ---------------------------------------------------------------------------

def test_mismatch_is_logged(monkeypatch, caplog):
    """event=fresh but prom=down → grep-able SHADOW_LIVENESS_MISMATCH at INFO."""
    monkeypatch.setattr(obs_mod, "PROM_SHADOW_URL", "http://prom:9090")
    _mock_prom(
        monkeypatch,
        '{"data":{"result":['
        '{"metric":{"node":"vps"},"value":[1720000000,"0"]}]}}',
    )
    obs = Observer()
    _fresh_node_world(obs)

    with caplog.at_level(logging.INFO, logger="observer"):
        obs._prune_stale_world()

    assert "SHADOW_LIVENESS_MISMATCH" in caplog.text
    assert "node=vps" in caplog.text
    assert "event=fresh" in caplog.text
    assert "prom=down" in caplog.text


def test_agreement_does_not_log_mismatch(monkeypatch, caplog):
    """event=fresh and prom=up → agreement, no mismatch line at INFO."""
    monkeypatch.setattr(obs_mod, "PROM_SHADOW_URL", "http://prom:9090")
    _mock_prom(
        monkeypatch,
        '{"data":{"result":['
        '{"metric":{"node":"vps"},"value":[1720000000,"1"]}]}}',
    )
    obs = Observer()
    _fresh_node_world(obs)

    with caplog.at_level(logging.INFO, logger="observer"):
        obs._prune_stale_world()

    assert "SHADOW_LIVENESS_MISMATCH" not in caplog.text


def test_dead_node_agrees_with_prom_down(monkeypatch, caplog):
    """event=dead and prom=down → agreement (DEAD maps to down)."""
    monkeypatch.setattr(obs_mod, "PROM_SHADOW_URL", "http://prom:9090")
    _mock_prom(
        monkeypatch,
        '{"data":{"result":['
        '{"metric":{"node":"vps"},"value":[1720000000,"0"]}]}}',
    )
    obs = Observer()
    obs.world_state["nodes"] = {
        "vps": {"status": "unknown", "last_seen": int(time.time()) - 700, "roles": ["control-plane"]},
    }

    with caplog.at_level(logging.INFO, logger="observer"):
        obs._prune_stale_world()

    assert obs.world_state["nodes"]["vps"]["liveness"] == DEAD
    assert "SHADOW_LIVENESS_MISMATCH" not in caplog.text


def test_node_unknown_to_prometheus_is_not_mismatch(monkeypatch, caplog):
    """A node with no up{} series (e.g. chelsty-infra) is not a disagreement."""
    monkeypatch.setattr(obs_mod, "PROM_SHADOW_URL", "http://prom:9090")
    _mock_prom(monkeypatch, '{"data":{"result":[]}}')  # empty — knows no node
    obs = Observer()
    _fresh_node_world(obs)

    with caplog.at_level(logging.INFO, logger="observer"):
        obs._prune_stale_world()

    assert "SHADOW_LIVENESS_MISMATCH" not in caplog.text
    # And liveness still computed from events.
    assert obs.world_state["nodes"]["vps"]["liveness"] == FRESH


# ---------------------------------------------------------------------------
# 4. PERSISTENT mismatch log (cutover etap 2) — survives container recreate
# ---------------------------------------------------------------------------

def _mismatch_prom(monkeypatch):
    """Prometheus says vps is DOWN → mismatch against a FRESH event liveness."""
    monkeypatch.setattr(obs_mod, "PROM_SHADOW_URL", "http://prom:9090")
    _mock_prom(
        monkeypatch,
        '{"data":{"result":['
        '{"metric":{"node":"vps"},"value":[1720000000,"0"]}]}}',
    )


def test_mismatch_is_persisted_to_file(monkeypatch, tmp_path, caplog):
    """The mismatch is written to the persistent log FILE, not only stdout.

    This is the whole point: the file lives on a host-mounted path, so it
    survives a `docker rm`/recreate that would drop stdout json-file logs.
    """
    _mismatch_prom(monkeypatch)
    obs = Observer()
    _fresh_node_world(obs)

    # stdout path still emits (unchanged behaviour) AND the file is written.
    with caplog.at_level(logging.INFO, logger="observer"):
        obs._prune_stale_world()

    log_file = tmp_path / "logs" / "observer" / "shadow-liveness.log"
    assert log_file.exists(), "persistent shadow log file was not created"
    content = log_file.read_text()
    assert "SHADOW_LIVENESS_MISMATCH" in content
    assert "node=vps" in content
    assert "event=fresh" in content
    assert "prom=down" in content
    # Line carries a timestamp (asctime format) — grep-able forensic record.
    assert " - INFO - SHADOW_LIVENESS_MISMATCH" in content
    # And stdout still got it too (no regression to docker logs visibility).
    assert "SHADOW_LIVENESS_MISMATCH" in caplog.text


def test_agreement_is_not_persisted_to_file(monkeypatch, tmp_path):
    """Agreement (event=fresh, prom=up) writes nothing to the persistent log."""
    monkeypatch.setattr(obs_mod, "PROM_SHADOW_URL", "http://prom:9090")
    _mock_prom(
        monkeypatch,
        '{"data":{"result":['
        '{"metric":{"node":"vps"},"value":[1720000000,"1"]}]}}',
    )
    obs = Observer()
    _fresh_node_world(obs)
    obs._prune_stale_world()

    log_file = tmp_path / "logs" / "observer" / "shadow-liveness.log"
    # File may or may not exist (handler creates it lazily); if it does, empty.
    if log_file.exists():
        assert "SHADOW_LIVENESS_MISMATCH" not in log_file.read_text()


def test_shadow_logger_failsafe_when_dir_unwritable(monkeypatch, tmp_path, caplog):
    """If the persistent log dir cannot be created, the observer must NOT crash.

    It logs one warning, the shadow logger is left handler-less (a silent no-op),
    and a subsequent mismatch cycle runs fine and STILL reaches stdout.
    """
    # Point SHADOW_LOG_DIR under a *file*, so os.makedirs() raises.
    blocker = tmp_path / "blocker"
    blocker.write_text("i am a file, not a directory")
    monkeypatch.setattr(obs_mod, "SHADOW_LOG_DIR", blocker / "observer")

    # Construction must not raise, and must warn about the failure.
    with caplog.at_level(logging.WARNING, logger="observer"):
        obs = Observer()
    assert "could not open persistent mismatch log" in caplog.text
    # No file handler was attached (fail-safe → handler-less logger).
    assert obs.shadow_logger.handlers == []

    # A real mismatch cycle still works: no raise, stdout still logs it.
    caplog.clear()
    _mismatch_prom(monkeypatch)
    obs2 = Observer()  # re-reads the (still broken) SHADOW_LOG_DIR
    _fresh_node_world(obs2)
    with caplog.at_level(logging.INFO, logger="observer"):
        obs2._prune_stale_world()  # must not raise
    assert "SHADOW_LIVENESS_MISMATCH" in caplog.text  # stdout intact