homelab-codex-ws/services/control-plane/tests/test_dormant_nodes.py

"""Dormant-node handling (topology `status: dormant`, etap 0 truth cleanup).

Contract (docs/architecture/ARCHITEKTURA.md): a dormant node keeps its
last-known world state, the observer emits NO liveness events for it, and the
supervisor generates NO actions of any kind for it.
"""
from __future__ import annotations

import json
import sys
import time
from pathlib import Path

import pytest

sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent / "scripts"))

import observer.observer as obs_mod
from observer.observer import Observer
import supervisor as supervisor_module
from supervisor import Supervisor

TOPOLOGY_YAML = (
    "nodes:\n"
    "  piha:\n"
    "    status: active\n"
    "    roles: [infra]\n"
    "  oldnode:\n"
    "    status: dormant\n"
    "    dormant_reason: \"hardware down\"\n"
    "    roles: [remote]\n"
)


# ---------------------------------------------------------------------------
# Observer
# ---------------------------------------------------------------------------

@pytest.fixture
def obs_paths(tmp_path, monkeypatch):
    world = tmp_path / "world"
    state = tmp_path / "state"
    events = tmp_path / "events"
    logs = tmp_path / "logs"
    repo = tmp_path / "repo"
    for d in (world, state, events, logs, repo / "inventory", repo / "hosts"):
        d.mkdir(parents=True, exist_ok=True)
    (repo / "inventory" / "topology.yaml").write_text(TOPOLOGY_YAML)
    monkeypatch.setattr(obs_mod, "WORLD_DIR", world)
    monkeypatch.setattr(obs_mod, "STATE_DIR", state)
    monkeypatch.setattr(obs_mod, "EVENTS_DIR", events)
    monkeypatch.setattr(obs_mod, "LOGS_DIR", logs)
    monkeypatch.setattr(obs_mod, "INVENTORY_TOPOLOGY", repo / "inventory" / "topology.yaml")
    monkeypatch.setattr(obs_mod, "REPO_ROOT", repo)
    monkeypatch.setattr(obs_mod, "FAILED_EVENTS_DIR", state / "observer_failed_events")
    monkeypatch.setattr(obs_mod, "OBSERVER_STATE_FILE", state / "observer_checkpoint.json")
    monkeypatch.setattr(obs_mod, "PROM_SHADOW_URL", "")
    monkeypatch.setattr(obs_mod, "SHADOW_LOG_DIR", logs / "observer")
    return tmp_path


def test_observer_loads_dormant_set_from_topology(obs_paths):
    obs = Observer()
    assert obs.dormant_nodes == {"oldnode"}


def test_observer_freezes_dormant_node_and_emits_nothing(obs_paths):
    """A dormant node whose heartbeats stopped long ago must NOT be
    reclassified (stale→dead would normally fire node_offline)."""
    obs = Observer()
    stale_age = int(time.time()) - 999999
    obs.world_state["nodes"] = {
        "oldnode": {"status": "stale", "liveness": "stale", "last_seen": stale_age},
        "piha": {"status": "unknown", "last_seen": int(time.time())},
    }

    obs._prune_stale_world()

    frozen = obs.world_state["nodes"]["oldnode"]
    assert frozen["liveness"] == "stale"      # unchanged, despite dead-level age
    assert frozen["status"] == "stale"        # unchanged
    # No liveness event file was written for the dormant node.
    assert not list((obs_paths / "events").glob("**/evt-oldnode-*"))
    # The active node is still processed normally.
    assert obs.world_state["nodes"]["piha"]["liveness"] == "fresh"


def test_observer_keeps_dormant_node_in_world_state(obs_paths):
    """Dormant nodes are in the inventory, so pruning must not remove them."""
    obs = Observer()
    obs.world_state["nodes"] = {"oldnode": {"status": "offline", "last_seen": 1}}

    obs._prune_stale_world()

    assert "oldnode" in obs.world_state["nodes"]


# ---------------------------------------------------------------------------
# Supervisor
# ---------------------------------------------------------------------------

@pytest.fixture
def sup(tmp_path, monkeypatch):
    actions = tmp_path / "actions"
    events = tmp_path / "events"
    world = tmp_path / "world"
    state = tmp_path / "state"
    repo = tmp_path / "repo"
    for d in (actions, events, world, state, repo / "inventory", repo / "hosts" / "oldnode"):
        d.mkdir(parents=True, exist_ok=True)
    (repo / "inventory" / "topology.yaml").write_text(TOPOLOGY_YAML)
    (repo / "hosts" / "oldnode" / "services.yaml").write_text(
        "host: oldnode\nservices:\n  some-svc:\n    role: x\n"
    )
    monkeypatch.setattr(supervisor_module, "ACTIONS_DIR", actions)
    monkeypatch.setattr(supervisor_module, "EVENTS_DIR", events)
    monkeypatch.setattr(supervisor_module, "WORLD_DIR", world)
    monkeypatch.setattr(supervisor_module, "REPO_ROOT", repo)
    s = Supervisor()
    return s


def _pending(tmp_path):
    return list((tmp_path / "actions" / "pending").glob("*.json"))


def test_supervisor_loads_dormant_set(sup):
    sup._load_dormant_nodes()
    assert sup.dormant_nodes == {"oldnode"}


def test_dormant_services_excluded_from_desired_state(sup):
    sup._load_dormant_nodes()
    sup._load_desired_state()
    assert "oldnode/some-svc" not in sup.desired_state["services"]


def test_reconcile_generates_no_actions_for_dormant_node(sup, tmp_path):
    """Full reconcile: missing service + high disk pressure on a dormant node
    must produce zero pending actions."""
    world = tmp_path / "world"
    (world / "nodes.json").write_text(json.dumps(
        {"oldnode": {"status": "offline", "disk_pressure": "high"}}
    ))
    (world / "services.json").write_text("{}")
    (world / "incidents.json").write_text("{}")

    sup.reconcile()

    assert _pending(tmp_path) == []


def test_node_event_from_dormant_node_not_routed(sup, tmp_path):
    sup._load_dormant_nodes()
    event = {
        "id": "evt-oldnode-1-node_offline-node",
        "type": "node_offline",
        "node": "oldnode",
        "payload": {"affected_node": "oldnode"},
    }
    (tmp_path / "events" / f"{event['id']}.json").write_text(json.dumps(event))

    sup._process_ha_events()

    assert _pending(tmp_path) == []


def test_ha_event_from_dormant_node_not_routed(sup, tmp_path):
    sup._load_dormant_nodes()
    event = {
        "id": "evt-oldnode-2-ha_entity_unavailable_long-homeassistant",
        "type": "ha_entity_unavailable_long",
        "node": "oldnode",
        "payload": {},
    }
    (tmp_path / "events" / f"{event['id']}.json").write_text(json.dumps(event))

    sup._process_ha_events()

    assert _pending(tmp_path) == []


def test_active_node_events_still_routed(sup, tmp_path):
    """Sanity: the dormant guard must not swallow active-node events."""
    sup._load_dormant_nodes()
    event = {
        "id": "evt-piha-3-node_stale-node",
        "type": "node_stale",
        "node": "piha",
        "payload": {"affected_node": "piha"},
    }
    (tmp_path / "events" / f"{event['id']}.json").write_text(json.dumps(event))

    sup._process_ha_events()

    assert len(_pending(tmp_path)) == 1