homelab-codex-ws/services/control-plane/tests/test_supervisor_duty_cycle.py

"""Duty-cycle node handling (topology `duty_cycle`).

Contract: a node that powers down on a schedule BY DESIGN (solaria ~16 h/day,
lustro nightly) still emits node_offline/node_stale/node_online events, but the
supervisor LOGS them instead of generating an alert_only action.

`duty_cycle` is deliberately NOT `dormant`: a duty-cycle node stays fully
active — its services are reconciled and its disk pressure is acted on exactly
like a 24/7 node.  Only the liveness alerts are suppressed.

Why this exists: before it, six pending liveness alerts from 2026-06-17/18 sat
in the queue forever for solaria and lustro.  An unapproved pending action
suppresses its own dedup ID indefinitely (recon D14), so those stale alerts
also meant a *real* future outage on those nodes would generate nothing.
"""
from __future__ import annotations

import json
import logging
import sys
from pathlib import Path

import pytest
import yaml

sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
import supervisor as supervisor_module
from supervisor import Supervisor

REPO_ROOT_REAL = Path(__file__).parent.parent.parent.parent

# solaria: duty-cycled.  vps: 24/7, no duty_cycle.  oldnode: dormant, to prove
# the two policies load independently from the same file.
TOPOLOGY_YAML = (
    "nodes:\n"
    "  solaria:\n"
    "    status: active\n"
    "    duty_cycle: nightly\n"
    "    duty_cycle_reason: \"powered off ~16 h/day\"\n"
    "    roles: [compute]\n"
    "  vps:\n"
    "    status: active\n"
    "    roles: [edge]\n"
    "  oldnode:\n"
    "    status: dormant\n"
    "    roles: [remote]\n"
)


@pytest.fixture
def sup(tmp_path, monkeypatch):
    actions = tmp_path / "actions"
    events = tmp_path / "events"
    world = tmp_path / "world"
    state = tmp_path / "state"
    repo = tmp_path / "repo"
    for d in (actions, events, world, state,
              repo / "inventory", repo / "hosts" / "solaria"):
        d.mkdir(parents=True, exist_ok=True)
    (repo / "inventory" / "topology.yaml").write_text(TOPOLOGY_YAML)
    (repo / "hosts" / "solaria" / "services.yaml").write_text(
        "host: solaria\nservices:\n  some-svc:\n    role: x\n"
    )
    monkeypatch.setattr(supervisor_module, "ACTIONS_DIR", actions)
    monkeypatch.setattr(supervisor_module, "EVENTS_DIR", events)
    monkeypatch.setattr(supervisor_module, "WORLD_DIR", world)
    monkeypatch.setattr(supervisor_module, "REPO_ROOT", repo)
    return Supervisor()


def _node_event(node: str, etype: str) -> dict:
    return {
        "id": f"evt-{node}-1-{etype}-node",
        "type": etype,
        "node": node,
        "service": None,
        "source": "observer",
        "message": f"Node {node} liveness {etype}",
        "payload": {"affected_node": node, "from": "fresh", "to": "dead"},
    }


def _write_event(tmp_path, event: dict) -> None:
    (tmp_path / "events" / f"{event['id']}.json").write_text(json.dumps(event))


def _pending(tmp_path):
    return list((tmp_path / "actions" / "pending").glob("*.json"))


def _write_world(tmp_path, nodes=None, services=None, incidents=None):
    world = tmp_path / "world"
    (world / "nodes.json").write_text(json.dumps(nodes or {}))
    (world / "services.json").write_text(json.dumps(services or {}))
    (world / "incidents.json").write_text(json.dumps(incidents or {}))


# ---------------------------------------------------------------------------
# 1. Policy load
# ---------------------------------------------------------------------------

def test_load_node_policy_populates_both_sets(sup):
    """duty_cycle and dormant are loaded independently from one topology read."""
    sup._load_node_policy()
    assert sup.duty_cycle_nodes == {"solaria"}
    assert sup.dormant_nodes == {"oldnode"}


def test_load_node_policy_fails_loud_on_unreadable_topology(sup, tmp_path):
    """Unreadable topology => empty sets => suppression disabled, alerts flow.

    Never fail-silent: a broken topology must not mute the fleet.
    """
    (tmp_path / "repo" / "inventory" / "topology.yaml").write_text("{[ not yaml")
    sup._load_node_policy()
    assert sup.duty_cycle_nodes == set()
    assert sup.dormant_nodes == set()

    _write_event(tmp_path, _node_event("solaria", "node_offline"))
    sup._process_ha_events()
    assert (tmp_path / "actions" / "pending" / "alert-node-offline-solaria.json").exists()


# ---------------------------------------------------------------------------
# 2. Liveness events from a duty-cycle node are logged, not actioned
# ---------------------------------------------------------------------------

@pytest.mark.parametrize("etype", ["node_offline", "node_stale", "node_online"])
def test_duty_cycle_liveness_event_generates_no_action(sup, tmp_path, caplog, etype):
    sup._load_node_policy()
    _write_event(tmp_path, _node_event("solaria", etype))

    with caplog.at_level(logging.INFO, logger="supervisor"):
        sup._process_ha_events()

    assert _pending(tmp_path) == []
    assert any(
        "Duty-cycle node solaria" in r.getMessage() and etype in r.getMessage()
        for r in caplog.records
    ), f"expected an INFO suppression line for {etype}, got: " \
       f"{[r.getMessage() for r in caplog.records]}"


# ---------------------------------------------------------------------------
# 3. Regression guard — a 24/7 node is unaffected
# ---------------------------------------------------------------------------

@pytest.mark.parametrize("etype", ["node_offline", "node_stale", "node_online"])
def test_non_duty_cycle_node_still_alerts(sup, tmp_path, etype):
    """An offline vps is still a real incident."""
    sup._load_node_policy()
    _write_event(tmp_path, _node_event("vps", etype))

    sup._process_ha_events()

    action_id = f"alert-{etype.replace('_', '-')}-vps"
    path = tmp_path / "actions" / "pending" / f"{action_id}.json"
    assert path.exists()
    action = json.loads(path.read_text())
    assert action["type"] == "alert_only"
    assert action["node"] == "vps"
    assert action["payload"]["reason"] == etype


# ---------------------------------------------------------------------------
# 4. Scope guard — duty_cycle is NOT dormant
# ---------------------------------------------------------------------------

def test_duty_cycle_node_missing_service_still_redeploys(sup, tmp_path):
    """The drift loop must be untouched: a missing desired service on a
    duty-cycle node still produces a redeploy action."""
    _write_world(tmp_path, nodes={}, services={}, incidents={})

    sup.reconcile()

    assert (tmp_path / "actions" / "pending" / "redeploy-solaria-some-svc.json").exists()


def test_duty_cycle_node_disk_pressure_still_cleans(sup, tmp_path):
    """disk_cleanup must be untouched for a duty-cycle node."""
    _write_world(
        tmp_path,
        nodes={"solaria": {"status": "online", "disk_pressure": "high"}},
        services={"solaria/some-svc": {"node": "solaria", "service": "some-svc",
                                       "status": "healthy"}},
        incidents={},
    )

    sup.reconcile()

    assert (tmp_path / "actions" / "pending" / "disk-cleanup-solaria.json").exists()


# ---------------------------------------------------------------------------
# 5. HA event routing on a duty-cycle node is untouched
# ---------------------------------------------------------------------------

def test_duty_cycle_node_ha_event_still_routed(sup, tmp_path):
    sup._load_node_policy()
    event = {
        "id": "evt-solaria-2-ha_update_available-homeassistant",
        "type": "ha_update_available",
        "node": "solaria",
        "service": "homeassistant",
        "message": "Update available: test",
        "payload": {},
    }
    (tmp_path / "events" / f"{event['id']}.json").write_text(json.dumps(event))

    sup._process_ha_events()

    assert (tmp_path / "actions" / "pending"
            / "alert-ha-update-available-solaria.json").exists()


# ---------------------------------------------------------------------------
# 6. The real topology carries the field — removing it fails CI
# ---------------------------------------------------------------------------

def test_real_topology_declares_expected_duty_cycle_nodes():
    """Guards the data half of the change: if someone drops `duty_cycle` from
    solaria or lustro, or adds it to a 24/7 node, this fails."""
    topo = yaml.safe_load((REPO_ROOT_REAL / "inventory" / "topology.yaml").read_text())
    duty_cycle = {
        name for name, info in (topo.get("nodes") or {}).items()
        if (info or {}).get("duty_cycle")
    }
    assert duty_cycle == {"solaria", "lustro"}

    # The 24/7 nodes must stay alertable.
    for name in ("vps", "piha"):
        assert not (topo["nodes"][name] or {}).get("duty_cycle"), \
            f"{name} is a 24/7 node and must not carry duty_cycle"

    # duty_cycle nodes stay `active` — duty_cycle is not a way to spell dormant.
    for name in ("solaria", "lustro"):
        assert topo["nodes"][name]["status"] == "active"