"""Duty-cycle node handling (topology `duty_cycle`). Contract: a node that powers down on a schedule BY DESIGN (solaria ~16 h/day, lustro nightly) still emits node_offline/node_stale/node_online events, but the supervisor LOGS them instead of generating an alert_only action. `duty_cycle` is deliberately NOT `dormant`: a duty-cycle node stays fully active — its services are reconciled and its disk pressure is acted on exactly like a 24/7 node. Only the liveness alerts are suppressed. Why this exists: before it, six pending liveness alerts from 2026-06-17/18 sat in the queue forever for solaria and lustro. An unapproved pending action suppresses its own dedup ID indefinitely (recon D14), so those stale alerts also meant a *real* future outage on those nodes would generate nothing. """ from __future__ import annotations import json import logging import sys from pathlib import Path import pytest import yaml sys.path.insert(0, str(Path(__file__).parent.parent / "src")) import supervisor as supervisor_module from supervisor import Supervisor REPO_ROOT_REAL = Path(__file__).parent.parent.parent.parent # solaria: duty-cycled. vps: 24/7, no duty_cycle. oldnode: dormant, to prove # the two policies load independently from the same file. TOPOLOGY_YAML = ( "nodes:\n" " solaria:\n" " status: active\n" " duty_cycle: nightly\n" " duty_cycle_reason: \"powered off ~16 h/day\"\n" " roles: [compute]\n" " vps:\n" " status: active\n" " roles: [edge]\n" " oldnode:\n" " status: dormant\n" " roles: [remote]\n" ) @pytest.fixture def sup(tmp_path, monkeypatch): actions = tmp_path / "actions" events = tmp_path / "events" world = tmp_path / "world" state = tmp_path / "state" repo = tmp_path / "repo" for d in (actions, events, world, state, repo / "inventory", repo / "hosts" / "solaria"): d.mkdir(parents=True, exist_ok=True) (repo / "inventory" / "topology.yaml").write_text(TOPOLOGY_YAML) (repo / "hosts" / "solaria" / "services.yaml").write_text( "host: solaria\nservices:\n some-svc:\n role: x\n" ) monkeypatch.setattr(supervisor_module, "ACTIONS_DIR", actions) monkeypatch.setattr(supervisor_module, "EVENTS_DIR", events) monkeypatch.setattr(supervisor_module, "WORLD_DIR", world) monkeypatch.setattr(supervisor_module, "REPO_ROOT", repo) return Supervisor() def _node_event(node: str, etype: str) -> dict: return { "id": f"evt-{node}-1-{etype}-node", "type": etype, "node": node, "service": None, "source": "observer", "message": f"Node {node} liveness {etype}", "payload": {"affected_node": node, "from": "fresh", "to": "dead"}, } def _write_event(tmp_path, event: dict) -> None: (tmp_path / "events" / f"{event['id']}.json").write_text(json.dumps(event)) def _pending(tmp_path): return list((tmp_path / "actions" / "pending").glob("*.json")) def _write_world(tmp_path, nodes=None, services=None, incidents=None): world = tmp_path / "world" (world / "nodes.json").write_text(json.dumps(nodes or {})) (world / "services.json").write_text(json.dumps(services or {})) (world / "incidents.json").write_text(json.dumps(incidents or {})) # --------------------------------------------------------------------------- # 1. Policy load # --------------------------------------------------------------------------- def test_load_node_policy_populates_both_sets(sup): """duty_cycle and dormant are loaded independently from one topology read.""" sup._load_node_policy() assert sup.duty_cycle_nodes == {"solaria"} assert sup.dormant_nodes == {"oldnode"} def test_load_node_policy_fails_loud_on_unreadable_topology(sup, tmp_path): """Unreadable topology => empty sets => suppression disabled, alerts flow. Never fail-silent: a broken topology must not mute the fleet. """ (tmp_path / "repo" / "inventory" / "topology.yaml").write_text("{[ not yaml") sup._load_node_policy() assert sup.duty_cycle_nodes == set() assert sup.dormant_nodes == set() _write_event(tmp_path, _node_event("solaria", "node_offline")) sup._process_ha_events() assert (tmp_path / "actions" / "pending" / "alert-node-offline-solaria.json").exists() # --------------------------------------------------------------------------- # 2. Liveness events from a duty-cycle node are logged, not actioned # --------------------------------------------------------------------------- @pytest.mark.parametrize("etype", ["node_offline", "node_stale", "node_online"]) def test_duty_cycle_liveness_event_generates_no_action(sup, tmp_path, caplog, etype): sup._load_node_policy() _write_event(tmp_path, _node_event("solaria", etype)) with caplog.at_level(logging.INFO, logger="supervisor"): sup._process_ha_events() assert _pending(tmp_path) == [] assert any( "Duty-cycle node solaria" in r.getMessage() and etype in r.getMessage() for r in caplog.records ), f"expected an INFO suppression line for {etype}, got: " \ f"{[r.getMessage() for r in caplog.records]}" # --------------------------------------------------------------------------- # 3. Regression guard — a 24/7 node is unaffected # --------------------------------------------------------------------------- @pytest.mark.parametrize("etype", ["node_offline", "node_stale", "node_online"]) def test_non_duty_cycle_node_still_alerts(sup, tmp_path, etype): """An offline vps is still a real incident.""" sup._load_node_policy() _write_event(tmp_path, _node_event("vps", etype)) sup._process_ha_events() action_id = f"alert-{etype.replace('_', '-')}-vps" path = tmp_path / "actions" / "pending" / f"{action_id}.json" assert path.exists() action = json.loads(path.read_text()) assert action["type"] == "alert_only" assert action["node"] == "vps" assert action["payload"]["reason"] == etype # --------------------------------------------------------------------------- # 4. Scope guard — duty_cycle is NOT dormant # --------------------------------------------------------------------------- def test_duty_cycle_node_missing_service_still_redeploys(sup, tmp_path): """The drift loop must be untouched: a missing desired service on a duty-cycle node still produces a redeploy action.""" _write_world(tmp_path, nodes={}, services={}, incidents={}) sup.reconcile() assert (tmp_path / "actions" / "pending" / "redeploy-solaria-some-svc.json").exists() def test_duty_cycle_node_disk_pressure_still_cleans(sup, tmp_path): """disk_cleanup must be untouched for a duty-cycle node.""" _write_world( tmp_path, nodes={"solaria": {"status": "online", "disk_pressure": "high"}}, services={"solaria/some-svc": {"node": "solaria", "service": "some-svc", "status": "healthy"}}, incidents={}, ) sup.reconcile() assert (tmp_path / "actions" / "pending" / "disk-cleanup-solaria.json").exists() # --------------------------------------------------------------------------- # 5. HA event routing on a duty-cycle node is untouched # --------------------------------------------------------------------------- def test_duty_cycle_node_ha_event_still_routed(sup, tmp_path): sup._load_node_policy() event = { "id": "evt-solaria-2-ha_update_available-homeassistant", "type": "ha_update_available", "node": "solaria", "service": "homeassistant", "message": "Update available: test", "payload": {}, } (tmp_path / "events" / f"{event['id']}.json").write_text(json.dumps(event)) sup._process_ha_events() assert (tmp_path / "actions" / "pending" / "alert-ha-update-available-solaria.json").exists() # --------------------------------------------------------------------------- # 6. The real topology carries the field — removing it fails CI # --------------------------------------------------------------------------- def test_real_topology_declares_expected_duty_cycle_nodes(): """Guards the data half of the change: if someone drops `duty_cycle` from solaria or lustro, or adds it to a 24/7 node, this fails.""" topo = yaml.safe_load((REPO_ROOT_REAL / "inventory" / "topology.yaml").read_text()) duty_cycle = { name for name, info in (topo.get("nodes") or {}).items() if (info or {}).get("duty_cycle") } assert duty_cycle == {"solaria", "lustro"} # The 24/7 nodes must stay alertable. for name in ("vps", "piha"): assert not (topo["nodes"][name] or {}).get("duty_cycle"), \ f"{name} is a 24/7 node and must not carry duty_cycle" # duty_cycle nodes stay `active` — duty_cycle is not a way to spell dormant. for name in ("solaria", "lustro"): assert topo["nodes"][name]["status"] == "active"