diff --git a/inventory/topology.yaml b/inventory/topology.yaml index 372554d..3f6dcdd 100644 --- a/inventory/topology.yaml +++ b/inventory/topology.yaml @@ -41,6 +41,16 @@ ingress: # dormant — node exists but is expected offline: the observer keeps its # last-known world state and emits NO liveness events for it; the # supervisor generates NO actions for it. See ARCHITEKTURA.md. +# +# duty_cycle: +# Present (any non-empty value) => the node powers down on a schedule BY +# DESIGN. Its node_offline / node_stale / node_online events are still +# emitted by the observer and still appear in the event feed, but the +# supervisor LOGS them instead of generating an alert_only action. +# Everything else is unchanged: the node is `active`, its services are +# reconciled and its disk pressure is acted on exactly like a 24/7 node — +# duty_cycle is NOT dormant. Absent => a liveness transition is a real +# incident (vps, piha). nodes: saturn: @@ -60,6 +70,8 @@ nodes: # offline/online liveness cycle is expected, not a fault. Nothing that # must run 24/7 may live here. status: active + duty_cycle: nightly + duty_cycle_reason: "on-demand compute desktop, powered off ~16 h/day" roles: - compute - ai @@ -79,6 +91,8 @@ nodes: # display power-off like solaria's. Daily stale/offline/online liveness # cycles are expected here, not faults. status: active + duty_cycle: nightly + duty_cycle_reason: "MagicMirror display power-off ~23:30 local (verified 2026-07-30)" roles: - edge - display diff --git a/services/control-plane/src/supervisor.py b/services/control-plane/src/supervisor.py index 48407e4..6de3dc9 100644 --- a/services/control-plane/src/supervisor.py +++ b/services/control-plane/src/supervisor.py @@ -105,6 +105,17 @@ HA_TRANSITION_WINDOW = 300 # 5 minutes # A node we cannot reach cannot be auto-remediated (you can't docker-restart a # host that is offline), so these are alert-only — they exist to make a silent # outage loud. node_online is the recovery notice. +# +# Exception: nodes carrying `duty_cycle` in inventory/topology.yaml power down +# on a schedule BY DESIGN (solaria ~16 h/day, lustro nightly). Their daily +# offline/stale/online transitions are not outages, and before this was handled +# they permanently occupied their own dedup IDs — six pending alerts from +# 2026-06-17/18 that no operator would ever approve, which in turn meant a +# *real* future outage on those nodes could generate nothing (an unapproved +# pending action suppresses its ID forever, recon D14). Their events are still +# emitted by the observer and still visible in the feed; the supervisor logs +# them and generates no action. A node with no duty_cycle (vps, piha) is +# unaffected: an offline vps is a real incident. NODE_ALERT_EVENTS = {"node_offline", "node_stale", "node_online"} NODE_ALERT_COOLDOWN = 3600 # 1-hour cooldown to avoid repeated Telegram noise @@ -149,6 +160,11 @@ class Supervisor: # actions), and node/HA events from them are not routed to alerts. # See kb/decisions/architektura-2026-07-28.md. self.dormant_nodes: set = set() + # Nodes declared with a `duty_cycle` in inventory/topology.yaml. Unlike + # dormant nodes these are fully active — only their liveness + # transitions are suppressed (logged, not actioned). Services, disk + # pressure and HA events on them are handled normally. + self.duty_cycle_nodes: set = set() # In-memory set of already-routed HA event IDs; prevents re-processing # on each reconcile cycle. Grows to at most ~hundreds of entries/day. self._ha_processed_event_ids: set = set() @@ -199,20 +215,34 @@ class Supervisor: # State loading # ------------------------------------------------------------------ - def _load_dormant_nodes(self): - """Refresh the set of dormant nodes from inventory/topology.yaml.""" + def _load_node_policy(self): + """Refresh per-node policy sets from inventory/topology.yaml. + + `status: dormant` -> no actions of any kind for that node. + `duty_cycle: ` -> node is active, but its liveness transitions are + expected and are logged instead of alerted. + + On any failure both sets are left empty: suppression is disabled and + alerts are generated. Fail-loud — an unreadable topology must never + silently mute the fleet. + """ dormant = set() + duty_cycle = set() topo_file = REPO_ROOT / "inventory" / "topology.yaml" try: if topo_file.exists(): with open(topo_file, "r") as f: topo = yaml.safe_load(f) or {} for name, info in (topo.get("nodes") or {}).items(): - if (info or {}).get("status") == "dormant": + info = info or {} + if info.get("status") == "dormant": dormant.add(name) + if info.get("duty_cycle"): + duty_cycle.add(name) except Exception as e: - logger.error(f"Failed to load dormant nodes from {topo_file}: {e}") + logger.error(f"Failed to load node policy from {topo_file}: {e}") self.dormant_nodes = dormant + self.duty_cycle_nodes = duty_cycle def _load_desired_state(self): services = {} @@ -339,7 +369,7 @@ class Supervisor: except Exception as e: logger.error(f"Failed to touch heartbeat file: {e}") - self._load_dormant_nodes() + self._load_node_policy() self._load_desired_state() if not self._load_actual_state(): return # world state unreadable this cycle — skip to avoid false drift @@ -822,6 +852,17 @@ class Supervisor: if node in self.dormant_nodes: logger.debug(f"Suppressing {event_type} on {node}: node dormant") return + if node in self.duty_cycle_nodes: + # Scheduled power-down, not an outage. INFO (not debug) so the + # transition stays visible in `docker logs` — the requirement is + # "log it, don't action it". Volume is ~3 lines/day/node in steady + # state; a supervisor restart re-scans the event store once and + # replays the retained history at this level. + logger.info( + "Duty-cycle node %s: %s logged, no action generated " + "(topology duty_cycle set)", node, event_type, + ) + return action_id = f"alert-{event_type.replace('_', '-')}-{node}" diff --git a/services/control-plane/tests/test_dormant_nodes.py b/services/control-plane/tests/test_dormant_nodes.py index d71c759..d703c70 100644 --- a/services/control-plane/tests/test_dormant_nodes.py +++ b/services/control-plane/tests/test_dormant_nodes.py @@ -126,12 +126,12 @@ def _pending(tmp_path): def test_supervisor_loads_dormant_set(sup): - sup._load_dormant_nodes() + sup._load_node_policy() assert sup.dormant_nodes == {"oldnode"} def test_dormant_services_excluded_from_desired_state(sup): - sup._load_dormant_nodes() + sup._load_node_policy() sup._load_desired_state() assert "oldnode/some-svc" not in sup.desired_state["services"] @@ -152,7 +152,7 @@ def test_reconcile_generates_no_actions_for_dormant_node(sup, tmp_path): def test_node_event_from_dormant_node_not_routed(sup, tmp_path): - sup._load_dormant_nodes() + sup._load_node_policy() event = { "id": "evt-oldnode-1-node_offline-node", "type": "node_offline", @@ -167,7 +167,7 @@ def test_node_event_from_dormant_node_not_routed(sup, tmp_path): def test_ha_event_from_dormant_node_not_routed(sup, tmp_path): - sup._load_dormant_nodes() + sup._load_node_policy() event = { "id": "evt-oldnode-2-ha_entity_unavailable_long-homeassistant", "type": "ha_entity_unavailable_long", @@ -183,7 +183,7 @@ def test_ha_event_from_dormant_node_not_routed(sup, tmp_path): def test_active_node_events_still_routed(sup, tmp_path): """Sanity: the dormant guard must not swallow active-node events.""" - sup._load_dormant_nodes() + sup._load_node_policy() event = { "id": "evt-piha-3-node_stale-node", "type": "node_stale", diff --git a/services/control-plane/tests/test_supervisor_duty_cycle.py b/services/control-plane/tests/test_supervisor_duty_cycle.py new file mode 100644 index 0000000..74d78c3 --- /dev/null +++ b/services/control-plane/tests/test_supervisor_duty_cycle.py @@ -0,0 +1,237 @@ +"""Duty-cycle node handling (topology `duty_cycle`). + +Contract: a node that powers down on a schedule BY DESIGN (solaria ~16 h/day, +lustro nightly) still emits node_offline/node_stale/node_online events, but the +supervisor LOGS them instead of generating an alert_only action. + +`duty_cycle` is deliberately NOT `dormant`: a duty-cycle node stays fully +active — its services are reconciled and its disk pressure is acted on exactly +like a 24/7 node. Only the liveness alerts are suppressed. + +Why this exists: before it, six pending liveness alerts from 2026-06-17/18 sat +in the queue forever for solaria and lustro. An unapproved pending action +suppresses its own dedup ID indefinitely (recon D14), so those stale alerts +also meant a *real* future outage on those nodes would generate nothing. +""" +from __future__ import annotations + +import json +import logging +import sys +from pathlib import Path + +import pytest +import yaml + +sys.path.insert(0, str(Path(__file__).parent.parent / "src")) +import supervisor as supervisor_module +from supervisor import Supervisor + +REPO_ROOT_REAL = Path(__file__).parent.parent.parent.parent + +# solaria: duty-cycled. vps: 24/7, no duty_cycle. oldnode: dormant, to prove +# the two policies load independently from the same file. +TOPOLOGY_YAML = ( + "nodes:\n" + " solaria:\n" + " status: active\n" + " duty_cycle: nightly\n" + " duty_cycle_reason: \"powered off ~16 h/day\"\n" + " roles: [compute]\n" + " vps:\n" + " status: active\n" + " roles: [edge]\n" + " oldnode:\n" + " status: dormant\n" + " roles: [remote]\n" +) + + +@pytest.fixture +def sup(tmp_path, monkeypatch): + actions = tmp_path / "actions" + events = tmp_path / "events" + world = tmp_path / "world" + state = tmp_path / "state" + repo = tmp_path / "repo" + for d in (actions, events, world, state, + repo / "inventory", repo / "hosts" / "solaria"): + d.mkdir(parents=True, exist_ok=True) + (repo / "inventory" / "topology.yaml").write_text(TOPOLOGY_YAML) + (repo / "hosts" / "solaria" / "services.yaml").write_text( + "host: solaria\nservices:\n some-svc:\n role: x\n" + ) + monkeypatch.setattr(supervisor_module, "ACTIONS_DIR", actions) + monkeypatch.setattr(supervisor_module, "EVENTS_DIR", events) + monkeypatch.setattr(supervisor_module, "WORLD_DIR", world) + monkeypatch.setattr(supervisor_module, "REPO_ROOT", repo) + return Supervisor() + + +def _node_event(node: str, etype: str) -> dict: + return { + "id": f"evt-{node}-1-{etype}-node", + "type": etype, + "node": node, + "service": None, + "source": "observer", + "message": f"Node {node} liveness {etype}", + "payload": {"affected_node": node, "from": "fresh", "to": "dead"}, + } + + +def _write_event(tmp_path, event: dict) -> None: + (tmp_path / "events" / f"{event['id']}.json").write_text(json.dumps(event)) + + +def _pending(tmp_path): + return list((tmp_path / "actions" / "pending").glob("*.json")) + + +def _write_world(tmp_path, nodes=None, services=None, incidents=None): + world = tmp_path / "world" + (world / "nodes.json").write_text(json.dumps(nodes or {})) + (world / "services.json").write_text(json.dumps(services or {})) + (world / "incidents.json").write_text(json.dumps(incidents or {})) + + +# --------------------------------------------------------------------------- +# 1. Policy load +# --------------------------------------------------------------------------- + +def test_load_node_policy_populates_both_sets(sup): + """duty_cycle and dormant are loaded independently from one topology read.""" + sup._load_node_policy() + assert sup.duty_cycle_nodes == {"solaria"} + assert sup.dormant_nodes == {"oldnode"} + + +def test_load_node_policy_fails_loud_on_unreadable_topology(sup, tmp_path): + """Unreadable topology => empty sets => suppression disabled, alerts flow. + + Never fail-silent: a broken topology must not mute the fleet. + """ + (tmp_path / "repo" / "inventory" / "topology.yaml").write_text("{[ not yaml") + sup._load_node_policy() + assert sup.duty_cycle_nodes == set() + assert sup.dormant_nodes == set() + + _write_event(tmp_path, _node_event("solaria", "node_offline")) + sup._process_ha_events() + assert (tmp_path / "actions" / "pending" / "alert-node-offline-solaria.json").exists() + + +# --------------------------------------------------------------------------- +# 2. Liveness events from a duty-cycle node are logged, not actioned +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("etype", ["node_offline", "node_stale", "node_online"]) +def test_duty_cycle_liveness_event_generates_no_action(sup, tmp_path, caplog, etype): + sup._load_node_policy() + _write_event(tmp_path, _node_event("solaria", etype)) + + with caplog.at_level(logging.INFO, logger="supervisor"): + sup._process_ha_events() + + assert _pending(tmp_path) == [] + assert any( + "Duty-cycle node solaria" in r.getMessage() and etype in r.getMessage() + for r in caplog.records + ), f"expected an INFO suppression line for {etype}, got: " \ + f"{[r.getMessage() for r in caplog.records]}" + + +# --------------------------------------------------------------------------- +# 3. Regression guard — a 24/7 node is unaffected +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("etype", ["node_offline", "node_stale", "node_online"]) +def test_non_duty_cycle_node_still_alerts(sup, tmp_path, etype): + """An offline vps is still a real incident.""" + sup._load_node_policy() + _write_event(tmp_path, _node_event("vps", etype)) + + sup._process_ha_events() + + action_id = f"alert-{etype.replace('_', '-')}-vps" + path = tmp_path / "actions" / "pending" / f"{action_id}.json" + assert path.exists() + action = json.loads(path.read_text()) + assert action["type"] == "alert_only" + assert action["node"] == "vps" + assert action["payload"]["reason"] == etype + + +# --------------------------------------------------------------------------- +# 4. Scope guard — duty_cycle is NOT dormant +# --------------------------------------------------------------------------- + +def test_duty_cycle_node_missing_service_still_redeploys(sup, tmp_path): + """The drift loop must be untouched: a missing desired service on a + duty-cycle node still produces a redeploy action.""" + _write_world(tmp_path, nodes={}, services={}, incidents={}) + + sup.reconcile() + + assert (tmp_path / "actions" / "pending" / "redeploy-solaria-some-svc.json").exists() + + +def test_duty_cycle_node_disk_pressure_still_cleans(sup, tmp_path): + """disk_cleanup must be untouched for a duty-cycle node.""" + _write_world( + tmp_path, + nodes={"solaria": {"status": "online", "disk_pressure": "high"}}, + services={"solaria/some-svc": {"node": "solaria", "service": "some-svc", + "status": "healthy"}}, + incidents={}, + ) + + sup.reconcile() + + assert (tmp_path / "actions" / "pending" / "disk-cleanup-solaria.json").exists() + + +# --------------------------------------------------------------------------- +# 5. HA event routing on a duty-cycle node is untouched +# --------------------------------------------------------------------------- + +def test_duty_cycle_node_ha_event_still_routed(sup, tmp_path): + sup._load_node_policy() + event = { + "id": "evt-solaria-2-ha_update_available-homeassistant", + "type": "ha_update_available", + "node": "solaria", + "service": "homeassistant", + "message": "Update available: test", + "payload": {}, + } + (tmp_path / "events" / f"{event['id']}.json").write_text(json.dumps(event)) + + sup._process_ha_events() + + assert (tmp_path / "actions" / "pending" + / "alert-ha-update-available-solaria.json").exists() + + +# --------------------------------------------------------------------------- +# 6. The real topology carries the field — removing it fails CI +# --------------------------------------------------------------------------- + +def test_real_topology_declares_expected_duty_cycle_nodes(): + """Guards the data half of the change: if someone drops `duty_cycle` from + solaria or lustro, or adds it to a 24/7 node, this fails.""" + topo = yaml.safe_load((REPO_ROOT_REAL / "inventory" / "topology.yaml").read_text()) + duty_cycle = { + name for name, info in (topo.get("nodes") or {}).items() + if (info or {}).get("duty_cycle") + } + assert duty_cycle == {"solaria", "lustro"} + + # The 24/7 nodes must stay alertable. + for name in ("vps", "piha"): + assert not (topo["nodes"][name] or {}).get("duty_cycle"), \ + f"{name} is a 24/7 node and must not carry duty_cycle" + + # duty_cycle nodes stay `active` — duty_cycle is not a way to spell dormant. + for name in ("solaria", "lustro"): + assert topo["nodes"][name]["status"] == "active"