From aa8276963cc7a104ac9ce59ed7ffd71b3958a10e Mon Sep 17 00:00:00 2001 From: oskar Date: Wed, 29 Jul 2026 18:47:01 +0200 Subject: [PATCH] feat(topology): node status active|dormant + dormant handling in control plane MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit topology.yaml (per its own rule that hosts/*/services.yaml are authoritative, recon F20.1): - add status field; chelsty-infra + chelsty-ha -> dormant (site hardware down since ~2026-06-01, revival planned) - add lustro as a full active node (runs node-agent, ships events, F20.11) - drop per-node service lists (vps list contradicted hosts/vps, F20.1; piha/solaria lists were stale too, F20.8/F20.9) — node-level truth only - deployment.mode pull -> push: every deploy script SSH-pushes from saturn (F20.10) Dormant semantics in code: - observer (scripts/observer/observer.py): loads status from topology; _prune_stale_world skips dormant nodes — last-known world state stays frozen, no node_offline/node_stale/node_online events emitted - supervisor (services/control-plane/src/supervisor.py): reloads dormant set each reconcile; dormant hosts' services excluded from desired state (existing pending actions auto-cancel via service_removed_from_desired_state), disk_cleanup skipped, node/HA events from dormant nodes not routed to alerts Tests: services/control-plane/tests/test_dormant_nodes.py (9 cases); full suite 145 passed. Co-Authored-By: Claude Fable 5 --- inventory/topology.yaml | 68 +++--- scripts/observer/observer.py | 20 +- services/control-plane/src/supervisor.py | 36 +++- .../control-plane/tests/test_dormant_nodes.py | 197 ++++++++++++++++++ 4 files changed, 283 insertions(+), 38 deletions(-) create mode 100644 services/control-plane/tests/test_dormant_nodes.py diff --git a/inventory/topology.yaml b/inventory/topology.yaml index 9de5a4a..f7e1abe 100644 --- a/inventory/topology.yaml +++ b/inventory/topology.yaml @@ -3,7 +3,9 @@ topology: git_provider: forgejo deployment: - mode: pull + # Every deploy script SSH-pushes from saturn to the target node (recon + # F20.10); "pull" never matched reality. + mode: push orchestrator: saturn # Ingress dla usług domowych przez NPM @ PIHA + wildcard cert kapala.org. @@ -31,54 +33,56 @@ ingress: access: tailscale cert: wildcard-kapala +# Node-level truth ONLY. Per-node service lists live in hosts//services.yaml +# (authoritative — the lists formerly duplicated here contradicted them, recon F20.1). +# +# status: +# active — node is monitored and remediated normally. +# dormant — node exists but is expected offline: the observer keeps its +# last-known world state and emits NO liveness events for it; the +# supervisor generates NO actions for it. See ARCHITEKTURA.md. nodes: saturn: + status: active roles: - control - development piha: + status: active roles: - infra - monitoring - services: - - node-agent - - ha-diag-agent - - brain-watchdog - - vikunja # Task management (vikunja + postgres), public via npm - - kb-postgres # KB spine: Postgres 16 + pgvector, port 5433 (always-on) - - llm-gateway # FastAPI router -> Ollama @ SOLARIA (Tailscale-internal :8080) - - node_exporter # per-host (also vps) — textfile collector for kb-ingest metrics solaria: + # Powered off ~16 h/day BY DESIGN (on-demand compute desktop): one daily + # offline/online liveness cycle is expected, not a fault. Nothing that + # must run 24/7 may live here. + status: active roles: - compute - ai - services: - - node-agent vps: + status: active roles: - edge - ingress - control-plane - services: - # Repo-managed GitOps services (hosts/vps/services.yaml is authoritative) - - node-agent - - control-plane # executor, observer, supervisor, operator-ui - - node_exporter - - fleet-prometheus # Fleet liveness source of truth (Tailscale-internal) - - stability-agent - - npm # Nginx Proxy Manager — public ingress, TLS termination - - outline # Team wiki (outline + postgres + redis) - - joplin # Note sync server (joplin-server + postgres) - - ai-cluster # AI workers: codex-worker, openclaw, planner-worker, - # service-ops-worker, redis, mosquitto - - gokapi # Public file-share (Firefox Send alt), share.okit.pl via npm@VPS, - # local disk storage, E2E encryption ON — separate from private Nextcloud + + lustro: + # MagicMirror Raspberry Pi — full monitored node: runs node-agent and + # ships events to the control plane (recon F20.11). + status: active + roles: + - edge + - display chelsty-infra: + status: dormant + dormant_reason: "site hardware down since ~2026-06-01; will be revived later" + dormant_since: "2026-06-01" site: chelsty roles: - remote @@ -90,15 +94,15 @@ nodes: intermittent: true home_automation: offline_operation_required: true - services: - - zigbee2mqtt - - mosquitto coordinator: model: SLZB-06U connection: network usb: false chelsty-ha: + status: dormant + dormant_reason: "site hardware down since ~2026-06-01; will be revived later" + dormant_since: "2026-06-01" site: chelsty roles: - remote @@ -108,11 +112,3 @@ nodes: intermittent: true home_automation: offline_operation_required: true - services: - - homeassistant - - lustro: - roles: - - edge - services: - - node-agent diff --git a/scripts/observer/observer.py b/scripts/observer/observer.py index 86ebff7..9f7402b 100644 --- a/scripts/observer/observer.py +++ b/scripts/observer/observer.py @@ -235,6 +235,18 @@ class Observer: } } self.inventory = self._load_inventory() + # Nodes declared `status: dormant` in inventory/topology.yaml: expected + # offline (e.g. chelsty site, hardware down since ~2026-06-01). The + # observer keeps their last-known world state but never reclassifies + # liveness for them, so no node_offline/node_stale/node_online events + # are emitted while a node is dormant. See docs/architecture/ARCHITEKTURA.md. + self.dormant_nodes = { + name for name, info in self.inventory["nodes"].items() + if info.get("status") == "dormant" + } + if self.dormant_nodes: + logger.info("Dormant nodes (no liveness tracking): %s", + sorted(self.dormant_nodes)) self._ensure_dirs() self._load_checkpoint() # Persistent SHADOW_LIVENESS_MISMATCH sink (survives container recreate). @@ -274,9 +286,12 @@ class Observer: with open(INVENTORY_TOPOLOGY, "r") as f: topo = yaml.safe_load(f) for node_name, node_info in topo.get("nodes", {}).items(): + node_info = node_info or {} inventory["nodes"][node_name] = { "roles": node_info.get("roles", []), - "connectivity": node_info.get("connectivity", {}) + "connectivity": node_info.get("connectivity", {}), + # topology node status: active (default) | dormant + "status": node_info.get("status", "active"), } # Load service assignments from hosts files @@ -558,6 +573,9 @@ class Observer: # when disabled/unreachable — fail-open, event liveness is authoritative. prom_liveness_map = self._query_prometheus_liveness() for node_name, node_info in self.world_state["nodes"].items(): + if node_name in self.dormant_nodes: + # Dormant node: keep last-known state frozen, emit nothing. + continue roles = (node_info.get("roles") or self.inventory["nodes"].get(node_name, {}).get("roles", [])) liveness = compute_liveness( diff --git a/services/control-plane/src/supervisor.py b/services/control-plane/src/supervisor.py index 3515b92..85c6da2 100644 --- a/services/control-plane/src/supervisor.py +++ b/services/control-plane/src/supervisor.py @@ -129,6 +129,13 @@ class Supervisor: def __init__(self): self.desired_state = {"services": {}} self.actual_state = {"services": {}, "nodes": {}, "incidents": {}} + # Nodes declared `status: dormant` in inventory/topology.yaml (e.g. the + # chelsty site, hardware down since ~2026-06-01). The supervisor + # generates NO actions of any kind for them: their services are excluded + # from desired state (which also auto-cancels their stale pending + # actions), and node/HA events from them are not routed to alerts. + # See docs/architecture/ARCHITEKTURA.md. + self.dormant_nodes: set = set() # In-memory set of already-routed HA event IDs; prevents re-processing # on each reconcile cycle. Grows to at most ~hundreds of entries/day. self._ha_processed_event_ids: set = set() @@ -179,6 +186,21 @@ class Supervisor: # State loading # ------------------------------------------------------------------ + def _load_dormant_nodes(self): + """Refresh the set of dormant nodes from inventory/topology.yaml.""" + dormant = set() + topo_file = REPO_ROOT / "inventory" / "topology.yaml" + try: + if topo_file.exists(): + with open(topo_file, "r") as f: + topo = yaml.safe_load(f) or {} + for name, info in (topo.get("nodes") or {}).items(): + if (info or {}).get("status") == "dormant": + dormant.add(name) + except Exception as e: + logger.error(f"Failed to load dormant nodes from {topo_file}: {e}") + self.dormant_nodes = dormant + def _load_desired_state(self): services = {} hosts_dir = REPO_ROOT / "hosts" @@ -194,6 +216,11 @@ class Supervisor: with open(svc_file, "r") as f: data = yaml.safe_load(f) host_name = data.get("host") + if host_name in self.dormant_nodes: + logger.debug( + f"Skipping desired services of {host_name}: node dormant" + ) + continue for svc_name, svc_info in data.get("services", {}).items(): svc_info = svc_info or {} # monitor: false — service is documented as desired but @@ -299,6 +326,7 @@ class Supervisor: except Exception as e: logger.error(f"Failed to touch heartbeat file: {e}") + self._load_dormant_nodes() self._load_desired_state() if not self._load_actual_state(): return # world state unreadable this cycle — skip to avoid false drift @@ -334,7 +362,7 @@ class Supervisor: # 3. Generate node-level recommendations (disk pressure) for node_name, node_info in self.actual_state["nodes"].items(): - if node_name in NO_DISK_CLEANUP_NODES: + if node_name in NO_DISK_CLEANUP_NODES or node_name in self.dormant_nodes: continue if node_info.get("disk_pressure") == "high": self._generate_disk_cleanup_recommendation(node_name) @@ -582,6 +610,9 @@ class Supervisor: node = event.get("node", "") if not node: return + if node in self.dormant_nodes: + logger.debug(f"Suppressing {event_type} on {node}: node dormant") + return if event_type in HA_CONTAINER_RESTART_EVENTS: if self._is_ha_in_transition(node): @@ -772,6 +803,9 @@ class Supervisor: node = event.get("node") or event.get("payload", {}).get("affected_node") if not node: return + if node in self.dormant_nodes: + logger.debug(f"Suppressing {event_type} on {node}: node dormant") + return action_id = f"alert-{event_type.replace('_', '-')}-{node}" diff --git a/services/control-plane/tests/test_dormant_nodes.py b/services/control-plane/tests/test_dormant_nodes.py new file mode 100644 index 0000000..0e85234 --- /dev/null +++ b/services/control-plane/tests/test_dormant_nodes.py @@ -0,0 +1,197 @@ +"""Dormant-node handling (topology `status: dormant`, etap 0 truth cleanup). + +Contract (docs/architecture/ARCHITEKTURA.md): a dormant node keeps its +last-known world state, the observer emits NO liveness events for it, and the +supervisor generates NO actions of any kind for it. +""" +from __future__ import annotations + +import json +import sys +import time +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent / "src")) +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent / "scripts")) + +import observer.observer as obs_mod +from observer.observer import Observer +import supervisor as supervisor_module +from supervisor import Supervisor + +TOPOLOGY_YAML = ( + "nodes:\n" + " piha:\n" + " status: active\n" + " roles: [infra]\n" + " oldnode:\n" + " status: dormant\n" + " dormant_reason: \"hardware down\"\n" + " roles: [remote]\n" +) + + +# --------------------------------------------------------------------------- +# Observer +# --------------------------------------------------------------------------- + +@pytest.fixture +def obs_paths(tmp_path, monkeypatch): + world = tmp_path / "world" + state = tmp_path / "state" + events = tmp_path / "events" + logs = tmp_path / "logs" + repo = tmp_path / "repo" + for d in (world, state, events, logs, repo / "inventory", repo / "hosts"): + d.mkdir(parents=True, exist_ok=True) + (repo / "inventory" / "topology.yaml").write_text(TOPOLOGY_YAML) + monkeypatch.setattr(obs_mod, "WORLD_DIR", world) + monkeypatch.setattr(obs_mod, "STATE_DIR", state) + monkeypatch.setattr(obs_mod, "EVENTS_DIR", events) + monkeypatch.setattr(obs_mod, "LOGS_DIR", logs) + monkeypatch.setattr(obs_mod, "INVENTORY_TOPOLOGY", repo / "inventory" / "topology.yaml") + monkeypatch.setattr(obs_mod, "REPO_ROOT", repo) + monkeypatch.setattr(obs_mod, "FAILED_EVENTS_DIR", state / "observer_failed_events") + monkeypatch.setattr(obs_mod, "OBSERVER_STATE_FILE", state / "observer_checkpoint.json") + monkeypatch.setattr(obs_mod, "PROM_SHADOW_URL", "") + monkeypatch.setattr(obs_mod, "SHADOW_LOG_DIR", logs / "observer") + return tmp_path + + +def test_observer_loads_dormant_set_from_topology(obs_paths): + obs = Observer() + assert obs.dormant_nodes == {"oldnode"} + + +def test_observer_freezes_dormant_node_and_emits_nothing(obs_paths): + """A dormant node whose heartbeats stopped long ago must NOT be + reclassified (stale→dead would normally fire node_offline).""" + obs = Observer() + stale_age = int(time.time()) - 999999 + obs.world_state["nodes"] = { + "oldnode": {"status": "stale", "liveness": "stale", "last_seen": stale_age}, + "piha": {"status": "unknown", "last_seen": int(time.time())}, + } + + obs._prune_stale_world() + + frozen = obs.world_state["nodes"]["oldnode"] + assert frozen["liveness"] == "stale" # unchanged, despite dead-level age + assert frozen["status"] == "stale" # unchanged + # No liveness event file was written for the dormant node. + assert not list((obs_paths / "events").glob("**/evt-oldnode-*")) + # The active node is still processed normally. + assert obs.world_state["nodes"]["piha"]["liveness"] == "fresh" + + +def test_observer_keeps_dormant_node_in_world_state(obs_paths): + """Dormant nodes are in the inventory, so pruning must not remove them.""" + obs = Observer() + obs.world_state["nodes"] = {"oldnode": {"status": "offline", "last_seen": 1}} + + obs._prune_stale_world() + + assert "oldnode" in obs.world_state["nodes"] + + +# --------------------------------------------------------------------------- +# Supervisor +# --------------------------------------------------------------------------- + +@pytest.fixture +def sup(tmp_path, monkeypatch): + actions = tmp_path / "actions" + events = tmp_path / "events" + world = tmp_path / "world" + state = tmp_path / "state" + repo = tmp_path / "repo" + for d in (actions, events, world, state, repo / "inventory", repo / "hosts" / "oldnode"): + d.mkdir(parents=True, exist_ok=True) + (repo / "inventory" / "topology.yaml").write_text(TOPOLOGY_YAML) + (repo / "hosts" / "oldnode" / "services.yaml").write_text( + "host: oldnode\nservices:\n some-svc:\n role: x\n" + ) + monkeypatch.setattr(supervisor_module, "ACTIONS_DIR", actions) + monkeypatch.setattr(supervisor_module, "EVENTS_DIR", events) + monkeypatch.setattr(supervisor_module, "WORLD_DIR", world) + monkeypatch.setattr(supervisor_module, "REPO_ROOT", repo) + s = Supervisor() + return s + + +def _pending(tmp_path): + return list((tmp_path / "actions" / "pending").glob("*.json")) + + +def test_supervisor_loads_dormant_set(sup): + sup._load_dormant_nodes() + assert sup.dormant_nodes == {"oldnode"} + + +def test_dormant_services_excluded_from_desired_state(sup): + sup._load_dormant_nodes() + sup._load_desired_state() + assert "oldnode/some-svc" not in sup.desired_state["services"] + + +def test_reconcile_generates_no_actions_for_dormant_node(sup, tmp_path): + """Full reconcile: missing service + high disk pressure on a dormant node + must produce zero pending actions.""" + world = tmp_path / "world" + (world / "nodes.json").write_text(json.dumps( + {"oldnode": {"status": "offline", "disk_pressure": "high"}} + )) + (world / "services.json").write_text("{}") + (world / "incidents.json").write_text("{}") + + sup.reconcile() + + assert _pending(tmp_path) == [] + + +def test_node_event_from_dormant_node_not_routed(sup, tmp_path): + sup._load_dormant_nodes() + event = { + "id": "evt-oldnode-1-node_offline-node", + "type": "node_offline", + "node": "oldnode", + "payload": {"affected_node": "oldnode"}, + } + (tmp_path / "events" / f"{event['id']}.json").write_text(json.dumps(event)) + + sup._process_ha_events() + + assert _pending(tmp_path) == [] + + +def test_ha_event_from_dormant_node_not_routed(sup, tmp_path): + sup._load_dormant_nodes() + event = { + "id": "evt-oldnode-2-ha_entity_unavailable_long-homeassistant", + "type": "ha_entity_unavailable_long", + "node": "oldnode", + "payload": {}, + } + (tmp_path / "events" / f"{event['id']}.json").write_text(json.dumps(event)) + + sup._process_ha_events() + + assert _pending(tmp_path) == [] + + +def test_active_node_events_still_routed(sup, tmp_path): + """Sanity: the dormant guard must not swallow active-node events.""" + sup._load_dormant_nodes() + event = { + "id": "evt-piha-3-node_stale-node", + "type": "node_stale", + "node": "piha", + "payload": {"affected_node": "piha"}, + } + (tmp_path / "events" / f"{event['id']}.json").write_text(json.dumps(event)) + + sup._process_ha_events() + + assert len(_pending(tmp_path)) == 1