feat(observer): mark nodes offline when last_seen exceeds TTL

Nodes that crash or lose connectivity without emitting node_offline
stay online in world state indefinitely. _prune_stale_world() now
flips any online node to offline if its last_seen is older than
NODE_OFFLINE_TTL_SECS (default 300 s = 5× the 60 s heartbeat interval).
Nodes with last_seen=None (never reported) and already-offline nodes
are left unchanged. Five new tests cover all branches.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

scripts/observer/observer.py

@@ -44,6 +44,9 @@ WORLD_DIR = Path(RUNTIME_PATH) / "world"
 OBSERVER_STATE_FILE = STATE_DIR / "observer_checkpoint.json"
 FAILED_EVENTS_DIR = STATE_DIR / "observer_failed_events"
 
+# node_health events arrive every 60 s; 5 missed beats → declare offline
+NODE_OFFLINE_TTL_SECS = int(os.getenv("NODE_OFFLINE_TTL_SECS", "300"))
+
 REPO_ROOT = Path(__file__).parent.parent.parent
 INVENTORY_TOPOLOGY = REPO_ROOT / "inventory" / "topology.yaml"
 
@@ -224,6 +227,22 @@ class Observer:
 
         now = time.time()
 
+        # Mark nodes offline when last_seen exceeds NODE_OFFLINE_TTL_SECS.
+        # Handles crash/network-partition cases where node_offline is never emitted.
+        for node_name, node_info in self.world_state["nodes"].items():
+            if node_info.get("status") != "online":
+                continue
+            last_seen = _parse_ts(node_info.get("last_seen"))
+            if last_seen == 0.0:
+                continue  # Never reported — keep status unknown, don't flip
+            age = now - last_seen
+            if age > NODE_OFFLINE_TTL_SECS:
+                logger.warning(
+                    "Node %s last seen %ds ago (> TTL %ds) — marking offline",
+                    node_name, int(age), NODE_OFFLINE_TTL_SECS,
+                )
+                node_info["status"] = "offline"
+
         try:
             # Collect incident_ids currently referenced by any service entry.
             linked_ids: set = {

services/control-plane/tests/test_incident_lifecycle.py

@@ -377,3 +377,93 @@ def test_run_once_quarantines_bad_event_and_processes_next_for_same_node(tmp_pat
     assert not bad_event.exists()
     assert obs.world_state["nodes"]["lustro"]["status"] == "online"
     assert obs.node_checkpoints["lustro"] == str(good_event)
+
+
+# ---------------------------------------------------------------------------
+# 7. Node offline TTL — liveness detection via last_seen age
+# ---------------------------------------------------------------------------
+
+def test_prune_marks_online_node_offline_after_ttl(tmp_path):
+    """Node online with last_seen > NODE_OFFLINE_TTL_SECS must be flipped to offline."""
+    obs = _make_observer_simple(tmp_path)
+    import observer.observer as obs_mod
+
+    # Use "vps" — present in the test topology so it won't be pruned as stale.
+    obs.world_state["nodes"]["vps"] = {
+        "status": "online",
+        "last_seen": time.time() - (obs_mod.NODE_OFFLINE_TTL_SECS + 60),  # over TTL
+        "roles": [],
+    }
+
+    obs._prune_stale_world()
+
+    assert obs.world_state["nodes"]["vps"]["status"] == "offline"
+
+
+def test_prune_does_not_mark_node_offline_within_ttl(tmp_path):
+    """Node online with recent last_seen must stay online."""
+    obs = _make_observer_simple(tmp_path)
+    import observer.observer as obs_mod
+
+    obs.world_state["nodes"]["vps"] = {
+        "status": "online",
+        "last_seen": time.time() - (obs_mod.NODE_OFFLINE_TTL_SECS - 60),  # within TTL
+        "roles": [],
+    }
+
+    obs._prune_stale_world()
+
+    assert obs.world_state["nodes"]["vps"]["status"] == "online"
+
+
+def test_prune_does_not_flip_already_offline_node(tmp_path):
+    """Node already offline must not be changed by TTL logic."""
+    obs = _make_observer_simple(tmp_path)
+
+    obs.world_state["nodes"]["vps"] = {
+        "status": "offline",
+        "last_seen": time.time() - 9999,
+        "roles": [],
+    }
+
+    obs._prune_stale_world()
+
+    assert obs.world_state["nodes"]["vps"]["status"] == "offline"
+
+
+def test_prune_does_not_flip_never_seen_node(tmp_path):
+    """Node with last_seen=None must stay in its original status."""
+    obs = _make_observer_simple(tmp_path)
+
+    obs.world_state["nodes"]["vps"] = {
+        "status": "online",
+        "last_seen": None,
+        "roles": [],
+    }
+
+    obs._prune_stale_world()
+
+    # last_seen=None means we have no data — don't flip to offline
+    assert obs.world_state["nodes"]["vps"]["status"] == "online"
+
+
+def test_prune_marks_node_offline_with_iso_last_seen(tmp_path):
+    """TTL check must work when last_seen is an ISO-8601 string."""
+    obs = _make_observer_simple(tmp_path)
+    import observer.observer as obs_mod
+
+    from datetime import datetime, timezone
+    stale_ts = datetime.fromtimestamp(
+        time.time() - (obs_mod.NODE_OFFLINE_TTL_SECS + 120),
+        tz=timezone.utc,
+    ).isoformat()
+
+    obs.world_state["nodes"]["vps"] = {
+        "status": "online",
+        "last_seen": stale_ts,
+        "roles": [],
+    }
+
+    obs._prune_stale_world()
+
+    assert obs.world_state["nodes"]["vps"]["status"] == "offline"