"""Shared node-liveness logic — single source of truth for freshness. This module owns the fresh/stale/dead state machine and its TTL thresholds. It is imported by every component that has to decide whether a node is alive: - scripts/observer/observer.py authoritative writer of world/nodes.json - services/control-plane/src/operator_ui.py read-time safety net (panel) - services/agent-system/webui/web.py read-time safety net (panel); this file is bind-mounted into the webui image at /app/liveness.py — see services/agent-system/docker-compose.yml. Why a single module: the previous bug ("dead node shown NOMINAL") was a missing TTL. Thresholds and the tier logic must live in exactly one place so the observer and both panels can never disagree about what "dead" means. Heartbeat interval is 60 s on every node (node-agent CHECK_INTERVAL=60), so the fresh ceiling is 3× the interval. If the interval ever changes, raise the thresholds here proportionally and nothing else needs to move. """ from __future__ import annotations import os import time from datetime import datetime # --- Liveness tiers --------------------------------------------------------- FRESH = "fresh" STALE = "stale" DEAD = "dead" UNKNOWN = "unknown" # --- TTL thresholds (seconds) ---------------------------------------------- # Always-on nodes (vps/piha/solaria/lustro): heartbeat every 60 s. # fresh ≤ 180 s → 3 missed beats tolerated (+ observer cycle + rsync latency) # dead > 600 s → 10 missed beats == really gone DEFAULT_TTLS = { "fresh": int(os.getenv("LIVENESS_TTL_FRESH", "180")), "dead": int(os.getenv("LIVENESS_TTL_DEAD", "600")), } # Remote / LTE nodes (chelsty-*): intermittent uplink, events buffer locally # and ship on reconnect. Wider so a typical multi-minute LTE blip reads as # stale (degraded), but a genuine outage still escalates to dead within an hour. REMOTE_TTLS = { "fresh": int(os.getenv("LIVENESS_REMOTE_TTL_FRESH", "900")), # 15 min "dead": int(os.getenv("LIVENESS_REMOTE_TTL_DEAD", "3600")), # 1 h } # A node uses REMOTE_TTLS if its topology role is in REMOTE_ROLES, or its name # is in REMOTE_NODES (fallback when roles are absent from the node record). REMOTE_ROLES = {"remote"} REMOTE_NODES = {"chelsty-infra", "chelsty-ha"} # --- Mappings --------------------------------------------------------------- # Persisted node.status string written by the observer for each tier. # UNKNOWN intentionally has no mapping: never overwrite status when we have no # timing data (last_seen missing) — we don't guess a node dead. _LIVENESS_TO_STATUS = {FRESH: "online", STALE: "stale", DEAD: "offline"} # UI health bucket per liveness tier and per persisted status. _LIVENESS_HEALTH = {FRESH: "nominal", STALE: "degraded", DEAD: "error", UNKNOWN: "unknown"} _STATUS_HEALTH = {"online": "nominal", "stale": "degraded", "offline": "error"} # Severity ordering for "take the worse of two health buckets". _HEALTH_RANK = {"nominal": 0, "unknown": 1, "degraded": 2, "error": 3} def parse_ts(ts): """Epoch float from int/float or ISO-8601 string; 0.0 on None/garbage. node-agent emits int(time.time()); events.py / stability-agent emit ISO strings. Both land in last_seen, so all arithmetic goes through here. """ if ts is None: return 0.0 if isinstance(ts, (int, float)): return float(ts) try: return datetime.fromisoformat(str(ts).replace("Z", "+00:00")).timestamp() except Exception: return 0.0 def ttls_for(node_name=None, roles=None): """Pick the TTL set for a node by its name or topology roles.""" roles = set(roles or ()) if (node_name in REMOTE_NODES) or (roles & REMOTE_ROLES): return REMOTE_TTLS return DEFAULT_TTLS def compute_liveness(last_seen, now=None, ttls=None): """Classify a node from age = now - last_seen. Returns FRESH / STALE / DEAD, or UNKNOWN when last_seen is missing or unparseable (no data → no opinion; callers must not flip status on UNKNOWN). """ ttls = ttls or DEFAULT_TTLS ts = parse_ts(last_seen) if ts <= 0: return UNKNOWN age = (now if now is not None else time.time()) - ts if age <= ttls["fresh"]: return FRESH if age <= ttls["dead"]: return STALE return DEAD def liveness_to_status(liveness): """Persisted node.status for a tier, or None for UNKNOWN (leave as-is).""" return _LIVENESS_TO_STATUS.get(liveness) def _worse(a, b): """Return the more severe of two UI health buckets.""" return a if _HEALTH_RANK.get(a, 1) >= _HEALTH_RANK.get(b, 1) else b def node_health(info, name=None, now=None): """UI health bucket for a node record, applying the read-time freshness net. Takes the WORSE of (a) the observer-persisted status and (b) the freshness derived from last_seen. Rationale: - if the observer is alive it already wrote the right status; freshness agrees and this is a no-op. - if the observer itself is dead, nodes.json freezes with stale "online" statuses — but last_seen keeps ageing, so freshness still surfaces the outage. This is the whole point of duplicating the check at read time. UNKNOWN freshness (no last_seen) never downgrades a node. """ stored_health = _STATUS_HEALTH.get(info.get("status", "unknown"), "unknown") liveness = compute_liveness( info.get("last_seen"), now=now, ttls=ttls_for(name, info.get("roles", [])) ) if liveness == UNKNOWN: health = stored_health else: health = _worse(stored_health, _LIVENESS_HEALTH[liveness]) if health == "nominal" and info.get("disk_pressure") == "high": return "degraded" return health def degrade_for_node(service_health, node_liveness): """Cascade node liveness onto a service's health (read-time, panel side). A service cannot be healthier than the node it runs on: a service on a dead node is error, on a stale node at least degraded. """ if node_liveness == DEAD: return "error" if node_liveness == STALE: return _worse(service_health, "degraded") return service_health