5f1528e4ab
Fixes the "dead node shown NOMINAL" silent outage: node status was set only by events and never expired, so a node that crashed/lost connectivity stayed "online" forever (chelsty-infra was online for 16d, piha ~6d). The only thing that flipped status to offline was a node_offline event, which an unreachable node can never emit. Now node status is derived from freshness (now - last_seen), recomputed every observer cycle (incl. cycles with no new events): - always-on: fresh <=180s, stale 180-600s, dead >600s (3x the 60s heartbeat) - remote/LTE (chelsty-*): fresh <=900s, stale 900-3600s, dead >3600s Thresholds + tier logic live in ONE shared helper, services/control-plane/src/ liveness.py, imported by the observer and both operator UIs (bind-mounted into the agent-system webui image). No 3x copy. Transitions are not silent: the observer emits node_stale / node_offline / node_online (recovery) events tagged source=observer (skipped on re-ingest so they never reset last_seen), routed by the supervisor to alert_only actions. Read-time safety net: both UIs recompute liveness from last_seen at request time, so a stalled observer still surfaces dead nodes. Services inherit their node's liveness (cascade, variant B) without mutating services.json. Replaces the earlier binary NODE_OFFLINE_TTL_SECS flip. Tests: liveness unit tests, observer 3-state + transitions/recovery/baseline + self-event skip, operator_ui read-time net + cascade, supervisor node-event routing. 89 passed. docker compose config valid for both stacks. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
78 lines
2.9 KiB
Python
78 lines
2.9 KiB
Python
"""Read-time liveness safety net + service cascade in the operator UI backend."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
|
|
|
import operator_ui
|
|
|
|
|
|
def _setup_world(tmp_path, monkeypatch, nodes, services):
|
|
world = tmp_path / "world"
|
|
world.mkdir(parents=True, exist_ok=True)
|
|
(world / "nodes.json").write_text(json.dumps(nodes))
|
|
(world / "services.json").write_text(json.dumps(services))
|
|
monkeypatch.setattr(operator_ui, "WORLD_DIR", world)
|
|
|
|
|
|
def test_current_nodes_fresh_is_nominal(tmp_path, monkeypatch):
|
|
_setup_world(tmp_path, monkeypatch,
|
|
{"vps": {"status": "online", "last_seen": time.time() - 30, "roles": []}},
|
|
{})
|
|
nodes = operator_ui.current_nodes()
|
|
assert nodes[0]["health"] == "nominal"
|
|
|
|
|
|
def test_current_nodes_frozen_online_but_dead_is_error(tmp_path, monkeypatch):
|
|
"""The bug: observer wrote 'online' but last_seen is ancient → must be error."""
|
|
_setup_world(tmp_path, monkeypatch,
|
|
{"chelsty-infra": {"status": "online",
|
|
"last_seen": time.time() - 16 * 86400,
|
|
"roles": ["remote"]}},
|
|
{})
|
|
nodes = operator_ui.current_nodes()
|
|
assert nodes[0]["health"] == "error"
|
|
|
|
|
|
def test_current_nodes_stale_is_degraded(tmp_path, monkeypatch):
|
|
_setup_world(tmp_path, monkeypatch,
|
|
{"piha": {"status": "online", "last_seen": time.time() - 300, "roles": []}},
|
|
{})
|
|
nodes = operator_ui.current_nodes()
|
|
assert nodes[0]["health"] == "degraded"
|
|
|
|
|
|
def test_service_cascade_dead_node_makes_service_error(tmp_path, monkeypatch):
|
|
"""A 'healthy' service on a dead node must read error (cascade, read-time)."""
|
|
_setup_world(
|
|
tmp_path, monkeypatch,
|
|
{"piha": {"status": "online", "last_seen": time.time() - 700, "roles": []}},
|
|
{"piha/vikunja": {"node": "piha", "service": "vikunja", "status": "healthy"}},
|
|
)
|
|
svcs = operator_ui.current_services()
|
|
assert svcs[0]["health"] == "error"
|
|
|
|
|
|
def test_service_cascade_stale_node_makes_service_degraded(tmp_path, monkeypatch):
|
|
_setup_world(
|
|
tmp_path, monkeypatch,
|
|
{"piha": {"status": "online", "last_seen": time.time() - 300, "roles": []}},
|
|
{"piha/vikunja": {"node": "piha", "service": "vikunja", "status": "healthy"}},
|
|
)
|
|
svcs = operator_ui.current_services()
|
|
assert svcs[0]["health"] == "degraded"
|
|
|
|
|
|
def test_service_on_fresh_node_stays_nominal(tmp_path, monkeypatch):
|
|
_setup_world(
|
|
tmp_path, monkeypatch,
|
|
{"vps": {"status": "online", "last_seen": time.time() - 30, "roles": []}},
|
|
{"vps/outline": {"node": "vps", "service": "outline", "status": "healthy"}},
|
|
)
|
|
svcs = operator_ui.current_services()
|
|
assert svcs[0]["health"] == "nominal"
|