5f1528e4ab
Fixes the "dead node shown NOMINAL" silent outage: node status was set only by events and never expired, so a node that crashed/lost connectivity stayed "online" forever (chelsty-infra was online for 16d, piha ~6d). The only thing that flipped status to offline was a node_offline event, which an unreachable node can never emit. Now node status is derived from freshness (now - last_seen), recomputed every observer cycle (incl. cycles with no new events): - always-on: fresh <=180s, stale 180-600s, dead >600s (3x the 60s heartbeat) - remote/LTE (chelsty-*): fresh <=900s, stale 900-3600s, dead >3600s Thresholds + tier logic live in ONE shared helper, services/control-plane/src/ liveness.py, imported by the observer and both operator UIs (bind-mounted into the agent-system webui image). No 3x copy. Transitions are not silent: the observer emits node_stale / node_offline / node_online (recovery) events tagged source=observer (skipped on re-ingest so they never reset last_seen), routed by the supervisor to alert_only actions. Read-time safety net: both UIs recompute liveness from last_seen at request time, so a stalled observer still surfaces dead nodes. Services inherit their node's liveness (cascade, variant B) without mutating services.json. Replaces the earlier binary NODE_OFFLINE_TTL_SECS flip. Tests: liveness unit tests, observer 3-state + transitions/recovery/baseline + self-event skip, operator_ui read-time net + cascade, supervisor node-event routing. 89 passed. docker compose config valid for both stacks. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
138 lines
4.6 KiB
Python
138 lines
4.6 KiB
Python
"""Unit tests for the shared liveness helper (thresholds + tier state machine)."""
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
|
|
|
import liveness
|
|
from liveness import (
|
|
compute_liveness,
|
|
ttls_for,
|
|
liveness_to_status,
|
|
node_health,
|
|
degrade_for_node,
|
|
parse_ts,
|
|
FRESH,
|
|
STALE,
|
|
DEAD,
|
|
UNKNOWN,
|
|
)
|
|
|
|
|
|
# --- parse_ts ---------------------------------------------------------------
|
|
|
|
def test_parse_ts_int_float_iso_none():
|
|
assert abs(parse_ts(1000) - 1000.0) < 0.001
|
|
assert abs(parse_ts(1000.5) - 1000.5) < 0.001
|
|
iso = "2026-06-01T00:00:00Z"
|
|
assert abs(parse_ts(iso) - datetime(2026, 6, 1, tzinfo=timezone.utc).timestamp()) < 1
|
|
assert parse_ts(None) == 0.0
|
|
assert parse_ts("garbage") == 0.0
|
|
|
|
|
|
# --- compute_liveness (default TTLs: fresh<=180, dead>600) ------------------
|
|
|
|
def test_compute_liveness_fresh():
|
|
assert compute_liveness(time.time() - 60) == FRESH
|
|
|
|
|
|
def test_compute_liveness_stale():
|
|
assert compute_liveness(time.time() - 300) == STALE
|
|
|
|
|
|
def test_compute_liveness_dead():
|
|
assert compute_liveness(time.time() - 700) == DEAD
|
|
|
|
|
|
def test_compute_liveness_boundaries():
|
|
now = 1_000_000
|
|
assert compute_liveness(now - 180, now=now) == FRESH # exactly fresh ceiling
|
|
assert compute_liveness(now - 181, now=now) == STALE
|
|
assert compute_liveness(now - 600, now=now) == STALE # exactly dead floor
|
|
assert compute_liveness(now - 601, now=now) == DEAD
|
|
|
|
|
|
def test_compute_liveness_unknown_on_missing():
|
|
assert compute_liveness(None) == UNKNOWN
|
|
assert compute_liveness("garbage") == UNKNOWN
|
|
assert compute_liveness(0) == UNKNOWN
|
|
|
|
|
|
def test_compute_liveness_iso_string():
|
|
iso = datetime.fromtimestamp(time.time() - 700, tz=timezone.utc).isoformat()
|
|
assert compute_liveness(iso) == DEAD
|
|
|
|
|
|
# --- ttls_for ---------------------------------------------------------------
|
|
|
|
def test_ttls_for_default_vs_remote():
|
|
assert ttls_for("vps", ["control-plane"]) == liveness.DEFAULT_TTLS
|
|
assert ttls_for("chelsty-infra", []) == liveness.REMOTE_TTLS # by name
|
|
assert ttls_for("anything", ["remote"]) == liveness.REMOTE_TTLS # by role
|
|
|
|
|
|
def test_remote_node_stale_where_default_would_be_dead():
|
|
now = 1_000_000
|
|
ts = now - 1200 # > default dead (600) and > remote fresh (900), < remote dead (3600)
|
|
assert compute_liveness(ts, now=now, ttls=ttls_for("chelsty-infra")) == STALE
|
|
assert compute_liveness(ts, now=now, ttls=ttls_for("vps")) == DEAD
|
|
|
|
|
|
# --- liveness_to_status -----------------------------------------------------
|
|
|
|
def test_liveness_to_status():
|
|
assert liveness_to_status(FRESH) == "online"
|
|
assert liveness_to_status(STALE) == "stale"
|
|
assert liveness_to_status(DEAD) == "offline"
|
|
assert liveness_to_status(UNKNOWN) is None
|
|
|
|
|
|
# --- node_health (read-time safety net) -------------------------------------
|
|
|
|
def test_node_health_fresh_online_is_nominal():
|
|
info = {"status": "online", "last_seen": time.time() - 30, "roles": []}
|
|
assert node_health(info, "vps") == "nominal"
|
|
|
|
|
|
def test_node_health_frozen_online_but_dead_is_error():
|
|
"""Observer dead → status frozen 'online' but last_seen old → error (the net)."""
|
|
info = {"status": "online", "last_seen": time.time() - 700, "roles": []}
|
|
assert node_health(info, "vps") == "error"
|
|
|
|
|
|
def test_node_health_stale_is_degraded():
|
|
info = {"status": "online", "last_seen": time.time() - 300, "roles": []}
|
|
assert node_health(info, "vps") == "degraded"
|
|
|
|
|
|
def test_node_health_unknown_freshness_does_not_downgrade():
|
|
info = {"status": "online", "last_seen": None, "roles": []}
|
|
assert node_health(info, "vps") == "nominal"
|
|
|
|
|
|
def test_node_health_disk_pressure_degrades_fresh_node():
|
|
info = {"status": "online", "last_seen": time.time() - 30,
|
|
"roles": [], "disk_pressure": "high"}
|
|
assert node_health(info, "vps") == "degraded"
|
|
|
|
|
|
def test_node_health_takes_worse_of_status_and_freshness():
|
|
# status already offline, freshness fresh → still error (worse wins)
|
|
info = {"status": "offline", "last_seen": time.time() - 5, "roles": []}
|
|
assert node_health(info, "vps") == "error"
|
|
|
|
|
|
# --- degrade_for_node (service cascade) -------------------------------------
|
|
|
|
def test_degrade_for_node():
|
|
assert degrade_for_node("nominal", DEAD) == "error"
|
|
assert degrade_for_node("nominal", STALE) == "degraded"
|
|
assert degrade_for_node("nominal", FRESH) == "nominal"
|
|
# never upgrades a service that is already worse
|
|
assert degrade_for_node("error", STALE) == "error"
|
|
assert degrade_for_node("error", FRESH) == "error"
|