homelab-codex-ws/services/control-plane/src/liveness.py

"""Shared node-liveness logic — single source of truth for freshness.

This module owns the fresh/stale/dead state machine and its TTL thresholds.
It is imported by every component that has to decide whether a node is alive:

  - scripts/observer/observer.py          authoritative writer of world/nodes.json
  - services/control-plane/src/operator_ui.py    read-time safety net (panel)
  - services/agent-system/webui/web.py           read-time safety net (panel);
        this file is bind-mounted into the webui image at /app/liveness.py —
        see services/agent-system/docker-compose.yml.

Why a single module: the previous bug ("dead node shown NOMINAL") was a missing
TTL.  Thresholds and the tier logic must live in exactly one place so the
observer and both panels can never disagree about what "dead" means.

Heartbeat interval is 60 s on every node (node-agent CHECK_INTERVAL=60), so the
fresh ceiling is 3× the interval.  If the interval ever changes, raise the
thresholds here proportionally and nothing else needs to move.
"""
from __future__ import annotations

import os
import time
from datetime import datetime

# --- Liveness tiers ---------------------------------------------------------
FRESH = "fresh"
STALE = "stale"
DEAD = "dead"
UNKNOWN = "unknown"

# --- TTL thresholds (seconds) ----------------------------------------------
# Always-on nodes (vps/piha/solaria/lustro): heartbeat every 60 s.
#   fresh ≤ 180 s  → 3 missed beats tolerated (+ observer cycle + rsync latency)
#   dead  > 600 s  → 10 missed beats == really gone
DEFAULT_TTLS = {
    "fresh": int(os.getenv("LIVENESS_TTL_FRESH", "180")),
    "dead": int(os.getenv("LIVENESS_TTL_DEAD", "600")),
}
# Remote / LTE nodes (chelsty-*): intermittent uplink, events buffer locally
# and ship on reconnect.  Wider so a typical multi-minute LTE blip reads as
# stale (degraded), but a genuine outage still escalates to dead within an hour.
REMOTE_TTLS = {
    "fresh": int(os.getenv("LIVENESS_REMOTE_TTL_FRESH", "900")),   # 15 min
    "dead": int(os.getenv("LIVENESS_REMOTE_TTL_DEAD", "3600")),    # 1 h
}
# A node uses REMOTE_TTLS if its topology role is in REMOTE_ROLES, or its name
# is in REMOTE_NODES (fallback when roles are absent from the node record).
REMOTE_ROLES = {"remote"}
REMOTE_NODES = {"chelsty-infra", "chelsty-ha"}

# --- Mappings ---------------------------------------------------------------
# Persisted node.status string written by the observer for each tier.
# UNKNOWN intentionally has no mapping: never overwrite status when we have no
# timing data (last_seen missing) — we don't guess a node dead.
_LIVENESS_TO_STATUS = {FRESH: "online", STALE: "stale", DEAD: "offline"}
# UI health bucket per liveness tier and per persisted status.
_LIVENESS_HEALTH = {FRESH: "nominal", STALE: "degraded", DEAD: "error", UNKNOWN: "unknown"}
_STATUS_HEALTH = {"online": "nominal", "stale": "degraded", "offline": "error"}
# Severity ordering for "take the worse of two health buckets".
_HEALTH_RANK = {"nominal": 0, "unknown": 1, "degraded": 2, "error": 3}


def parse_ts(ts):
    """Epoch float from int/float or ISO-8601 string; 0.0 on None/garbage.

    node-agent emits int(time.time()); events.py / stability-agent emit ISO
    strings.  Both land in last_seen, so all arithmetic goes through here.
    """
    if ts is None:
        return 0.0
    if isinstance(ts, (int, float)):
        return float(ts)
    try:
        return datetime.fromisoformat(str(ts).replace("Z", "+00:00")).timestamp()
    except Exception:
        return 0.0


def ttls_for(node_name=None, roles=None):
    """Pick the TTL set for a node by its name or topology roles."""
    roles = set(roles or ())
    if (node_name in REMOTE_NODES) or (roles & REMOTE_ROLES):
        return REMOTE_TTLS
    return DEFAULT_TTLS


def compute_liveness(last_seen, now=None, ttls=None):
    """Classify a node from age = now - last_seen.

    Returns FRESH / STALE / DEAD, or UNKNOWN when last_seen is missing or
    unparseable (no data → no opinion; callers must not flip status on UNKNOWN).
    """
    ttls = ttls or DEFAULT_TTLS
    ts = parse_ts(last_seen)
    if ts <= 0:
        return UNKNOWN
    age = (now if now is not None else time.time()) - ts
    if age <= ttls["fresh"]:
        return FRESH
    if age <= ttls["dead"]:
        return STALE
    return DEAD


def liveness_to_status(liveness):
    """Persisted node.status for a tier, or None for UNKNOWN (leave as-is)."""
    return _LIVENESS_TO_STATUS.get(liveness)


def _worse(a, b):
    """Return the more severe of two UI health buckets."""
    return a if _HEALTH_RANK.get(a, 1) >= _HEALTH_RANK.get(b, 1) else b


def node_health(info, name=None, now=None):
    """UI health bucket for a node record, applying the read-time freshness net.

    Takes the WORSE of (a) the observer-persisted status and (b) the freshness
    derived from last_seen.  Rationale:
      - if the observer is alive it already wrote the right status; freshness
        agrees and this is a no-op.
      - if the observer itself is dead, nodes.json freezes with stale "online"
        statuses — but last_seen keeps ageing, so freshness still surfaces the
        outage.  This is the whole point of duplicating the check at read time.
    UNKNOWN freshness (no last_seen) never downgrades a node.
    """
    stored_health = _STATUS_HEALTH.get(info.get("status", "unknown"), "unknown")
    liveness = compute_liveness(
        info.get("last_seen"), now=now, ttls=ttls_for(name, info.get("roles", []))
    )
    if liveness == UNKNOWN:
        health = stored_health
    else:
        health = _worse(stored_health, _LIVENESS_HEALTH[liveness])
    if health == "nominal" and info.get("disk_pressure") == "high":
        return "degraded"
    return health


def degrade_for_node(service_health, node_liveness):
    """Cascade node liveness onto a service's health (read-time, panel side).

    A service cannot be healthier than the node it runs on: a service on a dead
    node is error, on a stale node at least degraded.
    """
    if node_liveness == DEAD:
        return "error"
    if node_liveness == STALE:
        return _worse(service_health, "degraded")
    return service_health