78 lines
2.9 KiB
Python
78 lines
2.9 KiB
Python
|
"""Read-time liveness safety net + service cascade in the operator UI backend."""
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import json
|
||
|
|
import sys
|
||
|
|
import time
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
||
|
|
|
||
|
|
import operator_ui
|
||
|
|
|
||
|
|
|
||
|
|
def _setup_world(tmp_path, monkeypatch, nodes, services):
|
||
|
|
world = tmp_path / "world"
|
||
|
|
world.mkdir(parents=True, exist_ok=True)
|
||
|
|
(world / "nodes.json").write_text(json.dumps(nodes))
|
||
|
|
(world / "services.json").write_text(json.dumps(services))
|
||
|
|
monkeypatch.setattr(operator_ui, "WORLD_DIR", world)
|
||
|
|
|
||
|
|
|
||
|
|
def test_current_nodes_fresh_is_nominal(tmp_path, monkeypatch):
|
||
|
|
_setup_world(tmp_path, monkeypatch,
|
||
|
|
{"vps": {"status": "online", "last_seen": time.time() - 30, "roles": []}},
|
||
|
|
{})
|
||
|
|
nodes = operator_ui.current_nodes()
|
||
|
|
assert nodes[0]["health"] == "nominal"
|
||
|
|
|
||
|
|
|
||
|
|
def test_current_nodes_frozen_online_but_dead_is_error(tmp_path, monkeypatch):
|
||
|
|
"""The bug: observer wrote 'online' but last_seen is ancient → must be error."""
|
||
|
|
_setup_world(tmp_path, monkeypatch,
|
||
|
|
{"chelsty-infra": {"status": "online",
|
||
|
|
"last_seen": time.time() - 16 * 86400,
|
||
|
|
"roles": ["remote"]}},
|
||
|
|
{})
|
||
|
|
nodes = operator_ui.current_nodes()
|
||
|
|
assert nodes[0]["health"] == "error"
|
||
|
|
|
||
|
|
|
||
|
|
def test_current_nodes_stale_is_degraded(tmp_path, monkeypatch):
|
||
|
|
_setup_world(tmp_path, monkeypatch,
|
||
|
|
{"piha": {"status": "online", "last_seen": time.time() - 300, "roles": []}},
|
||
|
|
{})
|
||
|
|
nodes = operator_ui.current_nodes()
|
||
|
|
assert nodes[0]["health"] == "degraded"
|
||
|
|
|
||
|
|
|
||
|
|
def test_service_cascade_dead_node_makes_service_error(tmp_path, monkeypatch):
|
||
|
|
"""A 'healthy' service on a dead node must read error (cascade, read-time)."""
|
||
|
|
_setup_world(
|
||
|
|
tmp_path, monkeypatch,
|
||
|
|
{"piha": {"status": "online", "last_seen": time.time() - 700, "roles": []}},
|
||
|
|
{"piha/vikunja": {"node": "piha", "service": "vikunja", "status": "healthy"}},
|
||
|
|
)
|
||
|
|
svcs = operator_ui.current_services()
|
||
|
|
assert svcs[0]["health"] == "error"
|
||
|
|
|
||
|
|
|
||
|
|
def test_service_cascade_stale_node_makes_service_degraded(tmp_path, monkeypatch):
|
||
|
|
_setup_world(
|
||
|
|
tmp_path, monkeypatch,
|
||
|
|
{"piha": {"status": "online", "last_seen": time.time() - 300, "roles": []}},
|
||
|
|
{"piha/vikunja": {"node": "piha", "service": "vikunja", "status": "healthy"}},
|
||
|
|
)
|
||
|
|
svcs = operator_ui.current_services()
|
||
|
|
assert svcs[0]["health"] == "degraded"
|
||
|
|
|
||
|
|
|
||
|
|
def test_service_on_fresh_node_stays_nominal(tmp_path, monkeypatch):
|
||
|
|
_setup_world(
|
||
|
|
tmp_path, monkeypatch,
|
||
|
|
{"vps": {"status": "online", "last_seen": time.time() - 30, "roles": []}},
|
||
|
|
{"vps/outline": {"node": "vps", "service": "outline", "status": "healthy"}},
|
||
|
|
)
|
||
|
|
svcs = operator_ui.current_services()
|
||
|
|
assert svcs[0]["health"] == "nominal"
|