156 lines
5.8 KiB
Python
156 lines
5.8 KiB
Python
"""Tests for supervisor loop resilience: a single cycle must never be able to
|
|
freeze the process forever, and the loop must survive exceptions.
|
|
|
|
Regression coverage for the 2026-07-15 incident: control-plane-supervisor
|
|
reported "Up 24h (healthy)" while the reconcile loop had silently stopped
|
|
ticking after processing one ha_websocket_dead event, with zero exceptions
|
|
and zero further log lines — because reconcile() had no per-cycle timeout
|
|
and loop() had no try/except around the cycle body.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
|
import supervisor as supervisor_module
|
|
from supervisor import Supervisor
|
|
|
|
|
|
def _make_supervisor(tmp_path: Path, monkeypatch) -> Supervisor:
|
|
actions = tmp_path / "actions"
|
|
events = tmp_path / "events"
|
|
world = tmp_path / "world"
|
|
state = tmp_path / "state"
|
|
repo = tmp_path / "repo"
|
|
|
|
for d in (actions, events, world, state, repo / "hosts"):
|
|
d.mkdir(parents=True, exist_ok=True)
|
|
|
|
monkeypatch.setattr(supervisor_module, "ACTIONS_DIR", actions)
|
|
monkeypatch.setattr(supervisor_module, "EVENTS_DIR", events)
|
|
monkeypatch.setattr(supervisor_module, "WORLD_DIR", world)
|
|
monkeypatch.setattr(supervisor_module, "REPO_ROOT", repo)
|
|
|
|
sup = Supervisor()
|
|
sup.desired_state = {"services": {}}
|
|
sup.actual_state = {"services": {}, "nodes": {}, "incidents": {}}
|
|
return sup
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 1. Loop processes many cycles without hanging
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_loop_processes_multiple_cycles(tmp_path, monkeypatch):
|
|
sup = _make_supervisor(tmp_path, monkeypatch)
|
|
monkeypatch.setattr(time, "sleep", lambda s: None)
|
|
|
|
calls = []
|
|
monkeypatch.setattr(sup, "reconcile", lambda: calls.append(1))
|
|
|
|
sup.loop(interval=0, max_cycles=5)
|
|
|
|
assert len(calls) == 5
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 2. An exception in one cycle does not kill the loop
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_exception_in_one_cycle_does_not_kill_loop(tmp_path, monkeypatch, caplog):
|
|
sup = _make_supervisor(tmp_path, monkeypatch)
|
|
monkeypatch.setattr(time, "sleep", lambda s: None)
|
|
|
|
calls = []
|
|
|
|
def flaky_reconcile():
|
|
calls.append(1)
|
|
if len(calls) == 2:
|
|
raise RuntimeError("simulated blowup mid-cycle")
|
|
|
|
monkeypatch.setattr(sup, "reconcile", flaky_reconcile)
|
|
|
|
with caplog.at_level("ERROR"):
|
|
sup.loop(interval=0, max_cycles=4)
|
|
|
|
# All 4 cycles ran despite cycle #2 raising.
|
|
assert len(calls) == 4
|
|
assert any("unhandled exception" in r.message for r in caplog.records)
|
|
|
|
|
|
def test_run_cycle_safely_never_raises(tmp_path, monkeypatch):
|
|
sup = _make_supervisor(tmp_path, monkeypatch)
|
|
monkeypatch.setattr(sup, "reconcile", lambda: (_ for _ in ()).throw(ValueError("boom")))
|
|
|
|
# Must not raise.
|
|
sup._run_cycle_safely()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 3. A blocking cycle is bounded by a timeout; the loop continues afterward
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_blocking_cycle_times_out_and_loop_continues(tmp_path, monkeypatch, caplog):
|
|
"""A reconcile() that hangs on a blocking call must not freeze loop().
|
|
|
|
Uses a single, genuinely-blocking cycle (real time.sleep in a background
|
|
thread — this is what an unkillable blocked syscall like fsync() looks
|
|
like from Python) bounded by a short reconcile_timeout. If the fix
|
|
regresses to the old bare `while True: self.reconcile()`, this test
|
|
times out the whole test run instead of failing fast.
|
|
"""
|
|
sup = _make_supervisor(tmp_path, monkeypatch)
|
|
|
|
calls = []
|
|
|
|
def hanging_reconcile():
|
|
calls.append(1)
|
|
time.sleep(0.5) # much longer than reconcile_timeout below
|
|
|
|
monkeypatch.setattr(sup, "reconcile", hanging_reconcile)
|
|
|
|
start = time.monotonic()
|
|
with caplog.at_level("ERROR"):
|
|
sup.loop(interval=0, max_cycles=1, reconcile_timeout=0.1)
|
|
elapsed = time.monotonic() - start
|
|
|
|
# loop() returned control well before the 0.5s hang finished.
|
|
assert len(calls) == 1
|
|
assert elapsed < 0.5, f"loop should not block for the full hang duration, took {elapsed}s"
|
|
assert any("did not complete within" in r.message for r in caplog.records)
|
|
|
|
|
|
def test_loop_keeps_advancing_across_repeated_timeouts(tmp_path, monkeypatch, caplog):
|
|
"""The outer loop must reach a second iteration even while a prior cycle
|
|
is still stuck in the background — proving the control loop itself (tick
|
|
counting, logging, sleeping) is never blocked by a wedged cycle."""
|
|
sup = _make_supervisor(tmp_path, monkeypatch)
|
|
monkeypatch.setattr(sup, "reconcile", lambda: time.sleep(0.5))
|
|
|
|
start = time.monotonic()
|
|
with caplog.at_level("ERROR"):
|
|
sup.loop(interval=0, max_cycles=3, reconcile_timeout=0.05)
|
|
elapsed = time.monotonic() - start
|
|
|
|
assert elapsed < 0.5, f"outer loop should complete 3 iterations fast, took {elapsed}s"
|
|
timeout_logs = [r for r in caplog.records if "did not complete within" in r.message]
|
|
assert len(timeout_logs) == 3
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 4. Existing behavior is preserved: reconcile() itself still works normally
|
|
# when called directly (not just through the loop).
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_reconcile_still_works_standalone(tmp_path, monkeypatch):
|
|
sup = _make_supervisor(tmp_path, monkeypatch)
|
|
heartbeat = tmp_path / "state" / "supervisor.heartbeat"
|
|
|
|
sup.reconcile()
|
|
|
|
assert heartbeat.exists()
|