"""Tests for supervisor loop resilience: a single cycle must never be able to freeze the process forever, and the loop must survive exceptions. Regression coverage for the 2026-07-15 incident: control-plane-supervisor reported "Up 24h (healthy)" while the reconcile loop had silently stopped ticking after processing one ha_websocket_dead event, with zero exceptions and zero further log lines — because reconcile() had no per-cycle timeout and loop() had no try/except around the cycle body. """ from __future__ import annotations import sys import time from pathlib import Path import pytest sys.path.insert(0, str(Path(__file__).parent.parent / "src")) import supervisor as supervisor_module from supervisor import Supervisor def _make_supervisor(tmp_path: Path, monkeypatch) -> Supervisor: actions = tmp_path / "actions" events = tmp_path / "events" world = tmp_path / "world" state = tmp_path / "state" repo = tmp_path / "repo" for d in (actions, events, world, state, repo / "hosts"): d.mkdir(parents=True, exist_ok=True) monkeypatch.setattr(supervisor_module, "ACTIONS_DIR", actions) monkeypatch.setattr(supervisor_module, "EVENTS_DIR", events) monkeypatch.setattr(supervisor_module, "WORLD_DIR", world) monkeypatch.setattr(supervisor_module, "REPO_ROOT", repo) sup = Supervisor() sup.desired_state = {"services": {}} sup.actual_state = {"services": {}, "nodes": {}, "incidents": {}} return sup # --------------------------------------------------------------------------- # 1. Loop processes many cycles without hanging # --------------------------------------------------------------------------- def test_loop_processes_multiple_cycles(tmp_path, monkeypatch): sup = _make_supervisor(tmp_path, monkeypatch) monkeypatch.setattr(time, "sleep", lambda s: None) calls = [] monkeypatch.setattr(sup, "reconcile", lambda: calls.append(1)) sup.loop(interval=0, max_cycles=5) assert len(calls) == 5 # --------------------------------------------------------------------------- # 2. An exception in one cycle does not kill the loop # --------------------------------------------------------------------------- def test_exception_in_one_cycle_does_not_kill_loop(tmp_path, monkeypatch, caplog): sup = _make_supervisor(tmp_path, monkeypatch) monkeypatch.setattr(time, "sleep", lambda s: None) calls = [] def flaky_reconcile(): calls.append(1) if len(calls) == 2: raise RuntimeError("simulated blowup mid-cycle") monkeypatch.setattr(sup, "reconcile", flaky_reconcile) with caplog.at_level("ERROR"): sup.loop(interval=0, max_cycles=4) # All 4 cycles ran despite cycle #2 raising. assert len(calls) == 4 assert any("unhandled exception" in r.message for r in caplog.records) def test_run_cycle_safely_never_raises(tmp_path, monkeypatch): sup = _make_supervisor(tmp_path, monkeypatch) monkeypatch.setattr(sup, "reconcile", lambda: (_ for _ in ()).throw(ValueError("boom"))) # Must not raise. sup._run_cycle_safely() # --------------------------------------------------------------------------- # 3. A blocking cycle is bounded by a timeout; the loop continues afterward # --------------------------------------------------------------------------- def test_blocking_cycle_times_out_and_loop_continues(tmp_path, monkeypatch, caplog): """A reconcile() that hangs on a blocking call must not freeze loop(). Uses a single, genuinely-blocking cycle (real time.sleep in a background thread — this is what an unkillable blocked syscall like fsync() looks like from Python) bounded by a short reconcile_timeout. If the fix regresses to the old bare `while True: self.reconcile()`, this test times out the whole test run instead of failing fast. """ sup = _make_supervisor(tmp_path, monkeypatch) calls = [] def hanging_reconcile(): calls.append(1) time.sleep(0.5) # much longer than reconcile_timeout below monkeypatch.setattr(sup, "reconcile", hanging_reconcile) start = time.monotonic() with caplog.at_level("ERROR"): sup.loop(interval=0, max_cycles=1, reconcile_timeout=0.1) elapsed = time.monotonic() - start # loop() returned control well before the 0.5s hang finished. assert len(calls) == 1 assert elapsed < 0.5, f"loop should not block for the full hang duration, took {elapsed}s" assert any("did not complete within" in r.message for r in caplog.records) def test_loop_keeps_advancing_across_repeated_timeouts(tmp_path, monkeypatch, caplog): """The outer loop must reach a second iteration even while a prior cycle is still stuck in the background — proving the control loop itself (tick counting, logging, sleeping) is never blocked by a wedged cycle.""" sup = _make_supervisor(tmp_path, monkeypatch) monkeypatch.setattr(sup, "reconcile", lambda: time.sleep(0.5)) start = time.monotonic() with caplog.at_level("ERROR"): sup.loop(interval=0, max_cycles=3, reconcile_timeout=0.05) elapsed = time.monotonic() - start assert elapsed < 0.5, f"outer loop should complete 3 iterations fast, took {elapsed}s" timeout_logs = [r for r in caplog.records if "did not complete within" in r.message] assert len(timeout_logs) == 3 # --------------------------------------------------------------------------- # 4. Existing behavior is preserved: reconcile() itself still works normally # when called directly (not just through the loop). # --------------------------------------------------------------------------- def test_reconcile_still_works_standalone(tmp_path, monkeypatch): sup = _make_supervisor(tmp_path, monkeypatch) heartbeat = tmp_path / "state" / "supervisor.heartbeat" sup.reconcile() assert heartbeat.exists()