"""Tests for NodeAgent.check_containers() Docker-state classification. Regression coverage for the crash-loop blind spot: a container Docker reports as "restarting" previously matched neither the exited/dead branch nor the running branch, so a crash-looping container emitted ZERO events and stayed "healthy" forever in world-state. These tests pin down the full state table so no Docker state can silently fall through classification again. """ from __future__ import annotations from unittest.mock import MagicMock import node_agent # --------------------------------------------------------------------------- # Fake Docker container helper # --------------------------------------------------------------------------- def make_container( status, *, restart_policy="unless-stopped", restart_count=0, health="", name="svc", compose_service="svc", ): """Build a stand-in for a docker-py Container with the attrs check_containers reads.""" c = MagicMock() c.status = status c.name = name attrs = { "RestartCount": restart_count, "HostConfig": {"RestartPolicy": {"Name": restart_policy}}, "State": {"Health": {"Status": health}} if health else {"Health": {}}, "Config": {"Labels": {"com.docker.compose.service": compose_service}}, } c.attrs = attrs return c def run_check(agent, containers, monkeypatch): """Run check_containers() against a fixed container list, capturing emitted events. Returns the list of (event_type, severity, service, message, payload) tuples. """ emitted = [] def fake_emit(event_type, severity, service, message, payload=None): emitted.append((event_type, severity, service, message, payload or {})) monkeypatch.setattr(agent, "emit_event", fake_emit) client = MagicMock() client.containers.list.return_value = containers agent.docker_client = client agent.check_containers() return emitted # --------------------------------------------------------------------------- # The regression: "restarting" must not be silent # --------------------------------------------------------------------------- def test_restarting_crash_loop_emits_high_containers_not_running(agent, monkeypatch): """restarting + high RestartCount → containers_not_running, high, crash_loop=True.""" c = make_container("restarting", restart_count=7, name="pi-watchtower-1", compose_service="watchtower") events = run_check(agent, [c], monkeypatch) assert len(events) == 1 etype, severity, service, _msg, payload = events[0] assert etype == "containers_not_running" assert severity == "high" assert service == "watchtower" assert payload["crash_loop"] is True assert payload["restart_count"] == 7 assert payload["status"] == "restarting" def test_restarting_fresh_emits_low_container_restarting_no_alarm(agent, monkeypatch): """restarting + low RestartCount → observational container_restarting, low, not actionable.""" c = make_container("restarting", restart_count=1, name="svc") events = run_check(agent, [c], monkeypatch) assert len(events) == 1 etype, severity, _service, _msg, payload = events[0] assert etype == "container_restarting" assert severity == "low" assert payload["crash_loop"] is False assert payload["restart_count"] == 1 # It must NOT masquerade as an actionable "not running" or "healthy" signal. assert etype not in ("containers_not_running", "service_healthy", "healthcheck_failed") def test_restarting_at_threshold_is_crash_loop(agent, monkeypatch): """RestartCount exactly at the threshold counts as a crash-loop (>=).""" c = make_container("restarting", restart_count=node_agent.CRASH_LOOP_RESTART_THRESHOLD) events = run_check(agent, [c], monkeypatch) assert events[0][0] == "containers_not_running" assert events[0][1] == "high" # --------------------------------------------------------------------------- # Existing behaviour must be preserved # --------------------------------------------------------------------------- def test_exited_still_emits_containers_not_running(agent, monkeypatch): c = make_container("exited") events = run_check(agent, [c], monkeypatch) assert events[0][0] == "containers_not_running" assert events[0][1] == "high" def test_dead_still_emits_containers_not_running(agent, monkeypatch): c = make_container("dead") events = run_check(agent, [c], monkeypatch) assert events[0][0] == "containers_not_running" def test_running_healthy_emits_service_healthy(agent, monkeypatch): c = make_container("running") events = run_check(agent, [c], monkeypatch) assert events[0][0] == "service_healthy" assert events[0][1] == "info" def test_running_healthy_second_cycle_is_silent(agent, monkeypatch): """Regression for the event-flood fix: a service that stays healthy across cycles must NOT re-emit service_healthy every cycle — only on the transition into healthy.""" c = make_container("running") run_check(agent, [c], monkeypatch) # first cycle: transition, emits once events = run_check(agent, [c], monkeypatch) # second cycle: still healthy assert events == [] def test_running_healthy_after_unhealthy_reemits_service_healthy(agent, monkeypatch): """Recovery (unhealthy -> healthy) must still be visible exactly once.""" unhealthy = make_container("running", health="unhealthy", name="svc") events = run_check(agent, [unhealthy], monkeypatch) assert events[0][0] == "healthcheck_failed" healthy = make_container("running", name="svc") events = run_check(agent, [healthy], monkeypatch) assert len(events) == 1 assert events[0][0] == "service_healthy" # Third cycle, still healthy: silent again. events = run_check(agent, [healthy], monkeypatch) assert events == [] def test_running_healthy_after_crash_loop_reemits_service_healthy(agent, monkeypatch): """Recovery from a crash-loop (containers_not_running) is also a real transition and must re-confirm health exactly once.""" crashing = make_container("restarting", restart_count=7, name="svc") events = run_check(agent, [crashing], monkeypatch) assert events[0][0] == "containers_not_running" recovered = make_container("running", name="svc") events = run_check(agent, [recovered], monkeypatch) assert len(events) == 1 assert events[0][0] == "service_healthy" def test_different_services_tracked_independently(agent, monkeypatch): """One service's health transition must not gate another service's emission.""" svc_a = make_container("running", name="svc-a", compose_service="svc-a") svc_b = make_container("running", name="svc-b", compose_service="svc-b") events = run_check(agent, [svc_a, svc_b], monkeypatch) assert {e[0] for e in events} == {"service_healthy"} assert len(events) == 2 # Both still healthy next cycle: both silent. events = run_check(agent, [svc_a, svc_b], monkeypatch) assert events == [] def test_running_unhealthy_emits_healthcheck_failed(agent, monkeypatch): c = make_container("running", health="unhealthy") events = run_check(agent, [c], monkeypatch) assert events[0][0] == "healthcheck_failed" assert events[0][1] == "high" # --------------------------------------------------------------------------- # The other previously-silent states # --------------------------------------------------------------------------- def test_paused_emits_state_unexpected(agent, monkeypatch): c = make_container("paused") events = run_check(agent, [c], monkeypatch) assert len(events) == 1 assert events[0][0] == "container_state_unexpected" assert events[0][1] == "medium" def test_unknown_state_emits_fallback_diagnostic(agent, monkeypatch): """A Docker state we do not model must never be silent — fallback diagnostic.""" c = make_container("some-future-state") events = run_check(agent, [c], monkeypatch) assert len(events) == 1 etype, severity, _service, _msg, payload = events[0] assert etype == "container_state_unexpected" assert severity == "medium" assert payload["status"] == "some-future-state" def test_removing_is_conscious_silent_skip(agent, monkeypatch): """removing is an ephemeral teardown state — deliberately no event (documented).""" c = make_container("removing") events = run_check(agent, [c], monkeypatch) assert events == [] def test_created_is_skipped(agent, monkeypatch): c = make_container("created") events = run_check(agent, [c], monkeypatch) assert events == [] # --------------------------------------------------------------------------- # Managed-container filter still applies # --------------------------------------------------------------------------- def test_unmanaged_container_is_ignored(agent, monkeypatch): """No restart policy → not a long-running managed service → no events, even if restarting.""" c = make_container("restarting", restart_policy="no", restart_count=9) events = run_check(agent, [c], monkeypatch) assert events == [] def test_no_docker_client_is_noop(agent, monkeypatch): agent.docker_client = None # Should simply return without raising. agent.check_containers()