"""Tests for NodeAgent.check_containers() Docker-state classification. Regression coverage for the crash-loop blind spot: a container Docker reports as "restarting" previously matched neither the exited/dead branch nor the running branch, so a crash-looping container emitted ZERO events and stayed "healthy" forever in world-state. These tests pin down the full state table so no Docker state can silently fall through classification again. """ from __future__ import annotations from unittest.mock import MagicMock import node_agent # --------------------------------------------------------------------------- # Fake Docker container helper # --------------------------------------------------------------------------- def make_container( status, *, restart_policy="unless-stopped", restart_count=0, health="", name="svc", compose_service="svc", ): """Build a stand-in for a docker-py Container with the attrs check_containers reads.""" c = MagicMock() c.status = status c.name = name attrs = { "RestartCount": restart_count, "HostConfig": {"RestartPolicy": {"Name": restart_policy}}, "State": {"Health": {"Status": health}} if health else {"Health": {}}, "Config": {"Labels": {"com.docker.compose.service": compose_service}}, } c.attrs = attrs return c def run_check(agent, containers, monkeypatch): """Run check_containers() against a fixed container list, capturing emitted events. Returns the list of (event_type, severity, service, message, payload) tuples. """ emitted = [] def fake_emit(event_type, severity, service, message, payload=None): emitted.append((event_type, severity, service, message, payload or {})) monkeypatch.setattr(agent, "emit_event", fake_emit) client = MagicMock() client.containers.list.return_value = containers agent.docker_client = client agent.check_containers() return emitted # --------------------------------------------------------------------------- # The regression: "restarting" must not be silent # --------------------------------------------------------------------------- def test_restarting_crash_loop_emits_high_containers_not_running(agent, monkeypatch): """restarting + high RestartCount → containers_not_running, high, crash_loop=True.""" c = make_container("restarting", restart_count=7, name="pi-watchtower-1", compose_service="watchtower") events = run_check(agent, [c], monkeypatch) assert len(events) == 1 etype, severity, service, _msg, payload = events[0] assert etype == "containers_not_running" assert severity == "high" assert service == "watchtower" assert payload["crash_loop"] is True assert payload["restart_count"] == 7 assert payload["status"] == "restarting" def test_restarting_fresh_emits_low_container_restarting_no_alarm(agent, monkeypatch): """restarting + low RestartCount → observational container_restarting, low, not actionable.""" c = make_container("restarting", restart_count=1, name="svc") events = run_check(agent, [c], monkeypatch) assert len(events) == 1 etype, severity, _service, _msg, payload = events[0] assert etype == "container_restarting" assert severity == "low" assert payload["crash_loop"] is False assert payload["restart_count"] == 1 # It must NOT masquerade as an actionable "not running" or "healthy" signal. assert etype not in ("containers_not_running", "service_healthy", "healthcheck_failed") def test_restarting_at_threshold_is_crash_loop(agent, monkeypatch): """RestartCount exactly at the threshold counts as a crash-loop (>=).""" c = make_container("restarting", restart_count=node_agent.CRASH_LOOP_RESTART_THRESHOLD) events = run_check(agent, [c], monkeypatch) assert events[0][0] == "containers_not_running" assert events[0][1] == "high" # --------------------------------------------------------------------------- # Existing behaviour must be preserved # --------------------------------------------------------------------------- def test_exited_still_emits_containers_not_running(agent, monkeypatch): c = make_container("exited") events = run_check(agent, [c], monkeypatch) assert events[0][0] == "containers_not_running" assert events[0][1] == "high" def test_dead_still_emits_containers_not_running(agent, monkeypatch): c = make_container("dead") events = run_check(agent, [c], monkeypatch) assert events[0][0] == "containers_not_running" def test_running_healthy_emits_service_healthy(agent, monkeypatch): c = make_container("running") events = run_check(agent, [c], monkeypatch) assert events[0][0] == "service_healthy" assert events[0][1] == "info" def test_running_unhealthy_emits_healthcheck_failed(agent, monkeypatch): c = make_container("running", health="unhealthy") events = run_check(agent, [c], monkeypatch) assert events[0][0] == "healthcheck_failed" assert events[0][1] == "high" # --------------------------------------------------------------------------- # The other previously-silent states # --------------------------------------------------------------------------- def test_paused_emits_state_unexpected(agent, monkeypatch): c = make_container("paused") events = run_check(agent, [c], monkeypatch) assert len(events) == 1 assert events[0][0] == "container_state_unexpected" assert events[0][1] == "medium" def test_unknown_state_emits_fallback_diagnostic(agent, monkeypatch): """A Docker state we do not model must never be silent — fallback diagnostic.""" c = make_container("some-future-state") events = run_check(agent, [c], monkeypatch) assert len(events) == 1 etype, severity, _service, _msg, payload = events[0] assert etype == "container_state_unexpected" assert severity == "medium" assert payload["status"] == "some-future-state" def test_removing_is_conscious_silent_skip(agent, monkeypatch): """removing is an ephemeral teardown state — deliberately no event (documented).""" c = make_container("removing") events = run_check(agent, [c], monkeypatch) assert events == [] def test_created_is_skipped(agent, monkeypatch): c = make_container("created") events = run_check(agent, [c], monkeypatch) assert events == [] # --------------------------------------------------------------------------- # Managed-container filter still applies # --------------------------------------------------------------------------- def test_unmanaged_container_is_ignored(agent, monkeypatch): """No restart policy → not a long-running managed service → no events, even if restarting.""" c = make_container("restarting", restart_policy="no", restart_count=9) events = run_check(agent, [c], monkeypatch) assert events == [] def test_no_docker_client_is_noop(agent, monkeypatch): agent.docker_client = None # Should simply return without raising. agent.check_containers()