homelab-codex-ws/services/node-agent/tests/test_check_containers.py

"""Tests for NodeAgent.check_containers() Docker-state classification.

Regression coverage for the crash-loop blind spot: a container Docker reports
as "restarting" previously matched neither the exited/dead branch nor the
running branch, so a crash-looping container emitted ZERO events and stayed
"healthy" forever in world-state. These tests pin down the full state table so
no Docker state can silently fall through classification again.
"""
from __future__ import annotations

from unittest.mock import MagicMock

import node_agent


# ---------------------------------------------------------------------------
# Fake Docker container helper
# ---------------------------------------------------------------------------

def make_container(
    status,
    *,
    restart_policy="unless-stopped",
    restart_count=0,
    health="",
    name="svc",
    compose_service="svc",
):
    """Build a stand-in for a docker-py Container with the attrs check_containers reads."""
    c = MagicMock()
    c.status = status
    c.name = name
    attrs = {
        "RestartCount": restart_count,
        "HostConfig": {"RestartPolicy": {"Name": restart_policy}},
        "State": {"Health": {"Status": health}} if health else {"Health": {}},
        "Config": {"Labels": {"com.docker.compose.service": compose_service}},
    }
    c.attrs = attrs
    return c


def run_check(agent, containers, monkeypatch):
    """Run check_containers() against a fixed container list, capturing emitted events.

    Returns the list of (event_type, severity, service, message, payload) tuples.
    """
    emitted = []

    def fake_emit(event_type, severity, service, message, payload=None):
        emitted.append((event_type, severity, service, message, payload or {}))

    monkeypatch.setattr(agent, "emit_event", fake_emit)

    client = MagicMock()
    client.containers.list.return_value = containers
    agent.docker_client = client

    agent.check_containers()
    return emitted


# ---------------------------------------------------------------------------
# The regression: "restarting" must not be silent
# ---------------------------------------------------------------------------

def test_restarting_crash_loop_emits_high_containers_not_running(agent, monkeypatch):
    """restarting + high RestartCount → containers_not_running, high, crash_loop=True."""
    c = make_container("restarting", restart_count=7, name="pi-watchtower-1",
                       compose_service="watchtower")
    events = run_check(agent, [c], monkeypatch)

    assert len(events) == 1
    etype, severity, service, _msg, payload = events[0]
    assert etype == "containers_not_running"
    assert severity == "high"
    assert service == "watchtower"
    assert payload["crash_loop"] is True
    assert payload["restart_count"] == 7
    assert payload["status"] == "restarting"


def test_restarting_fresh_emits_low_container_restarting_no_alarm(agent, monkeypatch):
    """restarting + low RestartCount → observational container_restarting, low, not actionable."""
    c = make_container("restarting", restart_count=1, name="svc")
    events = run_check(agent, [c], monkeypatch)

    assert len(events) == 1
    etype, severity, _service, _msg, payload = events[0]
    assert etype == "container_restarting"
    assert severity == "low"
    assert payload["crash_loop"] is False
    assert payload["restart_count"] == 1
    # It must NOT masquerade as an actionable "not running" or "healthy" signal.
    assert etype not in ("containers_not_running", "service_healthy", "healthcheck_failed")


def test_restarting_at_threshold_is_crash_loop(agent, monkeypatch):
    """RestartCount exactly at the threshold counts as a crash-loop (>=)."""
    c = make_container("restarting", restart_count=node_agent.CRASH_LOOP_RESTART_THRESHOLD)
    events = run_check(agent, [c], monkeypatch)
    assert events[0][0] == "containers_not_running"
    assert events[0][1] == "high"


# ---------------------------------------------------------------------------
# Existing behaviour must be preserved
# ---------------------------------------------------------------------------

def test_exited_still_emits_containers_not_running(agent, monkeypatch):
    c = make_container("exited")
    events = run_check(agent, [c], monkeypatch)
    assert events[0][0] == "containers_not_running"
    assert events[0][1] == "high"


def test_dead_still_emits_containers_not_running(agent, monkeypatch):
    c = make_container("dead")
    events = run_check(agent, [c], monkeypatch)
    assert events[0][0] == "containers_not_running"


def test_running_healthy_emits_service_healthy(agent, monkeypatch):
    c = make_container("running")
    events = run_check(agent, [c], monkeypatch)
    assert events[0][0] == "service_healthy"
    assert events[0][1] == "info"


def test_running_unhealthy_emits_healthcheck_failed(agent, monkeypatch):
    c = make_container("running", health="unhealthy")
    events = run_check(agent, [c], monkeypatch)
    assert events[0][0] == "healthcheck_failed"
    assert events[0][1] == "high"


# ---------------------------------------------------------------------------
# The other previously-silent states
# ---------------------------------------------------------------------------

def test_paused_emits_state_unexpected(agent, monkeypatch):
    c = make_container("paused")
    events = run_check(agent, [c], monkeypatch)
    assert len(events) == 1
    assert events[0][0] == "container_state_unexpected"
    assert events[0][1] == "medium"


def test_unknown_state_emits_fallback_diagnostic(agent, monkeypatch):
    """A Docker state we do not model must never be silent — fallback diagnostic."""
    c = make_container("some-future-state")
    events = run_check(agent, [c], monkeypatch)
    assert len(events) == 1
    etype, severity, _service, _msg, payload = events[0]
    assert etype == "container_state_unexpected"
    assert severity == "medium"
    assert payload["status"] == "some-future-state"


def test_removing_is_conscious_silent_skip(agent, monkeypatch):
    """removing is an ephemeral teardown state — deliberately no event (documented)."""
    c = make_container("removing")
    events = run_check(agent, [c], monkeypatch)
    assert events == []


def test_created_is_skipped(agent, monkeypatch):
    c = make_container("created")
    events = run_check(agent, [c], monkeypatch)
    assert events == []


# ---------------------------------------------------------------------------
# Managed-container filter still applies
# ---------------------------------------------------------------------------

def test_unmanaged_container_is_ignored(agent, monkeypatch):
    """No restart policy → not a long-running managed service → no events, even if restarting."""
    c = make_container("restarting", restart_policy="no", restart_count=9)
    events = run_check(agent, [c], monkeypatch)
    assert events == []


def test_no_docker_client_is_noop(agent, monkeypatch):
    agent.docker_client = None
    # Should simply return without raising.
    agent.check_containers()