homelab-codex-ws/services/node-agent/tests/test_safe_cleanup.py

"""Tests for NodeAgent Docker cleanup — regression cover for the 2026-07-30
unfiltered-prune incident (kb/incidents/2026-07-30-ollama-solaria-vanish.md).

`containers.prune()` removes EVERY non-running container, ignoring restart
policy and compose labels; that is what deleted `ollama` on SOLARIA 19 s after
an operator stopped it. These tests pin down the replacement:

  R1 — explicit enumeration; a stopped container with `restart: unless-stopped`
       or a compose project label is never removed, and prune() is never called.
  R2 — the cleanup rate limit applies to ai_node / standard, not sd_card only.
  R3 — removals are named in a WARNING log line.
"""
from __future__ import annotations

import time
from unittest.mock import MagicMock

import pytest

import node_agent


# ---------------------------------------------------------------------------
# Fake Docker container helper
# ---------------------------------------------------------------------------

def make_stopped(name, *, restart_policy="no", compose_project=None):
    c = MagicMock()
    c.name = name
    c.status = "exited"
    c.attrs = {"HostConfig": {"RestartPolicy": {"Name": restart_policy}}}
    c.labels = {"com.docker.compose.project": compose_project} if compose_project else {}
    return c


def with_containers(agent, containers):
    client = MagicMock()
    client.containers.list.return_value = containers
    agent.docker_client = client
    return client


# ---------------------------------------------------------------------------
# R1 — the incident itself
# ---------------------------------------------------------------------------

def test_unless_stopped_container_survives(agent):
    """THE regression: ollama@solaria — exited, restart=unless-stopped, must live."""
    ollama = make_stopped("ollama", restart_policy="unless-stopped")
    client = with_containers(agent, [ollama])

    agent._prune_stopped_containers()

    ollama.remove.assert_not_called()
    client.containers.prune.assert_not_called()


@pytest.mark.parametrize("policy", ["unless-stopped", "always", "on-failure"])
def test_every_restart_policy_is_operator_intent(agent, policy):
    c = make_stopped("svc", restart_policy=policy)
    with_containers(agent, [c])
    agent._prune_stopped_containers()
    c.remove.assert_not_called()


def test_compose_managed_container_survives(agent):
    """No restart policy, but compose owns it → still off limits."""
    c = make_stopped("outline-redis-1", restart_policy="no", compose_project="outline")
    with_containers(agent, [c])
    agent._prune_stopped_containers()
    c.remove.assert_not_called()


def test_disposable_leftover_is_removed(agent):
    """restart=no and no compose project → a one-off leftover, safe to remove."""
    c = make_stopped("nervous-shell-42", restart_policy="no")
    with_containers(agent, [c])
    agent._prune_stopped_containers()
    c.remove.assert_called_once()


def test_only_exited_containers_are_enumerated(agent):
    """Running containers must never even enter the candidate list."""
    client = with_containers(agent, [])
    agent._prune_stopped_containers()
    _args, kwargs = client.containers.list.call_args
    assert kwargs["all"] is True
    assert kwargs["filters"] == {"status": "exited"}


def test_mixed_set_removes_only_the_disposable_one(agent):
    keep_policy = make_stopped("ollama", restart_policy="unless-stopped")
    keep_compose = make_stopped("umami-db", compose_project="umami")
    drop = make_stopped("tmp-build", restart_policy="no")
    with_containers(agent, [keep_policy, keep_compose, drop])

    agent._prune_stopped_containers()

    keep_policy.remove.assert_not_called()
    keep_compose.remove.assert_not_called()
    drop.remove.assert_called_once()


def test_remove_failure_does_not_abort_the_sweep(agent):
    boom = make_stopped("boom", restart_policy="no")
    boom.remove.side_effect = RuntimeError("device or resource busy")
    later = make_stopped("later", restart_policy="no")
    with_containers(agent, [boom, later])

    agent._prune_stopped_containers()

    later.remove.assert_called_once()


def test_no_docker_client_is_noop(agent):
    agent.docker_client = None
    agent._prune_stopped_containers()  # must not raise


# ---------------------------------------------------------------------------
# R3 — say what was deleted
# ---------------------------------------------------------------------------

def test_removal_is_logged_at_warning_with_names(agent, caplog):
    with_containers(agent, [make_stopped("tmp-build", restart_policy="no")])

    with caplog.at_level("INFO", logger="node-agent"):
        agent._prune_stopped_containers()

    warnings = [r for r in caplog.records if r.levelname == "WARNING"]
    assert len(warnings) == 1
    assert "tmp-build" in warnings[0].message


def test_nothing_removed_stays_at_info(agent, caplog):
    with_containers(agent, [make_stopped("ollama", restart_policy="unless-stopped")])

    with caplog.at_level("INFO", logger="node-agent"):
        agent._prune_stopped_containers()

    assert [r for r in caplog.records if r.levelname == "WARNING"] == []


# ---------------------------------------------------------------------------
# R2 — rate limit covers every node type
# ---------------------------------------------------------------------------

@pytest.fixture
def cleanup_calls(agent, monkeypatch):
    """Record which prune helpers run, without touching Docker."""
    calls = []
    for helper in ("_prune_dangling_images", "_prune_stopped_containers",
                   "_prune_build_cache"):
        monkeypatch.setattr(agent, helper, lambda h=helper: calls.append(h))
    return calls


@pytest.fixture
def fresh_cleanup_marker(tmp_path, monkeypatch):
    monkeypatch.setattr(node_agent, "LAST_CLEANUP_FILE", tmp_path / "last-docker-cleanup")
    return node_agent.LAST_CLEANUP_FILE


@pytest.mark.parametrize("node_type", ["ai_node", "standard", "sd_card"])
def test_recent_cleanup_blocks_next_run(agent, cleanup_calls, fresh_cleanup_marker,
                                        node_type):
    """Was true for sd_card only; after R2 it holds for ai_node and standard too."""
    agent.node_type = node_type
    fresh_cleanup_marker.write_text(str(int(time.time())))

    agent.run_safe_cleanup()

    assert cleanup_calls == []


@pytest.mark.parametrize("node_type,expected", [
    ("ai_node", ["_prune_dangling_images", "_prune_stopped_containers",
                 "_prune_build_cache"]),
    ("standard", ["_prune_dangling_images", "_prune_stopped_containers",
                  "_prune_build_cache"]),
    ("sd_card", ["_prune_dangling_images", "_prune_stopped_containers"]),
])
def test_stale_marker_allows_cleanup_and_is_refreshed(agent, cleanup_calls,
                                                      fresh_cleanup_marker,
                                                      node_type, expected):
    agent.node_type = node_type
    stale = int(time.time()) - node_agent.CLEANUP_INTERVAL_SECS - 1
    fresh_cleanup_marker.write_text(str(stale))

    agent.run_safe_cleanup()

    assert cleanup_calls == expected
    # Marker refreshed for every type, otherwise the guard never engages.
    assert int(fresh_cleanup_marker.read_text()) > stale


def test_lte_node_still_does_nothing(agent, cleanup_calls, fresh_cleanup_marker):
    agent.node_type = "lte_node"
    agent.run_safe_cleanup()
    assert cleanup_calls == []
    assert not fresh_cleanup_marker.exists()