dff76ecd30
- node-agent: service_healthy emitowany tylko przy przejsciu w stan zdrowy (per-service in-memory state), nie co cykl dla kazdego zdrowego serwisu. To samo dla control-plane HTTP probe. healthcheck_failed/containers_not_running/ incydenty pozostaja emitowane bez zmian (realne sygnaly). - node-agent: naprawiono _cleanup_control_plane_fs — istniejaca retencja eventow byla martwa od migracji observer_checkpoint.json na epoch-int (str(path) <= int rzucal TypeError, lapane cicho przez broad except). Teraz porownuje epoch-do-epoch i czysci wylacznie service_healthy/node_health starsze niz checkpoint + 3-dniowy bufor; healthcheck_failed/incydenty/ha_* zachowane bezterminowo. - scripts/maintenance/cleanup_event_backlog.py: jednorazowy skrypt do bezpiecznego czyszczenia backlogu na VPS (dry-run domyslnie, --apply do usuniecia). Ten sam warunek: typ szumu + starsze niz checkpoint + 1h bufor.
241 lines
9.2 KiB
Python
241 lines
9.2 KiB
Python
"""Tests for NodeAgent.check_containers() Docker-state classification.
|
|
|
|
Regression coverage for the crash-loop blind spot: a container Docker reports
|
|
as "restarting" previously matched neither the exited/dead branch nor the
|
|
running branch, so a crash-looping container emitted ZERO events and stayed
|
|
"healthy" forever in world-state. These tests pin down the full state table so
|
|
no Docker state can silently fall through classification again.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import MagicMock
|
|
|
|
import node_agent
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fake Docker container helper
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def make_container(
|
|
status,
|
|
*,
|
|
restart_policy="unless-stopped",
|
|
restart_count=0,
|
|
health="",
|
|
name="svc",
|
|
compose_service="svc",
|
|
):
|
|
"""Build a stand-in for a docker-py Container with the attrs check_containers reads."""
|
|
c = MagicMock()
|
|
c.status = status
|
|
c.name = name
|
|
attrs = {
|
|
"RestartCount": restart_count,
|
|
"HostConfig": {"RestartPolicy": {"Name": restart_policy}},
|
|
"State": {"Health": {"Status": health}} if health else {"Health": {}},
|
|
"Config": {"Labels": {"com.docker.compose.service": compose_service}},
|
|
}
|
|
c.attrs = attrs
|
|
return c
|
|
|
|
|
|
def run_check(agent, containers, monkeypatch):
|
|
"""Run check_containers() against a fixed container list, capturing emitted events.
|
|
|
|
Returns the list of (event_type, severity, service, message, payload) tuples.
|
|
"""
|
|
emitted = []
|
|
|
|
def fake_emit(event_type, severity, service, message, payload=None):
|
|
emitted.append((event_type, severity, service, message, payload or {}))
|
|
|
|
monkeypatch.setattr(agent, "emit_event", fake_emit)
|
|
|
|
client = MagicMock()
|
|
client.containers.list.return_value = containers
|
|
agent.docker_client = client
|
|
|
|
agent.check_containers()
|
|
return emitted
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# The regression: "restarting" must not be silent
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_restarting_crash_loop_emits_high_containers_not_running(agent, monkeypatch):
|
|
"""restarting + high RestartCount → containers_not_running, high, crash_loop=True."""
|
|
c = make_container("restarting", restart_count=7, name="pi-watchtower-1",
|
|
compose_service="watchtower")
|
|
events = run_check(agent, [c], monkeypatch)
|
|
|
|
assert len(events) == 1
|
|
etype, severity, service, _msg, payload = events[0]
|
|
assert etype == "containers_not_running"
|
|
assert severity == "high"
|
|
assert service == "watchtower"
|
|
assert payload["crash_loop"] is True
|
|
assert payload["restart_count"] == 7
|
|
assert payload["status"] == "restarting"
|
|
|
|
|
|
def test_restarting_fresh_emits_low_container_restarting_no_alarm(agent, monkeypatch):
|
|
"""restarting + low RestartCount → observational container_restarting, low, not actionable."""
|
|
c = make_container("restarting", restart_count=1, name="svc")
|
|
events = run_check(agent, [c], monkeypatch)
|
|
|
|
assert len(events) == 1
|
|
etype, severity, _service, _msg, payload = events[0]
|
|
assert etype == "container_restarting"
|
|
assert severity == "low"
|
|
assert payload["crash_loop"] is False
|
|
assert payload["restart_count"] == 1
|
|
# It must NOT masquerade as an actionable "not running" or "healthy" signal.
|
|
assert etype not in ("containers_not_running", "service_healthy", "healthcheck_failed")
|
|
|
|
|
|
def test_restarting_at_threshold_is_crash_loop(agent, monkeypatch):
|
|
"""RestartCount exactly at the threshold counts as a crash-loop (>=)."""
|
|
c = make_container("restarting", restart_count=node_agent.CRASH_LOOP_RESTART_THRESHOLD)
|
|
events = run_check(agent, [c], monkeypatch)
|
|
assert events[0][0] == "containers_not_running"
|
|
assert events[0][1] == "high"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Existing behaviour must be preserved
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_exited_still_emits_containers_not_running(agent, monkeypatch):
|
|
c = make_container("exited")
|
|
events = run_check(agent, [c], monkeypatch)
|
|
assert events[0][0] == "containers_not_running"
|
|
assert events[0][1] == "high"
|
|
|
|
|
|
def test_dead_still_emits_containers_not_running(agent, monkeypatch):
|
|
c = make_container("dead")
|
|
events = run_check(agent, [c], monkeypatch)
|
|
assert events[0][0] == "containers_not_running"
|
|
|
|
|
|
def test_running_healthy_emits_service_healthy(agent, monkeypatch):
|
|
c = make_container("running")
|
|
events = run_check(agent, [c], monkeypatch)
|
|
assert events[0][0] == "service_healthy"
|
|
assert events[0][1] == "info"
|
|
|
|
|
|
def test_running_healthy_second_cycle_is_silent(agent, monkeypatch):
|
|
"""Regression for the event-flood fix: a service that stays healthy across
|
|
cycles must NOT re-emit service_healthy every cycle — only on the
|
|
transition into healthy."""
|
|
c = make_container("running")
|
|
run_check(agent, [c], monkeypatch) # first cycle: transition, emits once
|
|
events = run_check(agent, [c], monkeypatch) # second cycle: still healthy
|
|
assert events == []
|
|
|
|
|
|
def test_running_healthy_after_unhealthy_reemits_service_healthy(agent, monkeypatch):
|
|
"""Recovery (unhealthy -> healthy) must still be visible exactly once."""
|
|
unhealthy = make_container("running", health="unhealthy", name="svc")
|
|
events = run_check(agent, [unhealthy], monkeypatch)
|
|
assert events[0][0] == "healthcheck_failed"
|
|
|
|
healthy = make_container("running", name="svc")
|
|
events = run_check(agent, [healthy], monkeypatch)
|
|
assert len(events) == 1
|
|
assert events[0][0] == "service_healthy"
|
|
|
|
# Third cycle, still healthy: silent again.
|
|
events = run_check(agent, [healthy], monkeypatch)
|
|
assert events == []
|
|
|
|
|
|
def test_running_healthy_after_crash_loop_reemits_service_healthy(agent, monkeypatch):
|
|
"""Recovery from a crash-loop (containers_not_running) is also a real
|
|
transition and must re-confirm health exactly once."""
|
|
crashing = make_container("restarting", restart_count=7, name="svc")
|
|
events = run_check(agent, [crashing], monkeypatch)
|
|
assert events[0][0] == "containers_not_running"
|
|
|
|
recovered = make_container("running", name="svc")
|
|
events = run_check(agent, [recovered], monkeypatch)
|
|
assert len(events) == 1
|
|
assert events[0][0] == "service_healthy"
|
|
|
|
|
|
def test_different_services_tracked_independently(agent, monkeypatch):
|
|
"""One service's health transition must not gate another service's emission."""
|
|
svc_a = make_container("running", name="svc-a", compose_service="svc-a")
|
|
svc_b = make_container("running", name="svc-b", compose_service="svc-b")
|
|
|
|
events = run_check(agent, [svc_a, svc_b], monkeypatch)
|
|
assert {e[0] for e in events} == {"service_healthy"}
|
|
assert len(events) == 2
|
|
|
|
# Both still healthy next cycle: both silent.
|
|
events = run_check(agent, [svc_a, svc_b], monkeypatch)
|
|
assert events == []
|
|
|
|
|
|
def test_running_unhealthy_emits_healthcheck_failed(agent, monkeypatch):
|
|
c = make_container("running", health="unhealthy")
|
|
events = run_check(agent, [c], monkeypatch)
|
|
assert events[0][0] == "healthcheck_failed"
|
|
assert events[0][1] == "high"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# The other previously-silent states
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_paused_emits_state_unexpected(agent, monkeypatch):
|
|
c = make_container("paused")
|
|
events = run_check(agent, [c], monkeypatch)
|
|
assert len(events) == 1
|
|
assert events[0][0] == "container_state_unexpected"
|
|
assert events[0][1] == "medium"
|
|
|
|
|
|
def test_unknown_state_emits_fallback_diagnostic(agent, monkeypatch):
|
|
"""A Docker state we do not model must never be silent — fallback diagnostic."""
|
|
c = make_container("some-future-state")
|
|
events = run_check(agent, [c], monkeypatch)
|
|
assert len(events) == 1
|
|
etype, severity, _service, _msg, payload = events[0]
|
|
assert etype == "container_state_unexpected"
|
|
assert severity == "medium"
|
|
assert payload["status"] == "some-future-state"
|
|
|
|
|
|
def test_removing_is_conscious_silent_skip(agent, monkeypatch):
|
|
"""removing is an ephemeral teardown state — deliberately no event (documented)."""
|
|
c = make_container("removing")
|
|
events = run_check(agent, [c], monkeypatch)
|
|
assert events == []
|
|
|
|
|
|
def test_created_is_skipped(agent, monkeypatch):
|
|
c = make_container("created")
|
|
events = run_check(agent, [c], monkeypatch)
|
|
assert events == []
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Managed-container filter still applies
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_unmanaged_container_is_ignored(agent, monkeypatch):
|
|
"""No restart policy → not a long-running managed service → no events, even if restarting."""
|
|
c = make_container("restarting", restart_policy="no", restart_count=9)
|
|
events = run_check(agent, [c], monkeypatch)
|
|
assert events == []
|
|
|
|
|
|
def test_no_docker_client_is_noop(agent, monkeypatch):
|
|
agent.docker_client = None
|
|
# Should simply return without raising.
|
|
agent.check_containers()
|