dff76ecd30
- node-agent: service_healthy emitowany tylko przy przejsciu w stan zdrowy (per-service in-memory state), nie co cykl dla kazdego zdrowego serwisu. To samo dla control-plane HTTP probe. healthcheck_failed/containers_not_running/ incydenty pozostaja emitowane bez zmian (realne sygnaly). - node-agent: naprawiono _cleanup_control_plane_fs — istniejaca retencja eventow byla martwa od migracji observer_checkpoint.json na epoch-int (str(path) <= int rzucal TypeError, lapane cicho przez broad except). Teraz porownuje epoch-do-epoch i czysci wylacznie service_healthy/node_health starsze niz checkpoint + 3-dniowy bufor; healthcheck_failed/incydenty/ha_* zachowane bezterminowo. - scripts/maintenance/cleanup_event_backlog.py: jednorazowy skrypt do bezpiecznego czyszczenia backlogu na VPS (dry-run domyslnie, --apply do usuniecia). Ten sam warunek: typ szumu + starsze niz checkpoint + 1h bufor.
112 lines
4.4 KiB
Python
112 lines
4.4 KiB
Python
"""Tests for cleanup_event_backlog.py — the event-flood backlog cleanup script.
|
|
|
|
Covers the safety invariant the whole cleanup depends on: only events that are
|
|
(a) a noise type, (b) strictly older than the observer's per-node checkpoint,
|
|
and (c) past the extra min-age safety buffer are ever deleted. Everything
|
|
else — unprocessed events, real signals, foreign/ghost node dirs — survives.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
import cleanup_event_backlog as cleanup # noqa: E402
|
|
|
|
|
|
def _write(node_dir: Path, ts: int, etype: str, svc: str = "svc") -> Path:
|
|
p = node_dir / f"evt-{node_dir.name}-{ts}-{etype}-{svc}.json"
|
|
p.write_text("{}")
|
|
return p
|
|
|
|
|
|
def test_deletes_only_processed_noise_past_buffer(tmp_path):
|
|
events_dir = tmp_path / "events"
|
|
node_dir = events_dir / "piha"
|
|
node_dir.mkdir(parents=True)
|
|
now = int(time.time())
|
|
checkpoint = now - 100
|
|
|
|
old_healthy = _write(node_dir, now - 7200, "service_healthy")
|
|
old_node_health = _write(node_dir, now - 7200, "node_health")
|
|
recent_healthy = _write(node_dir, now - 30, "service_healthy", svc="svc2") # within buffer
|
|
unprocessed = _write(node_dir, now + 50, "service_healthy", svc="svc3") # after checkpoint
|
|
real_signal = _write(node_dir, now - 999999, "healthcheck_failed")
|
|
|
|
checkpoints = {"piha": checkpoint}
|
|
candidates, counts = cleanup.scan(
|
|
{"service_healthy", "node_health"}, 3600, checkpoints, now, events_dir
|
|
)
|
|
|
|
assert set(candidates) == {old_healthy, old_node_health}
|
|
for f in (recent_healthy, unprocessed, real_signal):
|
|
assert f.exists()
|
|
assert counts["piha"]["healthcheck_failed"]["delete"] == 0
|
|
assert counts["piha"]["healthcheck_failed"]["keep"] == 1
|
|
|
|
|
|
def test_never_deletes_without_checkpoint_for_node(tmp_path):
|
|
"""A node dir with no checkpoint entry (ghost/foreign dir) must be left alone."""
|
|
events_dir = tmp_path / "events"
|
|
node_dir = events_dir / "be17cb6eb0f6"
|
|
node_dir.mkdir(parents=True)
|
|
now = int(time.time())
|
|
_write(node_dir, now - 999999, "service_healthy")
|
|
|
|
candidates, counts = cleanup.scan(
|
|
{"service_healthy", "node_health"}, 3600, {}, now, events_dir
|
|
)
|
|
assert candidates == []
|
|
assert counts["be17cb6eb0f6"]["service_healthy"]["keep"] == 1
|
|
|
|
|
|
def test_never_deletes_incidents_or_unknown_types(tmp_path):
|
|
events_dir = tmp_path / "events"
|
|
node_dir = events_dir / "vps"
|
|
node_dir.mkdir(parents=True)
|
|
now = int(time.time())
|
|
checkpoint = now - 100
|
|
ha_event = _write(node_dir, now - 999999, "ha_entity_unavailable_long")
|
|
incident_like = _write(node_dir, now - 999999, "containers_not_running")
|
|
|
|
candidates, _counts = cleanup.scan(
|
|
{"service_healthy", "node_health"}, 3600, {"vps": checkpoint}, now, events_dir
|
|
)
|
|
assert candidates == []
|
|
assert ha_event.exists()
|
|
assert incident_like.exists()
|
|
|
|
|
|
def test_apply_actually_deletes_and_dry_run_does_not(tmp_path, capsys, monkeypatch):
|
|
events_dir = tmp_path / "events"
|
|
state_dir = tmp_path / "state"
|
|
node_dir = events_dir / "piha"
|
|
node_dir.mkdir(parents=True)
|
|
state_dir.mkdir(parents=True)
|
|
now = int(time.time())
|
|
checkpoint = now - 100
|
|
old_healthy = _write(node_dir, now - 7200, "service_healthy")
|
|
(state_dir / "observer_checkpoint.json").write_text(
|
|
json.dumps({"node_checkpoints": {"piha": checkpoint}})
|
|
)
|
|
|
|
monkeypatch.setattr(cleanup, "EVENTS_DIR", events_dir)
|
|
monkeypatch.setattr(cleanup, "CHECKPOINT_FILE", state_dir / "observer_checkpoint.json")
|
|
monkeypatch.setattr(sys, "argv", ["cleanup_event_backlog.py"])
|
|
cleanup.main()
|
|
assert old_healthy.exists() # dry run: untouched
|
|
|
|
monkeypatch.setattr(sys, "argv", ["cleanup_event_backlog.py", "--apply"])
|
|
cleanup.main()
|
|
assert not old_healthy.exists() # applied: deleted
|
|
|
|
|
|
def test_type_extraction_matches_node_agent_filename_convention():
|
|
assert cleanup._type_from_name("evt-piha-1784215978-service_healthy-mosquitto.json") == "service_healthy"
|
|
assert cleanup._type_from_name("evt-vps-1784215978-node_health-node.json") == "node_health"
|
|
assert cleanup._type_from_name("evt-piha-1784215978-healthcheck_failed-z2m.json") == "healthcheck_failed"
|
|
assert cleanup._type_from_name("evt-piha-1784215978-ha_entity_unavailable_long-node.json") == \
|
|
"ha_entity_unavailable_long"
|