01db57ab82
126 plikow (md, yaml, sh, py) odwolywalo sie do sciezek sprzed migracji.
15 markdown-linkow [..](..) -> policzona sciezka WZGLEDNA wobec pliku
odsylajacego (wczesniej czesc z nich byla repo-root-relative i nie
rozwiazywala sie z katalogu, w ktorym lezala)
200 odwolan tekstowych (backticki, proza, yaml, importy w kodzie)
-> nowa sciezka repo-root-relative, zgodnie z konwencja repo
5 linkow rodzenstwa (gole nazwy plikow, np. "](DEPLOY.md)") — dzialaly
tylko w starym katalogu; przeliczone recznie
Objete m.in.: CLAUDE.md (scripts/onboard/README.md -> kb/runbooks/
node-onboarding-tool.md, docs/backlog.md -> kb/phases/backlog.md),
README.md, .claude/skills/, 20 session logow, kod jobow.
Ostatnie 5 odwolan pochodzi z tresci wciagnietej rebasem z origin/master
(session log 2026-07-31, override node-agenta na SOLARII, dwie pozycje
backlogu) — wskazywaly na docs/incidents/, docs/kb/modules/ i
services/narty27/README.md sprzed migracji.
Dodany wzajemny link miedzy kb/services/control-plane.md (stub kodu)
a kb/subsystems/control-plane.md (opis, deprecated) — dwa dokumenty o tym
samym systemie, latwe do pomylenia.
Weryfikacja na 790 plikach: 0 odwolan do starych sciezek,
0 martwych linkow markdown. Lint OKF: 190/190 plikow ZGODNE.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
1176 lines
54 KiB
Python
1176 lines
54 KiB
Python
"""
|
|
node_agent.py — Homelab node health monitor daemon.
|
|
|
|
Runs as a Docker container on every managed node. Each cycle it:
|
|
1. Collects system metrics (disk, memory, CPU).
|
|
2. Checks Docker container health.
|
|
3. Emits structured event JSON files to /opt/homelab/events/<node-name>/.
|
|
4. Applies safe Docker / filesystem cleanup per the conservative policy.
|
|
5. Pulls any action the control-plane executor has dispatched to this node
|
|
(container_restart only — whitelisted, self-restart guarded, idempotent)
|
|
and executes it locally through the docker socket, reporting the result
|
|
as an action_result event.
|
|
6. Optionally rsyncs events (including action_result) to VPS so the
|
|
control-plane observer/executor can process them.
|
|
|
|
Cleanup policy (matches health-monitor.sh). Every type below is additionally
|
|
rate-limited to one cleanup run per CLEANUP_INTERVAL_SECS (24 h):
|
|
lte_node (chelsty-infra, chelsty-ha) : NO cleanup, NO image operations
|
|
sd_card (piha, saturn) : dangling images + stopped containers
|
|
ai_node (solaria) : dangling + containers + build cache,
|
|
NEVER docker image prune -a
|
|
standard (vps) : dangling + containers + build cache +
|
|
control-plane filesystem rotation
|
|
|
|
NEVER TOUCHED on any node:
|
|
/opt/homelab/data/ Frigate recordings, Ollama models, HA db, MQTT state
|
|
/opt/homelab/config/ All hand-crafted and repo-seeded configuration
|
|
/opt/homelab/state/ Heartbeat files, observer checkpoint
|
|
actions/pending|approved|running Live work queue
|
|
Stopped containers with a restart policy or a compose project label — a
|
|
stopped service is recorded operator intent, not garbage. Container cleanup
|
|
enumerates explicitly and never calls the unfiltered containers.prune().
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import shutil
|
|
import socket
|
|
import subprocess
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
try:
|
|
import docker as docker_sdk
|
|
DOCKER_AVAILABLE = True
|
|
except ImportError:
|
|
DOCKER_AVAILABLE = False
|
|
docker_sdk = None
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Logging
|
|
# ---------------------------------------------------------------------------
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s - %(levelname)s - %(message)s"
|
|
)
|
|
logger = logging.getLogger("node-agent")
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Runtime paths
|
|
# ---------------------------------------------------------------------------
|
|
RUNTIME_PATH = Path(os.getenv("RUNTIME_PATH", "/opt/homelab"))
|
|
REPO_ROOT = Path(os.getenv("REPO_ROOT", "/repo"))
|
|
EVENTS_DIR = RUNTIME_PATH / "events"
|
|
STATE_DIR = RUNTIME_PATH / "state"
|
|
LOGS_DIR = RUNTIME_PATH / "logs"
|
|
ACTIONS_DIR = RUNTIME_PATH / "actions"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Node identity
|
|
# ---------------------------------------------------------------------------
|
|
NODE_NAME = os.getenv("NODE_NAME") or socket.gethostname()
|
|
NODE_TYPE = os.getenv("NODE_TYPE", "") # override auto-detection if set
|
|
|
|
VPS_NODE_NAME = "vps"
|
|
LTE_NODES = {"chelsty-infra", "chelsty-ha"}
|
|
SD_CARD_NODES = {"piha", "saturn"}
|
|
AI_NODES = {"solaria"}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Event shipping (optional — requires VPS_EVENTS_HOST + SSH key in container)
|
|
# ---------------------------------------------------------------------------
|
|
VPS_EVENTS_HOST = os.getenv("VPS_EVENTS_HOST", "")
|
|
VPS_EVENTS_USER = os.getenv("VPS_EVENTS_USER", "oskar")
|
|
VPS_EVENTS_PATH = os.getenv("VPS_EVENTS_PATH", "/opt/homelab/events")
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Remediation dispatch (pull-based — reuses the same VPS_EVENTS_HOST / SSH key
|
|
# as event shipping above, just in the opposite direction). See
|
|
# kb/phases/backlog.md "PROJEKT: remediacja bez SSH": the control-plane executor on
|
|
# VPS has no SSH client and no key to the fleet, so it never reaches out to a
|
|
# node directly. Instead it drops a small JSON file under
|
|
# /opt/homelab/actions/dispatch/<node>/ on VPS; the node-agent for that node
|
|
# pulls its own subdirectory (remote nodes, via rsync over the existing
|
|
# shipping key) or reads it directly (VPS's own node-agent — same filesystem,
|
|
# no network hop) and executes it locally through the docker socket it
|
|
# already holds.
|
|
# ---------------------------------------------------------------------------
|
|
VPS_DISPATCH_PATH = os.getenv("VPS_DISPATCH_PATH", "/opt/homelab/actions/dispatch")
|
|
|
|
# Action types the agent is willing to execute on its own. Deliberately just
|
|
# one to start: redeploy stays VPS/manual, disk_cleanup is out of scope for
|
|
# this change (see kb/phases/backlog.md). Anything not in this set is refused with
|
|
# a clear action_result error rather than silently ignored.
|
|
ALLOWED_DISPATCH_ACTION_TYPES = {"container_restart"}
|
|
|
|
# Container names the agent will never restart, regardless of what a dispatch
|
|
# file asks for. Restarting node-agent's own container mid-execution would
|
|
# kill the very process performing the restart. Matches the CLAUDE.md
|
|
# convention that a service's container name equals its service name.
|
|
SELF_RESTART_GUARD_NAMES = {"node-agent"}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Thresholds
|
|
# ---------------------------------------------------------------------------
|
|
DISK_WARN_PCT = 75
|
|
DISK_CRIT_PCT = 85
|
|
MEM_WARN_PCT = 85
|
|
MEM_CRIT_PCT = 95
|
|
|
|
# A container Docker reports as "restarting" that has accumulated at least this
|
|
# many restarts is treated as a genuine crash-loop (a real fault) rather than a
|
|
# benign post-deploy / one-off restart.
|
|
#
|
|
# Rationale for the threshold: Docker's restart backoff keeps a *genuinely*
|
|
# crash-looping container cycling through the "restarting" state, so a 60 s
|
|
# health cycle is very likely to catch it there. A container that merely
|
|
# restarted once or twice (fresh deploy, a single transient failure that then
|
|
# recovers) is almost always back in "running" by the next cycle. Requiring
|
|
# RestartCount >= 3 clears normal deploy churn while still catching a container
|
|
# that is actually stuck flapping. Configurable via env for tuning per fleet.
|
|
CRASH_LOOP_RESTART_THRESHOLD = int(os.getenv("CRASH_LOOP_RESTART_THRESHOLD", "3"))
|
|
|
|
# Minimum gap between Docker cleanup runs, on EVERY cleanup-eligible node type
|
|
# (was sd_card-only until the 2026-07-30 unfiltered-prune incident)
|
|
CLEANUP_INTERVAL_SECS = 86_400
|
|
LAST_CLEANUP_FILE = STATE_DIR / "last-docker-cleanup"
|
|
|
|
# How long to wait between full health-check cycles
|
|
HEALTH_CHECK_INTERVAL = int(os.getenv("CHECK_INTERVAL", "60"))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _resolve_node_type() -> str:
|
|
if NODE_TYPE:
|
|
return NODE_TYPE
|
|
if NODE_NAME in LTE_NODES:
|
|
return "lte_node"
|
|
if NODE_NAME in SD_CARD_NODES:
|
|
return "sd_card"
|
|
if NODE_NAME in AI_NODES:
|
|
return "ai_node"
|
|
return "standard"
|
|
|
|
|
|
def _utc_iso() -> str:
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
# Matches evt-<node>-<unixts>-<type>-<svc>.json (this module's own emit_event
|
|
# naming, mirrored by observer.py's _EVENT_TS_RE / _ts_from_event_name so both
|
|
# sides of the checkpoint agree on how to read a filename's embedded epoch).
|
|
_EVENT_TS_RE = re.compile(r"-(\d{9,11})-")
|
|
_EVENT_TYPE_RE = re.compile(r"^evt-.+?-\d{9,11}-(.+)$")
|
|
|
|
|
|
def _event_ts_from_filename(name: str):
|
|
"""Return the embedded <unixts> from an event filename, or None if absent."""
|
|
m = _EVENT_TS_RE.search(Path(name).stem)
|
|
return int(m.group(1)) if m else None
|
|
|
|
|
|
def _event_type_from_filename(name: str) -> str:
|
|
"""Best-effort event type from evt-<node>-<ts>-<type>-<svc>.json.
|
|
|
|
Used only to decide retention eligibility (never for dispatch), so an
|
|
imprecise match on an unrecognized type is safe: it simply keeps the file.
|
|
"""
|
|
stem = Path(name).stem
|
|
m = _EVENT_TYPE_RE.match(stem)
|
|
if not m:
|
|
return "unknown"
|
|
rest = m.group(1)
|
|
for t in _RETENTION_NOISE_TYPES:
|
|
if rest == t or rest.startswith(t + "-"):
|
|
return t
|
|
return rest
|
|
|
|
|
|
def _checkpoint_ts_from_value(value) -> int:
|
|
"""Coerce a stored observer_checkpoint.json value into an int epoch.
|
|
|
|
Mirrors observer.py's own _checkpoint_ts_from_value: current format is an
|
|
int/float epoch; older observer builds stored a lexical path string, from
|
|
which the embedded <unixts> is extracted. Unparseable/absent -> 0, which
|
|
compares as "nothing processed yet" and safely keeps every file for that
|
|
node rather than guessing a checkpoint too high and deleting unprocessed
|
|
events.
|
|
"""
|
|
if isinstance(value, bool):
|
|
return 0
|
|
if isinstance(value, (int, float)):
|
|
return int(value)
|
|
if isinstance(value, str) and value:
|
|
ts = _event_ts_from_filename(value)
|
|
if ts is not None:
|
|
return ts
|
|
return 0
|
|
|
|
|
|
# Event types eligible for the ongoing retention sweep in
|
|
# _cleanup_control_plane_fs: pure positive-confirmation noise, never the
|
|
# actionable signals (healthcheck_failed, containers_not_running, ha_*,
|
|
# incidents, disk/memory/cpu pressure, deployment/remediation records, node
|
|
# liveness transitions), which are kept indefinitely by this method — an
|
|
# operator or the panel event feed may need to look back at those.
|
|
_RETENTION_NOISE_TYPES = frozenset({"service_healthy", "node_health"})
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# NodeAgent
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class NodeAgent:
|
|
def __init__(self):
|
|
self.node_name = NODE_NAME
|
|
self.node_type = _resolve_node_type()
|
|
self._ensure_dirs()
|
|
|
|
# Tracks the last known health classification per service (by canonical
|
|
# name — container service names AND the synthetic "control-plane" key
|
|
# used by _check_control_plane_health share this dict) so service_healthy
|
|
# is emitted only on the unhealthy->healthy transition, not once per cycle
|
|
# for a service that is already healthy. In-memory only: reset on process
|
|
# restart is acceptable — it just re-emits one confirmation event, not a
|
|
# flood, and the observer treats that re-confirmation as idempotent (see
|
|
# process_event).
|
|
self._service_health_state: dict = {}
|
|
|
|
self.docker_client = None
|
|
if DOCKER_AVAILABLE:
|
|
try:
|
|
self.docker_client = docker_sdk.from_env()
|
|
logger.info("Docker SDK connected")
|
|
except Exception as exc:
|
|
logger.warning(f"Docker unavailable: {exc}")
|
|
|
|
logger.info(f"node-agent starting: node={self.node_name} type={self.node_type}")
|
|
|
|
# ------------------------------------------------------------------
|
|
# Directories
|
|
# ------------------------------------------------------------------
|
|
|
|
def _ensure_dirs(self):
|
|
for d in [EVENTS_DIR, STATE_DIR, LOGS_DIR,
|
|
EVENTS_DIR / self.node_name]:
|
|
d.mkdir(parents=True, exist_ok=True)
|
|
|
|
def _node_events_dir(self) -> Path:
|
|
return EVENTS_DIR / self.node_name
|
|
|
|
# ------------------------------------------------------------------
|
|
# Event emission
|
|
# ------------------------------------------------------------------
|
|
|
|
def emit_event(self, event_type: str, severity: str, service,
|
|
message: str, payload: dict = None):
|
|
ts = int(time.time())
|
|
# Include service slug in the ID so that multiple events of the same type
|
|
# emitted within the same second (e.g. service_healthy for N containers)
|
|
# don't overwrite each other — each gets a unique filename.
|
|
svc_slug = re.sub(r"[^a-z0-9]", "-", (service or "node").lower())[:32].strip("-")
|
|
event_id = f"evt-{self.node_name}-{ts}-{event_type}-{svc_slug}"
|
|
event = {
|
|
"id": event_id,
|
|
"timestamp": ts,
|
|
"date": _utc_iso(),
|
|
"type": event_type,
|
|
"severity": severity,
|
|
"node": self.node_name,
|
|
"service": service or "",
|
|
"message": message,
|
|
"payload": payload or {},
|
|
}
|
|
path = self._node_events_dir() / f"{event_id}.json"
|
|
try:
|
|
path.write_text(json.dumps(event, indent=2))
|
|
except Exception as exc:
|
|
logger.error(f"Failed to write event {event_id}: {exc}")
|
|
|
|
# ------------------------------------------------------------------
|
|
# System metrics
|
|
# ------------------------------------------------------------------
|
|
|
|
def check_disk(self) -> int:
|
|
"""
|
|
Check disk usage of the filesystem that hosts RUNTIME_PATH.
|
|
Using shutil.disk_usage on the mounted runtime path works both
|
|
natively on the host and inside a container that mounts /opt/homelab
|
|
from the host — the reported usage reflects the host partition.
|
|
"""
|
|
try:
|
|
usage = shutil.disk_usage(str(RUNTIME_PATH))
|
|
pct = int(usage.used * 100 / usage.total)
|
|
avail_mb = int(usage.free / (1024 * 1024))
|
|
total_mb = int(usage.total / (1024 * 1024))
|
|
payload = {
|
|
"usage_pct": pct, "avail_mb": avail_mb,
|
|
"total_mb": total_mb, "mount": str(RUNTIME_PATH),
|
|
}
|
|
if pct >= DISK_CRIT_PCT:
|
|
logger.warning(f"Disk critical: {pct}% used")
|
|
self.emit_event(
|
|
"disk_pressure", "high", None,
|
|
f"Disk usage critical: {pct}% on {RUNTIME_PATH} ({avail_mb} MB free)",
|
|
payload,
|
|
)
|
|
elif pct >= DISK_WARN_PCT:
|
|
logger.info(f"Disk elevated: {pct}% used")
|
|
self.emit_event(
|
|
"disk_pressure", "medium", None,
|
|
f"Disk usage elevated: {pct}% on {RUNTIME_PATH} ({avail_mb} MB free)",
|
|
payload,
|
|
)
|
|
return pct
|
|
except Exception as exc:
|
|
logger.error(f"Disk check failed: {exc}")
|
|
return 0
|
|
|
|
def check_memory(self) -> int:
|
|
"""Read host memory from /proc/meminfo (visible in container without special mounts)."""
|
|
try:
|
|
info: dict = {}
|
|
with open("/proc/meminfo") as fh:
|
|
for line in fh:
|
|
k, v = line.split(":")
|
|
info[k.strip()] = int(v.strip().split()[0])
|
|
|
|
total = info.get("MemTotal", 1)
|
|
avail = info.get("MemAvailable", total)
|
|
pct = int((total - avail) * 100 / total)
|
|
avail_mb = avail // 1024
|
|
total_mb = total // 1024
|
|
payload = {"usage_pct": pct, "avail_mb": avail_mb, "total_mb": total_mb}
|
|
|
|
if pct >= MEM_CRIT_PCT:
|
|
logger.warning(f"Memory critical: {pct}%")
|
|
self.emit_event("high_memory", "high", None,
|
|
f"Memory usage critical: {pct}% ({avail_mb} MB available)", payload)
|
|
elif pct >= MEM_WARN_PCT:
|
|
self.emit_event("high_memory", "medium", None,
|
|
f"Memory usage elevated: {pct}% ({avail_mb} MB available)", payload)
|
|
return pct
|
|
except Exception as exc:
|
|
logger.error(f"Memory check failed: {exc}")
|
|
return 0
|
|
|
|
def check_cpu(self) -> int:
|
|
"""Two-sample /proc/stat delta for accurate instantaneous CPU usage."""
|
|
try:
|
|
def _read():
|
|
with open("/proc/stat") as fh:
|
|
for line in fh:
|
|
if line.startswith("cpu "):
|
|
vals = list(map(int, line.split()[1:]))
|
|
return vals[3], sum(vals) # idle, total
|
|
return 0, 1
|
|
|
|
idle1, total1 = _read()
|
|
time.sleep(0.5)
|
|
idle2, total2 = _read()
|
|
|
|
d_total = total2 - total1
|
|
d_idle = idle2 - idle1
|
|
pct = 100 - int(d_idle * 100 / d_total) if d_total else 0
|
|
|
|
if pct >= 90:
|
|
self.emit_event("high_cpu", "medium", None,
|
|
f"CPU usage elevated: {pct}%", {"usage_pct": pct})
|
|
return pct
|
|
except Exception as exc:
|
|
logger.error(f"CPU check failed: {exc}")
|
|
return 0
|
|
|
|
# ------------------------------------------------------------------
|
|
# Docker container health
|
|
# ------------------------------------------------------------------
|
|
|
|
def _canonical_container_name(self, c) -> str:
|
|
"""Return a stable, human-readable service name for a container.
|
|
|
|
Priority:
|
|
1. com.docker.compose.service label — always the clean compose-file key
|
|
(e.g. "mosquitto", "zigbee2mqtt"), immune to the hash-prefix corruption
|
|
Docker uses for stale project-state tracking entries.
|
|
2. c.name with hash prefix stripped — fallback for non-Compose containers.
|
|
When a container is removed outside of compose and then recreated, Docker
|
|
stores the old container record as "<12-char-hex-id>_<original-name>".
|
|
c.name returns that corrupted form; we strip the prefix here.
|
|
|
|
Using c.name directly is the source of ghost service keys like
|
|
"vps/9e36297651e7_control-plane-observer" that accumulate in services.json
|
|
every time containers are rebuilt.
|
|
"""
|
|
labels = c.attrs.get("Config", {}).get("Labels", {}) or {}
|
|
compose_svc = labels.get("com.docker.compose.service", "").strip()
|
|
if compose_svc:
|
|
return compose_svc
|
|
# Strip Docker-internal stale-state prefix: "<12-char hex>_<real-name>"
|
|
name = c.name
|
|
if (len(name) > 13
|
|
and name[12] == "_"
|
|
and all(ch in "0123456789abcdef" for ch in name[:12])):
|
|
return name[13:]
|
|
return name
|
|
|
|
def check_containers(self):
|
|
if not self.docker_client:
|
|
return
|
|
try:
|
|
containers = self.docker_client.containers.list(all=True)
|
|
except Exception as exc:
|
|
logger.error(f"Docker container list failed: {exc}")
|
|
return
|
|
|
|
for c in containers:
|
|
try:
|
|
name = self._canonical_container_name(c)
|
|
status = c.status
|
|
host_config = c.attrs.get("HostConfig", {})
|
|
restart_policy = host_config.get("RestartPolicy", {}).get("Name", "")
|
|
health_status = (c.attrs.get("State", {})
|
|
.get("Health", {})
|
|
.get("Status", ""))
|
|
# RestartCount is a top-level inspect field (sibling of State),
|
|
# not nested under State. It monotonically counts how many times
|
|
# Docker has auto-restarted this container — the signal that
|
|
# distinguishes a crash-loop from a one-off restart.
|
|
restart_count = c.attrs.get("RestartCount", 0)
|
|
|
|
# Skip containers in "created" state — these are Docker Compose
|
|
# internal tracking artifacts (never started, often hash-prefixed)
|
|
# that appear when a container is rebuilt outside of compose.
|
|
# This is a conscious skip, not a blind spot: a "created" container
|
|
# is not a running service and has no fault to report.
|
|
if status == "created":
|
|
continue
|
|
|
|
# Only track containers with a restart policy (long-running services)
|
|
is_managed = restart_policy in ("unless-stopped", "always", "on-failure")
|
|
if not is_managed:
|
|
continue
|
|
|
|
base_payload = {
|
|
"container": name,
|
|
"status": status,
|
|
"restart_policy": restart_policy,
|
|
"restart_count": restart_count,
|
|
}
|
|
|
|
# Exited container that carries an auto-restart policy
|
|
if status in ("exited", "dead"):
|
|
self._service_health_state[name] = False
|
|
logger.warning(f"Container exited: {name} (restart={restart_policy})")
|
|
self.emit_event(
|
|
"containers_not_running", "high", name,
|
|
f"Container '{name}' has exited (restart={restart_policy})",
|
|
base_payload,
|
|
)
|
|
|
|
# Container stuck in Docker's "restarting" state. This is the gap
|
|
# that previously made crash-loops invisible: "restarting" matched
|
|
# neither the exited/dead branch nor the running branch, so a
|
|
# container flapping under its restart policy emitted ZERO events
|
|
# and the observer kept showing its last (stale) "healthy" state.
|
|
#
|
|
# We split on RestartCount so a benign post-deploy restart does not
|
|
# alarm, while a real crash-loop escalates to the same actionable
|
|
# signal as an exited container.
|
|
elif status == "restarting":
|
|
self._service_health_state[name] = False
|
|
if restart_count >= CRASH_LOOP_RESTART_THRESHOLD:
|
|
# Genuine crash-loop: reuse containers_not_running so it
|
|
# rides the existing, supervisor-wired remediation path
|
|
# (parity with exited/dead — the container is, by
|
|
# definition, not staying running).
|
|
logger.warning(
|
|
f"Container crash-looping: {name} "
|
|
f"(restarting, {restart_count} restarts, restart={restart_policy})"
|
|
)
|
|
self.emit_event(
|
|
"containers_not_running", "high", name,
|
|
f"Container '{name}' is crash-looping "
|
|
f"(restarting, {restart_count} restarts, restart={restart_policy})",
|
|
{**base_payload, "crash_loop": True},
|
|
)
|
|
else:
|
|
# Fresh / transient restart. Make it VISIBLE (so the state
|
|
# is no longer a silent hole) but do NOT trigger remediation
|
|
# — a restart action here would be premature and noisy. If
|
|
# it keeps flapping, RestartCount crosses the threshold on a
|
|
# later cycle and escalates to containers_not_running above.
|
|
# container_restarting is intentionally observational: it is
|
|
# not wired to any supervisor trigger.
|
|
logger.info(
|
|
f"Container restarting: {name} "
|
|
f"({restart_count} restarts) — watching for crash-loop"
|
|
)
|
|
self.emit_event(
|
|
"container_restarting", "low", name,
|
|
f"Container '{name}' is restarting "
|
|
f"({restart_count} restarts) — watching for crash-loop",
|
|
{**base_payload, "crash_loop": False},
|
|
)
|
|
|
|
# Running container with a failing health check
|
|
elif status == "running" and health_status == "unhealthy":
|
|
self._service_health_state[name] = False
|
|
logger.warning(f"Container unhealthy: {name}")
|
|
self.emit_event(
|
|
"healthcheck_failed", "high", name,
|
|
f"Container '{name}' is running but its health check is failing",
|
|
{**base_payload, "health_status": health_status},
|
|
)
|
|
|
|
# Running container that is healthy — confirm to observer so that
|
|
# services.json stays populated for the supervisor's drift detection.
|
|
# Without this, the supervisor sees services.json as empty and treats
|
|
# all desired services as "missing", flooding the action queue.
|
|
#
|
|
# Emitted ONLY on the unhealthy/unknown -> healthy transition, not
|
|
# every cycle: a healthy service otherwise re-confirms itself once
|
|
# per health-check interval forever, which is exactly the flood
|
|
# that paralyzed the supervisor's reconcile() glob (358k backlog
|
|
# files, 91% service_healthy). The positive "still healthy" state is
|
|
# already carried by the presence of a `healthy` services.json entry
|
|
# (written once here) plus the node_health heartbeat every cycle —
|
|
# so a stuck/frozen node_agent is still visible via node liveness,
|
|
# without needing a fresh service_healthy file per service per cycle.
|
|
elif status == "running":
|
|
if self._service_health_state.get(name) is not True:
|
|
self.emit_event(
|
|
"service_healthy", "info", name,
|
|
f"Container '{name}' is running",
|
|
{**base_payload, "health_status": health_status or "none"},
|
|
)
|
|
self._service_health_state[name] = True
|
|
|
|
# Deliberately paused. docker pause is always an operator/tool
|
|
# action (Docker never auto-pauses), so it is not a fault — but a
|
|
# managed service should not normally sit paused, so surface it
|
|
# observationally rather than swallowing it. Non-actionable.
|
|
elif status == "paused":
|
|
self._service_health_state[name] = False
|
|
logger.info(f"Container paused: {name}")
|
|
self.emit_event(
|
|
"container_state_unexpected", "medium", name,
|
|
f"Container '{name}' is paused (restart={restart_policy})",
|
|
base_payload,
|
|
)
|
|
|
|
# Ephemeral teardown state: the container is being removed and will
|
|
# be gone (or replaced) by the next cycle. Emitting a fault here
|
|
# would be a guaranteed false positive on every recreate/deploy, so
|
|
# this is a conscious, documented skip — not a silent hole.
|
|
elif status == "removing":
|
|
logger.debug(f"Container removing (transient teardown): {name}")
|
|
|
|
# FALLBACK — any Docker state we do not explicitly handle (including
|
|
# states a future Docker engine might introduce). Never fall through
|
|
# silently: emit a diagnostic so a new/unknown state becomes visible
|
|
# instead of quietly recreating the very blind spot this fix closes.
|
|
else:
|
|
self._service_health_state[name] = False
|
|
logger.warning(
|
|
f"Container in unhandled Docker state: {name} (state='{status}')"
|
|
)
|
|
self.emit_event(
|
|
"container_state_unexpected", "medium", name,
|
|
f"Container '{name}' is in unhandled Docker state '{status}' "
|
|
f"(restart={restart_policy})",
|
|
base_payload,
|
|
)
|
|
|
|
except Exception as exc:
|
|
logger.error(f"Error checking container {c.name}: {exc}")
|
|
|
|
# ------------------------------------------------------------------
|
|
# Safe Docker cleanup
|
|
# ------------------------------------------------------------------
|
|
|
|
def _cleanup_rate_ok(self) -> bool:
|
|
"""Return True only if CLEANUP_INTERVAL_SECS has elapsed since last cleanup.
|
|
|
|
Applies to EVERY cleanup-eligible node type, not just sd_card. Before
|
|
R2 this guard covered sd_card alone, so ai_node and standard pruned
|
|
every 60 s — 1440 chances a day to delete a deliberately stopped
|
|
container, for `0 MB reclaimed` in practically every cycle.
|
|
"""
|
|
if LAST_CLEANUP_FILE.exists():
|
|
try:
|
|
last_ts = int(LAST_CLEANUP_FILE.read_text().strip())
|
|
elapsed = time.time() - last_ts
|
|
if elapsed < CLEANUP_INTERVAL_SECS:
|
|
logger.debug(
|
|
f"Docker cleanup rate-limited ({elapsed:.0f}s < {CLEANUP_INTERVAL_SECS}s)"
|
|
)
|
|
return False
|
|
except Exception:
|
|
pass
|
|
return True
|
|
|
|
def _mark_cleanup_done(self):
|
|
try:
|
|
LAST_CLEANUP_FILE.write_text(str(int(time.time())))
|
|
except Exception as exc:
|
|
logger.error(f"Failed to update cleanup timestamp: {exc}")
|
|
|
|
def _prune_dangling_images(self):
|
|
if not self.docker_client:
|
|
return
|
|
try:
|
|
result = self.docker_client.images.prune(filters={"dangling": True})
|
|
reclaimed = result.get("SpaceReclaimed", 0) // (1024 * 1024)
|
|
logger.info(f"Pruned dangling images ({reclaimed} MB reclaimed)")
|
|
except Exception as exc:
|
|
logger.error(f"Image prune failed: {exc}")
|
|
|
|
def _prune_stopped_containers(self):
|
|
"""Remove exited containers that are unambiguously disposable.
|
|
|
|
NEVER uses containers.prune(): the Docker API removes *every*
|
|
non-running container, ignoring RestartPolicy and compose labels. That
|
|
is what deleted `ollama` on SOLARIA 19 s after an operator stopped it
|
|
(docs/incidents/2026-07-30-ollama-solaria-vanish.md). No filter can fix
|
|
it either — `until` filters on creation time, not stop time, so it
|
|
never protects a long-lived service.
|
|
|
|
A container carrying `restart: unless-stopped|always|on-failure` or a
|
|
compose project label is recorded operator intent, exactly like the
|
|
paths in this module's NEVER TOUCHED list. Only one-off leftovers
|
|
(restart policy `no`, no compose project) are removed.
|
|
"""
|
|
if not self.docker_client:
|
|
return
|
|
try:
|
|
containers = self.docker_client.containers.list(
|
|
all=True, filters={"status": "exited"}
|
|
)
|
|
except Exception as exc:
|
|
logger.error(f"Container list for prune failed: {exc}")
|
|
return
|
|
|
|
removed, kept = [], 0
|
|
for c in containers:
|
|
try:
|
|
policy = (
|
|
(c.attrs.get("HostConfig") or {})
|
|
.get("RestartPolicy", {})
|
|
.get("Name", "")
|
|
)
|
|
if policy in ("unless-stopped", "always", "on-failure"):
|
|
kept += 1 # operator intent — do not touch
|
|
continue
|
|
if (c.labels or {}).get("com.docker.compose.project"):
|
|
kept += 1 # compose-managed — do not touch
|
|
continue
|
|
c.remove()
|
|
removed.append(c.name)
|
|
except Exception as exc:
|
|
logger.error(f"Failed to remove stopped container {c.name}: {exc}")
|
|
|
|
# R3: name what was deleted. The old log line reported megabytes only,
|
|
# which is why ollama's removal looked identical to every no-op cycle.
|
|
if removed:
|
|
logger.warning(
|
|
f"Removed {len(removed)} disposable stopped container(s): "
|
|
f"{', '.join(removed)} (kept {kept} managed)"
|
|
)
|
|
else:
|
|
logger.info(f"No disposable stopped containers (kept {kept} managed)")
|
|
|
|
def _prune_build_cache(self):
|
|
if not self.docker_client:
|
|
return
|
|
try:
|
|
result = self.docker_client.api.prune_builds()
|
|
reclaimed = result.get("SpaceReclaimed", 0) // (1024 * 1024)
|
|
logger.info(f"Pruned build cache ({reclaimed} MB reclaimed)")
|
|
except Exception as exc:
|
|
# prune_builds() was added in docker-py 5.0; log and continue
|
|
logger.warning(f"Build cache prune unavailable or failed: {exc}")
|
|
|
|
def run_safe_cleanup(self):
|
|
"""Apply the per-node-type Docker cleanup policy."""
|
|
if self.node_type == "lte_node":
|
|
# No cleanup on LTE nodes — any Docker op risks a pull over a
|
|
# metered/intermittent connection.
|
|
logger.debug("Skipping Docker cleanup: LTE node")
|
|
return
|
|
|
|
# Rate limit applies to every cleanup-eligible node type (R2). Cleanup
|
|
# is housekeeping, not a health function: once per CLEANUP_INTERVAL_SECS
|
|
# reclaims the same space as once per minute, at a fraction of the I/O
|
|
# and of the exposure to accidental deletion.
|
|
if not self._cleanup_rate_ok():
|
|
return
|
|
|
|
if self.node_type == "sd_card":
|
|
self._prune_dangling_images()
|
|
self._prune_stopped_containers()
|
|
# No builder prune: minimise write cycles on SD card
|
|
self._mark_cleanup_done()
|
|
return
|
|
|
|
# ai_node and standard: dangling + containers + build cache
|
|
# ai_node: NEVER -a (would remove Ollama runtime images)
|
|
self._prune_dangling_images()
|
|
self._prune_stopped_containers()
|
|
self._prune_build_cache()
|
|
self._mark_cleanup_done()
|
|
|
|
# ------------------------------------------------------------------
|
|
# VPS-specific: control-plane filesystem rotation
|
|
# ------------------------------------------------------------------
|
|
|
|
def _cleanup_control_plane_fs(self):
|
|
"""
|
|
Rotate control-plane filesystem artefacts on VPS.
|
|
Safe targets only — never touches data/, config/, state/, or
|
|
live action directories (pending/approved/running).
|
|
"""
|
|
now = time.time()
|
|
seven_days = 7 * 86_400
|
|
thirty_days = 30 * 86_400
|
|
three_days = 3 * 86_400
|
|
|
|
# 1. Completed / failed actions older than 7 days
|
|
for status in ("completed", "failed"):
|
|
sdir = ACTIONS_DIR / status
|
|
if not sdir.exists():
|
|
continue
|
|
for f in sdir.glob("*.json"):
|
|
try:
|
|
if now - f.stat().st_mtime > seven_days:
|
|
f.unlink(missing_ok=True)
|
|
logger.info(f"Cleaned old {status} action: {f.name}")
|
|
except Exception as exc:
|
|
logger.error(f"Failed to remove {f}: {exc}")
|
|
|
|
# 2. Deploy logs older than 30 days
|
|
deploy_dir = LOGS_DIR / "deploy"
|
|
if deploy_dir.exists():
|
|
for f in deploy_dir.glob("*.log"):
|
|
try:
|
|
if now - f.stat().st_mtime > thirty_days:
|
|
f.unlink(missing_ok=True)
|
|
logger.info(f"Cleaned old deploy log: {f.name}")
|
|
except Exception as exc:
|
|
logger.error(f"Failed to remove {f}: {exc}")
|
|
|
|
# 3. Noise event files (service_healthy / node_health) that are BOTH
|
|
# already past the observer's per-node checkpoint (guaranteed
|
|
# processed into world_state — see _checkpoint_ts_from_value) AND
|
|
# older than 3 days (safety buffer past the checkpoint). The dual
|
|
# condition guarantees we never delete an unprocessed event, and
|
|
# the type restriction guarantees healthcheck_failed / incidents /
|
|
# ha_* / node-liveness events are kept indefinitely by this method.
|
|
#
|
|
# BUG FIX (event-flood cleanup): this previously compared str(f)
|
|
# (a full file path) against the checkpoint value with <=, which
|
|
# only ever worked when the checkpoint stored the pre-fix lexical
|
|
# path format. Since observer.py migrated node_checkpoints to int
|
|
# epoch timestamps, every comparison here raised
|
|
# "'<=' not supported between instances of 'str' and 'int'",
|
|
# silently caught by the broad except below — so this cleanup has
|
|
# been a no-op ever since, which is the direct cause of the 358k-file
|
|
# backlog that paralyzed the supervisor's reconcile(). Fixed by
|
|
# comparing epoch-to-epoch, same as observer.py's own checkpoint
|
|
# logic.
|
|
checkpoint_file = STATE_DIR / "observer_checkpoint.json"
|
|
node_checkpoints: dict = {}
|
|
if checkpoint_file.exists():
|
|
try:
|
|
cp = json.loads(checkpoint_file.read_text())
|
|
if "node_checkpoints" in cp:
|
|
node_checkpoints = {
|
|
node: _checkpoint_ts_from_value(val)
|
|
for node, val in (cp["node_checkpoints"] or {}).items()
|
|
}
|
|
elif "last_processed_file" in cp:
|
|
# Migrate old single-file format
|
|
old = cp.get("last_processed_file", "")
|
|
if old:
|
|
try:
|
|
node_dir = Path(old).relative_to(EVENTS_DIR).parts[0]
|
|
node_checkpoints = {node_dir: _checkpoint_ts_from_value(old)}
|
|
except Exception:
|
|
pass
|
|
except Exception as exc:
|
|
logger.error(f"Failed to read observer checkpoint: {exc}")
|
|
|
|
if node_checkpoints:
|
|
for f in EVENTS_DIR.glob("**/*.json"):
|
|
try:
|
|
etype = _event_type_from_filename(f.name)
|
|
if etype not in _RETENTION_NOISE_TYPES:
|
|
continue
|
|
# Determine which node directory this event belongs to
|
|
rel = Path(f).relative_to(EVENTS_DIR)
|
|
node_dir = str(rel.parts[0]) if rel.parts else "__unknown__"
|
|
checkpoint_ts = node_checkpoints.get(node_dir)
|
|
if not checkpoint_ts:
|
|
continue
|
|
event_ts = _event_ts_from_filename(f.name)
|
|
if event_ts is None:
|
|
event_ts = int(f.stat().st_mtime)
|
|
if (event_ts < checkpoint_ts
|
|
and now - f.stat().st_mtime > three_days):
|
|
f.unlink(missing_ok=True)
|
|
logger.info(f"Cleaned old event: {f.name}")
|
|
except Exception as exc:
|
|
logger.error(f"Failed to remove {f}: {exc}")
|
|
else:
|
|
logger.debug("No observer checkpoint present; skipping event cleanup")
|
|
|
|
# ------------------------------------------------------------------
|
|
# Optional: ship events to VPS via rsync
|
|
# ------------------------------------------------------------------
|
|
|
|
def _ship_events_to_vps(self):
|
|
"""
|
|
Rsync local events to VPS so the observer can process them.
|
|
Requires:
|
|
- VPS_EVENTS_HOST env var set to the VPS hostname/IP
|
|
- SSH key accessible inside the container (mount via docker-compose)
|
|
- The node is NOT VPS itself
|
|
"""
|
|
if not VPS_EVENTS_HOST or self.node_name == VPS_NODE_NAME:
|
|
return
|
|
|
|
local_dir = str(self._node_events_dir()) + "/"
|
|
remote_dir = (f"{VPS_EVENTS_USER}@{VPS_EVENTS_HOST}:"
|
|
f"{VPS_EVENTS_PATH}/{self.node_name}/")
|
|
cmd = [
|
|
"rsync", "-az", "--remove-source-files",
|
|
# --omit-dir-times: the remote per-node event dir on VPS is owned
|
|
# by aerbot:aerbot; the ssh user (oskar) is only a group member,
|
|
# so it can write files into the dir but cannot chown/chmod/touch
|
|
# the dir itself. -a implies -t (preserve times) which also
|
|
# applies to directories, so rsync would otherwise try to set the
|
|
# dir's mtime, get EPERM, and return non-zero even though every
|
|
# file transferred fine. Files still keep their mtime via -t;
|
|
# this only skips the (harmless, doomed) directory mtime set.
|
|
# TODO tech-debt: fix ownership of /opt/homelab/events/<node> on
|
|
# VPS so this workaround isn't needed (see kb/phases/backlog.md,
|
|
# "Tech-debt: globalny porządek uid/gid/uprawnień").
|
|
"--omit-dir-times",
|
|
# --no-perms/--no-owner/--no-group: same root cause as --omit-dir-times above.
|
|
# After omitting dir times, rsync next tried chmod on the same aerbot-owned
|
|
# dir and got EPERM ("failed to set permissions", exit 23) — then would try
|
|
# chown/chgrp for the same reason. -a implies -pgo; we drop all three for the
|
|
# destination dir. File CONTENT still transfers correctly (that is all the
|
|
# observer needs); only attribute-setting on the remote dir is skipped.
|
|
# TODO tech-debt: fix ownership of /opt/homelab/events/<node> on VPS so none
|
|
# of these workarounds are needed (see kb/phases/backlog.md, uid/gid section).
|
|
"--no-perms",
|
|
"--no-owner",
|
|
"--no-group",
|
|
# -F /dev/null: skip ~/.ssh/config entirely. The .ssh dir is
|
|
# mounted from the host oskar user into the container which runs
|
|
# as root; OpenSSH rejects config files owned by a different UID.
|
|
# UserKnownHostsFile=/dev/null pairs with StrictHostKeyChecking=no
|
|
# so we never try to write a known_hosts inside a read-only mount.
|
|
"-e", ("ssh -F /dev/null"
|
|
" -o StrictHostKeyChecking=no"
|
|
" -o UserKnownHostsFile=/dev/null"
|
|
" -o ConnectTimeout=10"
|
|
" -o BatchMode=yes"),
|
|
local_dir,
|
|
remote_dir,
|
|
]
|
|
try:
|
|
result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
|
|
if result.returncode == 0:
|
|
logger.debug(f"Events shipped to {remote_dir}")
|
|
else:
|
|
logger.warning(f"Event shipping failed: {result.stderr.strip()}")
|
|
except Exception as exc:
|
|
logger.warning(f"Event shipping error: {exc}")
|
|
|
|
# ------------------------------------------------------------------
|
|
# Remediation dispatch: pull queued actions for this node and execute them
|
|
# ------------------------------------------------------------------
|
|
|
|
def _dispatch_inbox_dir(self) -> Path:
|
|
return ACTIONS_DIR / "dispatch" / self.node_name
|
|
|
|
def _processed_marker_path(self, action_id: str) -> Path:
|
|
return STATE_DIR / "processed-actions" / f"{action_id}.done"
|
|
|
|
def _already_processed(self, action_id: str) -> bool:
|
|
return self._processed_marker_path(action_id).exists()
|
|
|
|
def _mark_processed(self, action_id: str):
|
|
marker = self._processed_marker_path(action_id)
|
|
try:
|
|
marker.parent.mkdir(parents=True, exist_ok=True)
|
|
marker.touch()
|
|
except Exception as exc:
|
|
logger.error(f"Failed to record action {action_id} as processed: {exc}")
|
|
|
|
def pull_dispatched_actions(self):
|
|
"""
|
|
Rsync-pull this node's dispatch inbox from VPS.
|
|
|
|
Reuses the exact same SSH key / connection settings as
|
|
_ship_events_to_vps, just as sender/receiver reversed: VPS is now the
|
|
source, this node the destination. --remove-source-files deletes the
|
|
action file on VPS once it has been fetched, so a dispatch file is
|
|
collected by exactly one node and cannot be re-pulled after that.
|
|
|
|
Requires VPS_EVENTS_HOST (same var event shipping uses) and is a
|
|
no-op on VPS itself, whose node-agent reads the dispatch dir directly
|
|
off the shared /opt/homelab mount — no network hop needed.
|
|
"""
|
|
if not VPS_EVENTS_HOST or self.node_name == VPS_NODE_NAME:
|
|
return
|
|
|
|
inbox = self._dispatch_inbox_dir()
|
|
inbox.mkdir(parents=True, exist_ok=True)
|
|
local_dir = str(inbox) + "/"
|
|
remote_dir = (f"{VPS_EVENTS_USER}@{VPS_EVENTS_HOST}:"
|
|
f"{VPS_DISPATCH_PATH}/{self.node_name}/")
|
|
cmd = [
|
|
"rsync", "-az", "--remove-source-files",
|
|
"--omit-dir-times", "--no-perms", "--no-owner", "--no-group",
|
|
"-e", ("ssh -F /dev/null"
|
|
" -o StrictHostKeyChecking=no"
|
|
" -o UserKnownHostsFile=/dev/null"
|
|
" -o ConnectTimeout=10"
|
|
" -o BatchMode=yes"),
|
|
remote_dir,
|
|
local_dir,
|
|
]
|
|
try:
|
|
result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
|
|
# rsync returns 23/24 ("partial transfer"/"vanished source files")
|
|
# when the remote dispatch dir is simply empty — the common case,
|
|
# not an error worth logging every cycle.
|
|
if result.returncode not in (0, 23, 24):
|
|
logger.warning(f"Dispatch pull failed: {result.stderr.strip()}")
|
|
except Exception as exc:
|
|
logger.warning(f"Dispatch pull error: {exc}")
|
|
|
|
def process_dispatched_actions(self):
|
|
"""Execute every action currently sitting in this node's dispatch inbox."""
|
|
inbox = self._dispatch_inbox_dir()
|
|
if not inbox.exists():
|
|
return
|
|
for action_file in sorted(inbox.glob("*.json")):
|
|
try:
|
|
action = json.loads(action_file.read_text())
|
|
except Exception as exc:
|
|
logger.error(f"Failed to read dispatched action {action_file.name}: {exc}")
|
|
action_file.unlink(missing_ok=True)
|
|
continue
|
|
self._execute_dispatched_action(action)
|
|
# Delete regardless of outcome: a rejected/failed action is not
|
|
# retried automatically (the whole point of the action_result
|
|
# report is that the executor decides what happens next — retry
|
|
# is a fresh dispatch with a fresh action_id, not this file living on).
|
|
action_file.unlink(missing_ok=True)
|
|
|
|
def _execute_dispatched_action(self, action: dict):
|
|
"""
|
|
Validate and execute one dispatched action, then report the result.
|
|
|
|
Security gates, in order (each rejection short-circuits execution and
|
|
reports a clear action_result error rather than running anything):
|
|
1. Idempotency — already processed this action_id? no-op.
|
|
2. Node scoping — action addressed to a DIFFERENT node? refuse.
|
|
(Defense in depth: the dispatch dir is already scoped by node
|
|
name, but a dispatch file could in principle be mis-delivered.)
|
|
3. Type whitelist — only container_restart, nothing else.
|
|
4. Self-restart guard — never restart node-agent's own container.
|
|
"""
|
|
action_id = action.get("action_id") or "unknown"
|
|
node = action.get("node")
|
|
action_type = action.get("type")
|
|
container_name = action.get("container_name") or action.get("service")
|
|
|
|
if self._already_processed(action_id):
|
|
logger.info(f"Action {action_id} already processed — skipping (idempotency)")
|
|
return
|
|
|
|
if node != self.node_name:
|
|
self._report_action_result(
|
|
action_id, container_name, False,
|
|
f"Action addressed to node '{node}', not '{self.node_name}' — refused",
|
|
)
|
|
self._mark_processed(action_id)
|
|
return
|
|
|
|
if action_type not in ALLOWED_DISPATCH_ACTION_TYPES:
|
|
self._report_action_result(
|
|
action_id, container_name, False,
|
|
f"Action type '{action_type}' is not whitelisted for agent-side "
|
|
f"execution (allowed: {sorted(ALLOWED_DISPATCH_ACTION_TYPES)})",
|
|
)
|
|
self._mark_processed(action_id)
|
|
return
|
|
|
|
if not container_name:
|
|
self._report_action_result(
|
|
action_id, container_name, False, "No container_name in dispatched action",
|
|
)
|
|
self._mark_processed(action_id)
|
|
return
|
|
|
|
if container_name in SELF_RESTART_GUARD_NAMES:
|
|
self._report_action_result(
|
|
action_id, container_name, False,
|
|
f"Refusing to restart '{container_name}': node-agent will not restart itself",
|
|
)
|
|
self._mark_processed(action_id)
|
|
return
|
|
|
|
if not self.docker_client:
|
|
self._report_action_result(
|
|
action_id, container_name, False, "Docker SDK unavailable on this node",
|
|
)
|
|
self._mark_processed(action_id)
|
|
return
|
|
|
|
try:
|
|
container = self.docker_client.containers.get(container_name)
|
|
container.restart()
|
|
logger.info(f"Restarted container '{container_name}' for action {action_id}")
|
|
self._report_action_result(action_id, container_name, True, "")
|
|
except Exception as exc:
|
|
logger.error(f"Failed to restart '{container_name}' for action {action_id}: {exc}")
|
|
self._report_action_result(action_id, container_name, False, str(exc))
|
|
|
|
self._mark_processed(action_id)
|
|
|
|
def _report_action_result(self, action_id: str, container_name, success: bool, error: str):
|
|
"""Emit an action_result event — picked up by the executor's
|
|
_reconcile_running_actions() to move the action to completed/failed.
|
|
Rides the existing event pipeline (and, for remote nodes, the existing
|
|
_ship_events_to_vps rsync) with no changes to either.
|
|
"""
|
|
self.emit_event(
|
|
"action_result",
|
|
"info" if success else "high",
|
|
container_name or action_id,
|
|
f"Action {action_id} {'succeeded' if success else 'failed'}",
|
|
{
|
|
"action_id": action_id,
|
|
"success": success,
|
|
"error": error,
|
|
"node": self.node_name,
|
|
},
|
|
)
|
|
|
|
# ------------------------------------------------------------------
|
|
# VPS-specific: control-plane service health check
|
|
# ------------------------------------------------------------------
|
|
|
|
def _check_control_plane_health(self):
|
|
"""
|
|
VPS-only: probe the control-plane HTTP endpoint and emit a service
|
|
health event so the observer can populate services.json for the
|
|
'control-plane' entry in services.yaml.
|
|
|
|
The control-plane is a multi-container stack (observer, supervisor,
|
|
executor, ui), so individual container names don't match the service
|
|
name in services.yaml. Checking the HTTP endpoint gives a clean
|
|
boundary that maps 1-to-1 with the logical service.
|
|
"""
|
|
import urllib.request
|
|
endpoint = "http://localhost:18180/summary"
|
|
try:
|
|
resp = urllib.request.urlopen(endpoint, timeout=5)
|
|
if resp.status == 200:
|
|
# Transition-only, same rationale as check_containers(): without
|
|
# this gate, control-plane re-confirmed itself every cycle
|
|
# forever, contributing to the same service_healthy flood.
|
|
if self._service_health_state.get("control-plane") is not True:
|
|
self.emit_event(
|
|
"service_healthy", "info", "control-plane",
|
|
"Control-plane HTTP endpoint is reachable",
|
|
{"endpoint": endpoint},
|
|
)
|
|
self._service_health_state["control-plane"] = True
|
|
else:
|
|
self._service_health_state["control-plane"] = False
|
|
self.emit_event(
|
|
"service_unhealthy", "high", "control-plane",
|
|
f"Control-plane HTTP endpoint returned HTTP {resp.status}",
|
|
{"endpoint": endpoint, "http_status": resp.status},
|
|
)
|
|
except Exception as exc:
|
|
self._service_health_state["control-plane"] = False
|
|
self.emit_event(
|
|
"service_unhealthy", "high", "control-plane",
|
|
f"Control-plane HTTP endpoint unreachable: {exc}",
|
|
{"endpoint": endpoint, "error": str(exc)},
|
|
)
|
|
|
|
# ------------------------------------------------------------------
|
|
# Heartbeat
|
|
# ------------------------------------------------------------------
|
|
|
|
def _update_heartbeat(self):
|
|
try:
|
|
(STATE_DIR / "node-agent.heartbeat").touch()
|
|
except Exception as exc:
|
|
logger.error(f"Failed to update heartbeat: {exc}")
|
|
|
|
# ------------------------------------------------------------------
|
|
# Main loop
|
|
# ------------------------------------------------------------------
|
|
|
|
def run_once(self):
|
|
self._update_heartbeat()
|
|
|
|
disk_pct = self.check_disk()
|
|
mem_pct = self.check_memory()
|
|
cpu_pct = self.check_cpu()
|
|
self.check_containers()
|
|
|
|
self.run_safe_cleanup()
|
|
|
|
if self.node_name == VPS_NODE_NAME:
|
|
self._cleanup_control_plane_fs()
|
|
self._check_control_plane_health()
|
|
|
|
# Remediation dispatch: fetch and execute any action the executor has
|
|
# queued for this node, then report the outcome (via emit_event below,
|
|
# shipped in the same cycle by _ship_events_to_vps).
|
|
self.pull_dispatched_actions()
|
|
self.process_dispatched_actions()
|
|
|
|
# Emit a node_health heartbeat so the observer can update node status
|
|
# and the supervisor can correlate disk/memory metrics with service issues.
|
|
self.emit_event(
|
|
"node_health", "info", None,
|
|
f"Health check completed on {self.node_name}",
|
|
{"disk_pct": disk_pct, "mem_pct": mem_pct, "cpu_pct": cpu_pct},
|
|
)
|
|
|
|
self._ship_events_to_vps()
|
|
|
|
def loop(self, interval: int = HEALTH_CHECK_INTERVAL):
|
|
logger.info(
|
|
f"node-agent ready — node={self.node_name} type={self.node_type} "
|
|
f"interval={interval}s"
|
|
)
|
|
while True:
|
|
try:
|
|
self.run_once()
|
|
except Exception as exc:
|
|
logger.error(f"Health cycle error: {exc}")
|
|
time.sleep(interval)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
NodeAgent().loop()
|