Compare commits
1 commit
75d96956a5
...
a62c36c0b6
| Author | SHA1 | Date | |
|---|---|---|---|
|
a62c36c0b6 |
|
|
@ -1,3 +1,16 @@
|
|||
# MITYGACJA TYMCZASOWA (M1) — założona 2026-08-04.
|
||||
# NODE_TYPE=lte_node wyłącza run_safe_cleanup() (niefiltrowany
|
||||
# `docker container prune`) na czas backfillu embed (faza mailowa KB).
|
||||
# Incydent: kb/incidents/2026-07-30-ollama-solaria-vanish.md (§7, M1).
|
||||
# Bez tego każdy zatrzymany kontener na SOLARII znika w ≤60 s — również taki
|
||||
# z `restart: unless-stopped`, zatrzymany świadomie przez operatora.
|
||||
# Warunek zdjęcia: R1 (filtrowanie prune po restart policy / labelu compose)
|
||||
# wdrożony na tym nodzie — R1–R3 są w toku po stronie subsystemu A.
|
||||
# Po zdjęciu przywrócić: NODE_TYPE=ai_node.
|
||||
# Zakres wyłączenia: `lte_node` pomija CAŁY cleanup, więc na czas mitygacji
|
||||
# nie są też sprzątane dangling images ani build cache — pilnować miejsca
|
||||
# na dysku. Monitoring, eventy i dispatch akcji działają bez zmian
|
||||
# (self.node_type jest czytane wyłącznie w run_safe_cleanup i dwóch liniach logu).
|
||||
services:
|
||||
node-agent:
|
||||
# Docker GID on SOLARIA is 996 (not the Debian default 999 the base compose
|
||||
|
|
@ -10,11 +23,7 @@ services:
|
|||
- "996" # host docker gid, verified 2026-07-30 (getent group docker → 996)
|
||||
environment:
|
||||
- NODE_NAME=solaria
|
||||
# ai_node = dangling images + kontenery + build cache, ale NIGDY
|
||||
# `image prune -a` (skasowałoby obrazy runtime Ollamy). Ustawione jawnie,
|
||||
# zgodnie z konwencją pozostałych hostów, mimo że solaria jest w AI_NODES
|
||||
# w node_agent.py i default dałby to samo.
|
||||
- NODE_TYPE=ai_node
|
||||
- NODE_TYPE=lte_node # M1 (2026-08-04) — było: ai_node; przywrócić po R1
|
||||
- VPS_EVENTS_HOST=100.95.58.48
|
||||
- VPS_EVENTS_USER=oskar
|
||||
- VPS_EVENTS_PATH=/opt/homelab/events
|
||||
|
|
|
|||
|
|
@ -3,11 +3,18 @@ services:
|
|||
environment:
|
||||
- NODE_NAME=vps
|
||||
- CHECK_INTERVAL=60
|
||||
# No NODE_TYPE here on purpose: `vps` is in none of node_agent.py's
|
||||
# LTE_NODES / SD_CARD_NODES / AI_NODES sets, so _resolve_node_type() falls
|
||||
# through to "standard" — dangling images + stopped containers + build
|
||||
# cache, plus the control-plane filesystem rotation (that one is gated on
|
||||
# node_name == VPS_NODE_NAME, not on node_type). This is the pre-M1 state.
|
||||
# TEMPORARY mitigation (M1) for the unfiltered-prune incident
|
||||
# (kb/incidents/2026-07-30-ollama-solaria-vanish.md §7). node-agent runs
|
||||
# `docker container prune()` with NO filters every CHECK_INTERVAL, and the
|
||||
# Docker API removes EVERY non-running container regardless of restart
|
||||
# policy or compose labels — this already destroyed ollama@solaria. On VPS
|
||||
# the loss is worse: humanai-mailer and humanai-landing have no compose
|
||||
# definition in this repo, so a pruned container cannot be recreated.
|
||||
# node_type is read ONLY by run_safe_cleanup() (plus two log lines), so
|
||||
# lte_node disables cleanup and nothing else — monitoring, event shipping
|
||||
# and action dispatch keep working.
|
||||
# REMOVE once R1 (explicit-enumeration prune) is deployed to VPS.
|
||||
- NODE_TYPE=lte_node
|
||||
# host network mode: node-agent on VPS shares the host's network namespace
|
||||
# so that localhost:18180 resolves to the control-plane's exposed port.
|
||||
# Without this, localhost inside the container is the container's own loopback
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ def _atomic_write_json(path: Path, data) -> None:
|
|||
os.fsync(f.fileno())
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
# Constants and Paths
|
||||
RUNTIME_PATH = os.getenv("RUNTIME_PATH", "/opt/homelab")
|
||||
ACTIONS_DIR = Path(RUNTIME_PATH) / "actions"
|
||||
|
|
@ -28,17 +27,6 @@ DISPATCH_DIR = ACTIONS_DIR / "dispatch"
|
|||
# failure-reports every file it finds in its inbox, so a redeploy landing there
|
||||
# would be killed before the runner ever saw it.
|
||||
DEPLOY_DISPATCH_DIR = ACTIONS_DIR / "deploy"
|
||||
|
||||
# Mode for the per-node inboxes under DISPATCH_DIR / DEPLOY_DISPATCH_DIR.
|
||||
# These dirs are written by the executor (as the control-plane user on VPS) but
|
||||
# drained by the target node, whose rsync-pull authenticates as a *different*
|
||||
# user that is only a member of the owning group. --remove-source-files must
|
||||
# unlink the fetched file, and unlink needs write permission on the containing
|
||||
# directory — at 0o755 the group has none, so the source survives, rsync exits
|
||||
# 23, and the node re-pulls the same action every cycle forever, bouncing off
|
||||
# the idempotency gate. Observed on dispatch/lustro 2026-08-06 (session log,
|
||||
# follow-up #1); the older dispatch/piha happened to be 775 and worked.
|
||||
INBOX_DIR_MODE = 0o775
|
||||
# The executor no longer reads the repo at all (the old redeploy path ran a
|
||||
# script out of it). Kept only so an operator can still see which checkout the
|
||||
# container is wired to; nothing in this module resolves paths against it.
|
||||
|
|
@ -80,27 +68,6 @@ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(
|
|||
logger = logging.getLogger("executor")
|
||||
|
||||
|
||||
def _ensure_inbox_dir(path: Path) -> None:
|
||||
"""Create (or repair) a per-node inbox so the target node can drain it.
|
||||
|
||||
chmod runs unconditionally rather than only on creation, for two reasons:
|
||||
mkdir(mode=...) is masked by the process umask and so cannot be relied on to
|
||||
produce INBOX_DIR_MODE, and inboxes created by an earlier executor build
|
||||
already exist at 0o755 across the fleet. Repairing here — on the same code
|
||||
path that writes the dispatch file — keeps the fix to one idempotent call
|
||||
and needs no startup scan that could drift out of sync with the writers.
|
||||
"""
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
os.chmod(path, INBOX_DIR_MODE)
|
||||
except OSError as e:
|
||||
# Deliberately not fatal: the dispatch file still gets written and the
|
||||
# node still executes the action. Only the post-fetch source delete
|
||||
# stays broken — and that now surfaces as a WARNING on the node side
|
||||
# (node_agent.pull_dispatched_actions) instead of being swallowed.
|
||||
logger.warning(f"Could not set mode {oct(INBOX_DIR_MODE)} on {path}: {e}")
|
||||
|
||||
|
||||
class Executor:
|
||||
def __init__(self):
|
||||
self._ensure_dirs()
|
||||
|
|
@ -250,7 +217,7 @@ class Executor:
|
|||
logger.error(f"Action {action_id}: container_restart with no node set")
|
||||
return
|
||||
inbox = DISPATCH_DIR / node
|
||||
_ensure_inbox_dir(inbox)
|
||||
inbox.mkdir(parents=True, exist_ok=True)
|
||||
payload = {
|
||||
"action_id": action_id,
|
||||
"type": "container_restart",
|
||||
|
|
@ -282,7 +249,7 @@ class Executor:
|
|||
Does not resolve the action itself; _reconcile_running_actions() does.
|
||||
"""
|
||||
inbox = DEPLOY_DISPATCH_DIR / node
|
||||
_ensure_inbox_dir(inbox)
|
||||
inbox.mkdir(parents=True, exist_ok=True)
|
||||
payload = {
|
||||
"action_id": action_id,
|
||||
"type": "redeploy",
|
||||
|
|
|
|||
|
|
@ -9,11 +9,9 @@ forever.
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -241,78 +239,3 @@ def test_alert_only_action_resolves_synchronously_not_via_dispatch(tmp_path, mon
|
|||
assert _exists(tmp_path, "completed", "alert-1")
|
||||
assert not (tmp_path / "actions" / "dispatch").exists() or \
|
||||
not list((tmp_path / "actions" / "dispatch").glob("**/*.json"))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Inbox permissions (dispatch leak, 2026-08-06)
|
||||
#
|
||||
# The per-node inbox is written here but drained by the target node over rsync
|
||||
# --remove-source-files, authenticating as a different user that is only a
|
||||
# group member. Unlinking the fetched file needs write permission on the
|
||||
# containing dir; at 0o755 it silently fails and the node re-pulls the same
|
||||
# action forever.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest.fixture
|
||||
def restrictive_umask():
|
||||
"""0o022 masks the group-write bit out of mkdir(mode=0o775) — the reason
|
||||
the fix cannot rely on the mode argument alone."""
|
||||
old = os.umask(0o022)
|
||||
yield
|
||||
os.umask(old)
|
||||
|
||||
|
||||
def _mode(path: Path) -> int:
|
||||
return path.stat().st_mode & 0o777
|
||||
|
||||
|
||||
def test_dispatch_inbox_is_group_writable(tmp_path, monkeypatch, restrictive_umask):
|
||||
ex = _setup_executor(tmp_path, monkeypatch)
|
||||
|
||||
ex._execute_action(_write_approved(tmp_path, "cr-perm", node="piha"))
|
||||
|
||||
inbox = tmp_path / "actions" / "dispatch" / "piha"
|
||||
assert (inbox / "cr-perm.json").exists()
|
||||
assert _mode(inbox) == 0o775
|
||||
|
||||
|
||||
def test_deploy_inbox_is_group_writable(tmp_path, monkeypatch, restrictive_umask):
|
||||
"""Same defect, same rsync-pull drain — the deploy runner's inbox needs it too."""
|
||||
ex = _setup_executor(tmp_path, monkeypatch)
|
||||
|
||||
ex._execute_action(
|
||||
_write_approved(tmp_path, "rd-perm", node="piha", action_type="redeploy")
|
||||
)
|
||||
|
||||
inbox = tmp_path / "actions" / "deploy" / "piha"
|
||||
assert (inbox / "rd-perm.json").exists()
|
||||
assert _mode(inbox) == 0o775
|
||||
|
||||
|
||||
def test_existing_inbox_is_repaired_in_place(tmp_path, monkeypatch, restrictive_umask):
|
||||
"""Inboxes already on disk fleet-wide were created at 0o755 by an earlier
|
||||
build; dispatching to one must fix it rather than inherit it."""
|
||||
ex = _setup_executor(tmp_path, monkeypatch)
|
||||
inbox = tmp_path / "actions" / "dispatch" / "piha"
|
||||
inbox.mkdir(parents=True)
|
||||
os.chmod(inbox, 0o755)
|
||||
|
||||
ex._execute_action(_write_approved(tmp_path, "cr-repair", node="piha"))
|
||||
|
||||
assert _mode(inbox) == 0o775
|
||||
|
||||
|
||||
def test_dispatch_survives_unsettable_mode(tmp_path, monkeypatch, restrictive_umask):
|
||||
"""A chmod failure (inbox owned by another user) must not cost us the
|
||||
dispatch — the action still executes on the node; only the source delete
|
||||
stays broken, and the node warns about that."""
|
||||
ex = _setup_executor(tmp_path, monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
executor_module.os, "chmod",
|
||||
MagicMock(side_effect=PermissionError("Operation not permitted")),
|
||||
)
|
||||
|
||||
ex._execute_action(_write_approved(tmp_path, "cr-chmod-fail", node="piha"))
|
||||
|
||||
assert (tmp_path / "actions" / "dispatch" / "piha" / "cr-chmod-fail.json").exists()
|
||||
assert _exists(tmp_path, "running", "cr-chmod-fail")
|
||||
|
|
|
|||
|
|
@ -169,23 +169,6 @@ def _utc_iso() -> str:
|
|||
_EVENT_TS_RE = re.compile(r"-(\d{9,11})-")
|
||||
_EVENT_TYPE_RE = re.compile(r"^evt-.+?-\d{9,11}-(.+)$")
|
||||
|
||||
# rsync exits 23 for two very different situations and the dispatch pull hits
|
||||
# both routinely, so the stderr has to be read to tell them apart:
|
||||
#
|
||||
# * The remote inbox does not exist yet. The executor creates
|
||||
# actions/dispatch/<node>/ only on its first dispatch to that node, so until
|
||||
# then every pull reports `change_dir "..." failed: No such file or
|
||||
# directory`. Expected, and warning about it would mean one line a minute on
|
||||
# every node that has never been sent an action.
|
||||
# * The files WERE fetched but rsync could not unlink the source
|
||||
# (`sender failed to remove <file>: Permission denied`) — the dispatch leak
|
||||
# of 2026-08-06. That one has to be loud: it means the same actions come
|
||||
# back on every single cycle until someone fixes the directory mode on VPS.
|
||||
#
|
||||
# (An empty-but-existing remote inbox — by far the most common case — is a plain
|
||||
# rc=0 and never reaches here.)
|
||||
_RSYNC_MISSING_SRC_RE = re.compile(r"change_dir .* failed: No such file or directory")
|
||||
|
||||
|
||||
def _event_ts_from_filename(name: str):
|
||||
"""Return the embedded <unixts> from an event filename, or None if absent."""
|
||||
|
|
@ -968,45 +951,14 @@ class NodeAgent:
|
|||
]
|
||||
try:
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
|
||||
self._log_dispatch_pull_result(result.returncode, result.stderr or "")
|
||||
# rsync returns 23/24 ("partial transfer"/"vanished source files")
|
||||
# when the remote dispatch dir is simply empty — the common case,
|
||||
# not an error worth logging every cycle.
|
||||
if result.returncode not in (0, 23, 24):
|
||||
logger.warning(f"Dispatch pull failed: {result.stderr.strip()}")
|
||||
except Exception as exc:
|
||||
logger.warning(f"Dispatch pull error: {exc}")
|
||||
|
||||
@staticmethod
|
||||
def _log_dispatch_pull_result(returncode: int, stderr: str) -> None:
|
||||
"""Classify an rsync exit code from the dispatch pull.
|
||||
|
||||
Visibility only — the caller retries on the next cycle either way, and
|
||||
an action that was fetched is executed regardless of what the source
|
||||
side did. Codes:
|
||||
|
||||
0 — clean, including "remote inbox exists and is empty".
|
||||
24 — a source file vanished between the file list and the transfer.
|
||||
A benign race with the executor writing the inbox concurrently.
|
||||
23 — "some files were not transferred", which covers two situations
|
||||
that must NOT be logged alike (see _RSYNC_MISSING_SRC_RE).
|
||||
* — real transport failure (ssh down, auth, timeout).
|
||||
|
||||
Before 2026-08-06 every one of these was benign-listed, which is how the
|
||||
dispatch leak stayed invisible: the node re-pulled the same actions
|
||||
every 60 s for days, silently bouncing them off the idempotency gate.
|
||||
"""
|
||||
stderr = stderr.strip()
|
||||
if returncode in (0, 24):
|
||||
return
|
||||
if returncode == 23:
|
||||
if _RSYNC_MISSING_SRC_RE.search(stderr) and "failed to remove" not in stderr:
|
||||
logger.debug(f"Dispatch inbox not present on VPS yet: {stderr}")
|
||||
return
|
||||
logger.warning(
|
||||
"Dispatch pull incomplete (rsync rc=23): action files were "
|
||||
"fetched but their source copy on VPS was NOT removed, so they "
|
||||
"will be re-pulled every cycle. Check the mode of "
|
||||
f"actions/dispatch/<node>/ on VPS (needs group write). {stderr}"
|
||||
)
|
||||
return
|
||||
logger.error(f"Dispatch pull failed (rsync rc={returncode}): {stderr}")
|
||||
|
||||
def process_dispatched_actions(self):
|
||||
"""Execute every action currently sitting in this node's dispatch inbox."""
|
||||
inbox = self._dispatch_inbox_dir()
|
||||
|
|
|
|||
|
|
@ -238,77 +238,13 @@ def test_pull_invokes_rsync_pull_direction(agent, monkeypatch):
|
|||
assert cmd[-1] == str(agent._dispatch_inbox_dir()) + "/"
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# rsync exit-code classification (dispatch leak, 2026-08-06)
|
||||
#
|
||||
# rc=23 used to be benign-listed together with 0 and 24, which is why the
|
||||
# leak — files fetched but never removed from VPS, so re-pulled every 60 s —
|
||||
# produced no log line at all for days. These pin the four outcomes.
|
||||
# ----------------------------------------------------------------------
|
||||
def test_pull_treats_empty_source_returncodes_as_non_error(agent, monkeypatch, caplog):
|
||||
def fake_run(cmd, **kwargs):
|
||||
return MagicMock(returncode=23, stderr="rsync: some vanished-source message")
|
||||
|
||||
# Verbatim rsync 3.4.1 stderr for the two distinct rc=23 causes.
|
||||
_STDERR_UNDELETABLE_SOURCE = (
|
||||
"rsync: [sender] sender failed to remove act-123.json: Permission denied (13)\n"
|
||||
"rsync error: some files/attrs were not transferred "
|
||||
"(see previous errors) (code 23) at main.c(1356) [sender=3.4.1]"
|
||||
)
|
||||
_STDERR_MISSING_INBOX = (
|
||||
'rsync: [sender] change_dir "/opt/homelab/actions/dispatch/test-node" '
|
||||
"failed: No such file or directory (2)\n"
|
||||
"rsync error: some files/attrs were not transferred "
|
||||
"(see previous errors) (code 23) at main.c(1356) [sender=3.4.1]"
|
||||
)
|
||||
monkeypatch.setattr(node_agent.subprocess, "run", fake_run)
|
||||
|
||||
|
||||
def _pull_with(agent, monkeypatch, returncode, stderr=""):
|
||||
monkeypatch.setattr(
|
||||
node_agent.subprocess, "run",
|
||||
lambda cmd, **kwargs: MagicMock(returncode=returncode, stderr=stderr),
|
||||
)
|
||||
with caplog.at_level("WARNING"):
|
||||
agent.pull_dispatched_actions()
|
||||
|
||||
|
||||
def test_pull_rc0_logs_nothing(agent, monkeypatch, caplog):
|
||||
with caplog.at_level("DEBUG"):
|
||||
_pull_with(agent, monkeypatch, 0)
|
||||
|
||||
assert "Dispatch pull" not in caplog.text
|
||||
|
||||
|
||||
def test_pull_rc24_vanished_source_stays_benign(agent, monkeypatch, caplog):
|
||||
"""A file removed between file-list and transfer is a race with the
|
||||
executor writing the inbox, not a fault."""
|
||||
with caplog.at_level("WARNING"):
|
||||
_pull_with(agent, monkeypatch, 24, "rsync warning: some files vanished")
|
||||
|
||||
assert caplog.text == ""
|
||||
|
||||
|
||||
def test_pull_rc23_undeletable_source_warns_with_stderr(agent, monkeypatch, caplog):
|
||||
"""The leak itself: loud, and carrying the rsync stderr that names it."""
|
||||
with caplog.at_level("WARNING"):
|
||||
_pull_with(agent, monkeypatch, 23, _STDERR_UNDELETABLE_SOURCE)
|
||||
|
||||
assert "WARNING" in caplog.text
|
||||
assert "rc=23" in caplog.text
|
||||
# Full stderr forwarded, so the operator sees which file and why.
|
||||
assert "sender failed to remove act-123.json: Permission denied" in caplog.text
|
||||
|
||||
|
||||
def test_pull_rc23_missing_remote_inbox_is_quiet(agent, monkeypatch, caplog):
|
||||
"""A node that has never been dispatched to gets rc=23 on every cycle
|
||||
because the executor has not created its inbox yet. Warning here would be
|
||||
a line a minute on most of the fleet — and would bury the case above."""
|
||||
with caplog.at_level("WARNING"):
|
||||
_pull_with(agent, monkeypatch, 23, _STDERR_MISSING_INBOX)
|
||||
|
||||
assert caplog.text == ""
|
||||
|
||||
|
||||
def test_pull_other_returncode_logs_error(agent, monkeypatch, caplog):
|
||||
with caplog.at_level("WARNING"):
|
||||
_pull_with(agent, monkeypatch, 255, "ssh: connect to host vps port 22: No route to host")
|
||||
|
||||
assert "ERROR" in caplog.text
|
||||
assert "rc=255" in caplog.text
|
||||
assert "No route to host" in caplog.text
|
||||
assert "Dispatch pull failed" not in caplog.text
|
||||
|
|
|
|||
Loading…
Reference in a new issue