fix(supervisor): route healthcheck_failed to container_restart

healthcheck_failed incidents fell through to redeploy, which is broken as
wired (executor calls deploy-node.sh with arguments it ignores, at a path
that does not exist in the container) — so 3376 healthcheck_failed events
dead-ended with no working remediation (recon D14/D15). A container restart
plausibly heals a failing healthcheck and rides the executor path that
actually works; redeploy returns to the map once etap 2 fixes the executor.

service_unhealthy / deployment_failed / missing_service stay on redeploy —
theoretical until etap 2, kept so drift remains visible in pending actions
(noted in comments). CLAUDE.md routing table updated to match; stale
mqtt_unreachable example in the observer's trigger_type comment refreshed.

Tests: trigger-type recognition and the end-to-end observer→supervisor
reconcile test parametrized over both container_restart triggers, with an
assertion that no redeploy action is also generated. Full control-plane
suite: 147 passed.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

CLAUDE.md

@@ -108,7 +108,8 @@ Normalized event types: `deployment_started/completed/failed`, `service_unhealth
 | Event type | Source | Action generated | Cooldown |
 |---|---|---|---|
 | `containers_not_running` | stability-agent | `container_restart` | dedup via stable ID |
-| `service_unhealthy` / other | stability-agent | `redeploy` | dedup via stable ID |
+| `healthcheck_failed` | node-agent | `container_restart` | dedup via stable ID |
+| `service_unhealthy` / other | stability-agent | `redeploy` (broken until etap 2 executor fix) | dedup via stable ID |
 | `disk_pressure` (high) | stability-agent | `disk_cleanup` | dedup via stable ID |
 | `ha_websocket_dead` | ha-diag-agent | `container_restart` (homeassistant) | 30 min after completion |
 | `ha_websocket_recovered` | ha-diag-agent | cancels matching restart | — |

hosts/solaria/runtime/node-agent/docker-compose.override.yml

@@ -1,5 +1,13 @@
 services:
   node-agent:
+    # Docker GID on SOLARIA is 996 (not the Debian default 999 the base compose
+    # assumes). Without this the agent starts with "Docker unavailable:
+    # Permission denied" on /var/run/docker.sock and reports no containers
+    # (recon RECON-multiagent-2026-07-27.md, A2/E19). Compose concatenates
+    # group_add lists, so the base 999 stays alongside; 996 is what grants
+    # socket access here. Same per-host pattern as piha (123) and lustro (991).
+    group_add:
+      - "996"
     environment:
       - NODE_NAME=solaria
       - NODE_TYPE=ai_node

scripts/observer/observer.py

@@ -863,7 +863,7 @@ class Observer:
             "severity": event.get("severity"),
             # trigger_type records the event type that opened this incident so that
             # the supervisor can choose the appropriate remediation action
-            # (e.g. container_restart for containers_not_running / mqtt_unreachable
+            # (container_restart for containers_not_running / healthcheck_failed
             # vs. a full redeploy for other causes).
             "trigger_type": event.get("type"),
             "started_at": event.get("timestamp"),

services/control-plane/src/supervisor.py

@@ -34,12 +34,21 @@ except Exception:
     NODE_ALIAS_MAP = {}
 
 # Incident trigger types that should result in a lightweight container_restart
-# rather than a full redeploy: the container is present but not running.
+# rather than a full redeploy: the container is present but not running, or
+# running with a failing health check — a restart plausibly heals both.
+# healthcheck_failed added 2026-07-29 (recon D14/D15): it used to route to
+# redeploy, which is broken as wired (executor calls deploy-node.sh with
+# arguments it ignores, at a path that does not exist in the container), so
+# 3376 healthcheck_failed events dead-ended. redeploy returns to the map only
+# once it actually works (etap 2). Everything not listed here
+# (service_unhealthy, deployment_failed, missing_service) still falls through
+# to redeploy — theoretical until etap 2 fixes the executor, kept as-is so the
+# drift stays visible in pending actions.
 # mqtt_unreachable was removed 2026-07-28: the observer never creates incidents
 # with that trigger_type, so the branch was dead code (recon
 # docs/architecture/RECON-multiagent-2026-07-27.md, D15). stability-agent still
 # emits the mqtt_unreachable *event*; it just never becomes an incident.
-CONTAINER_RESTART_TRIGGERS = {"containers_not_running"}
+CONTAINER_RESTART_TRIGGERS = {"containers_not_running", "healthcheck_failed"}
 
 # Nodes where automatic disk_cleanup actions must NOT be generated.
 # On chelsty nodes disk fullness is overwhelmingly caused by Frigate recordings
@@ -412,7 +421,8 @@ class Supervisor:
 
         if trigger_type in CONTAINER_RESTART_TRIGGERS:
             # Lightweight remediation: the container exists but is not running
-            # (containers_not_running). A docker restart is sufficient and low-risk.
+            # (containers_not_running) or is up with a failing health check
+            # (healthcheck_failed). A docker restart is sufficient and low-risk.
             container_name = self._get_container_name(service)
             action = {
                 "action_id": action_id,
@@ -436,6 +446,10 @@ class Supervisor:
         else:
             # Full redeploy: container is running but service is broken,
             # or the cause is unknown / not a simple restart candidate.
+            # NOTE: the redeploy action type is currently theoretical — the
+            # executor's redeploy path is broken until etap 2 (see
+            # CONTAINER_RESTART_TRIGGERS comment). Generated anyway so the
+            # drift is visible to the operator in pending actions.
             action = {
                 "action_id": action_id,
                 "timestamp": time.time(),

services/control-plane/tests/test_observer_container_events.py

@@ -106,12 +106,15 @@ def test_containers_not_running_creates_incident_with_trigger_type(observer):
     assert inc["trigger_type"] == "containers_not_running"
 
 
-def test_incident_trigger_type_is_recognized_by_supervisor(observer):
+@pytest.mark.parametrize("etype", ["containers_not_running", "healthcheck_failed"])
+def test_incident_trigger_type_is_recognized_by_supervisor(observer, etype):
     """The trigger_type the observer stamps MUST be one the supervisor routes to
-    a container_restart — otherwise remediation silently never fires."""
-    observer.process_event(_container_event("containers_not_running"))
+    a container_restart — otherwise remediation silently never fires (or, for
+    healthcheck_failed pre-etap-1, dead-ended in the broken redeploy path)."""
+    observer.process_event(_container_event(etype))
     svc = observer.world_state["services"]["piha/paperless"]
     inc = observer.world_state["incidents"][svc["incident_id"]]
+    assert inc["trigger_type"] == etype
     assert inc["trigger_type"] in CONTAINER_RESTART_TRIGGERS
 
 
@@ -131,15 +134,21 @@ def test_containers_not_running_after_healthy_transitions_to_unhealthy(observer)
 # 2. End-to-end: observer output → supervisor generates container_restart
 # ---------------------------------------------------------------------------
 
+@pytest.mark.parametrize("etype", ["containers_not_running", "healthcheck_failed"])
 def test_supervisor_generates_container_restart_from_observer_output(
-    observer, tmp_path, monkeypatch
+    observer, tmp_path, monkeypatch, etype
 ):
-    """Integration: run the real observer to produce world state for a dead
+    """Integration: run the real observer to produce world state for a failing
     container, then run the real supervisor.reconcile() against it and assert a
-    container_restart action lands in pending/. Proves the whole loop works with
-    NO change to the supervisor."""
-    # 1. Observer ingests the dead-container event and writes world state to disk.
-    observer.process_event(_container_event("containers_not_running"))
+    container_restart action lands in pending/.
+
+    healthcheck_failed routes to container_restart since etap 1 (2026-07-29,
+    recon D14/D15): it used to fall through to redeploy, which is broken as
+    wired in the executor, so those events dead-ended. A restart plausibly
+    heals a failing healthcheck; redeploy returns once the executor works
+    (etap 2)."""
+    # 1. Observer ingests the failure event and writes world state to disk.
+    observer.process_event(_container_event(etype))
     observer._save_world()
 
     # 2. Point the supervisor at a fresh tmp tree; copy observer world output in.
@@ -176,7 +185,9 @@ def test_supervisor_generates_container_restart_from_observer_output(
     assert action["type"] == "container_restart"
     assert action["node"] == "piha"
     assert action["service"] == "paperless"
-    assert action["payload"]["reason"] == "containers_not_running"
+    assert action["payload"]["reason"] == etype
+    # Must not ALSO fall through to the (broken until etap 2) redeploy path.
+    assert not (actions / "pending" / "redeploy-piha-paperless.json").exists()
 
 
 # ---------------------------------------------------------------------------

services/stability-agent/src/stability_agent.py

@@ -119,6 +119,29 @@ class DockerClient:
     def get_containers(self):
         return self._request("/containers/json?all=1")
 
+def container_service_name(container):
+    """Return the compose service name for a /containers/json entry.
+
+    Priority (same pattern as node-agent's _canonical_container_name):
+    1. com.docker.compose.service label — the clean compose-file key, immune
+       to the "<12-hex>_" prefix Docker uses for stale project-state entries.
+    2. Container name with that stale-state prefix stripped — fallback for
+       non-Compose containers.
+    Never raises: missing/None Labels or Names degrade to "unknown".
+    """
+    labels = container.get("Labels") or {}
+    if isinstance(labels, dict):
+        compose_svc = (labels.get("com.docker.compose.service") or "").strip()
+        if compose_svc:
+            return compose_svc
+    names = container.get("Names") or []
+    name = names[0].lstrip("/") if names else ""
+    if (len(name) > 13
+            and name[12] == "_"
+            and all(ch in "0123456789abcdef" for ch in name[:12])):
+        name = name[13:]
+    return name or "unknown"
+
 def check_docker():
     client = DockerClient()
     if not os.path.exists(client.socket_path):
@@ -138,19 +161,28 @@ def check_docker():
 
         container_info = {
             "name": name,
+            "service": container_service_name(c),
             "state": state,
             "status": status
         }
         summary.append(container_info)
 
-        if state != "running":
+        # "created" containers are Docker Compose internal tracking artifacts
+        # (never started) — not a running service, nothing to remediate.
+        if state != "running" and state != "created":
             unhealthy_containers.append(container_info)
 
-    if unhealthy_containers:
-        names = [c["name"] for c in unhealthy_containers]
-        # Only emit warning for containers that should be running?
-        # For now, we report any non-running container found by Docker.
-        emit_event("containers_not_running", "warning", f"Some containers are not running: {', '.join(names)}", details={"containers": unhealthy_containers})
+    # One event per container, tagged with its compose service name. The
+    # observer keys service state and incidents on event["service"] and skips
+    # events without it — the old aggregate event (service=None, all names in
+    # one message) never opened an incident (recon D15).
+    for info in unhealthy_containers:
+        emit_event(
+            "containers_not_running", "warning",
+            f"Container '{info['name']}' is not running (state={info['state']})",
+            service=info["service"],
+            details={"container": info},
+        )
 
     return {"status": "ok", "containers": summary}
 

services/stability-agent/tests/test_container_service_name.py

@@ -0,0 +1,47 @@
+"""Tests for container_service_name — the compose-label extraction that fixes
+the service=None containers_not_running events (recon D15: observer skips
+events without a service, so stability-agent's flagship signal never opened
+an incident). Input shape is a /containers/json (Docker list API) entry.
+"""
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
+
+from stability_agent import container_service_name
+
+
+def test_compose_label_wins_over_container_name():
+    c = {
+        "Names": ["/kb-postgres-db-1"],
+        "Labels": {"com.docker.compose.service": "kb-postgres"},
+    }
+    assert container_service_name(c) == "kb-postgres"
+
+
+def test_unlabeled_falls_back_to_container_name():
+    assert container_service_name({"Names": ["/paperless"], "Labels": {}}) == "paperless"
+
+
+def test_missing_labels_key_does_not_crash():
+    assert container_service_name({"Names": ["/mosquitto"]}) == "mosquitto"
+
+
+def test_null_labels_does_not_crash():
+    assert container_service_name({"Names": ["/mosquitto"], "Labels": None}) == "mosquitto"
+
+
+def test_blank_label_falls_back_to_name():
+    c = {"Names": ["/zigbee2mqtt"], "Labels": {"com.docker.compose.service": "   "}}
+    assert container_service_name(c) == "zigbee2mqtt"
+
+
+def test_stale_state_hash_prefix_stripped():
+    # Docker stores stale project-state records as "<12-hex>_<original-name>";
+    # same ghost-key corruption node-agent's _canonical_container_name handles.
+    c = {"Names": ["/9e36297651e7_control-plane-observer"], "Labels": {}}
+    assert container_service_name(c) == "control-plane-observer"
+
+
+def test_empty_container_dict_degrades_to_unknown():
+    assert container_service_name({}) == "unknown"