homelab-codex-ws/jobs/deploy-runner/deploy-runner.sh

#!/usr/bin/env bash
# jobs/deploy-runner/deploy-runner.sh — host-side executor for `redeploy` actions.
#
# WHY THIS EXISTS (docs/architecture/RECON-multiagent-2026-07-27.md D14/D15):
# the control-plane executor could never carry out a redeploy — it ran
# deploy-node.sh inside its own container, a script that ignores its arguments
# and expects a repo at ${HOME}/homelab-codex-ws that does not exist there (nor
# does git or the docker CLI). Every healthcheck_failed → redeploy action
# dead-ended, which is the single biggest reason the self-healing loop had
# 18 pending / 0 completed actions.
#
# The fix keeps the architecture decision from docs/backlog.md ("Remediacja
# floty bez SSH"): the VPS never initiates a connection to a node. The executor
# only WRITES an action file; this runner, on the target node, pulls it, runs
# the deploy locally, and reports the outcome back through the existing event
# pipeline.
#
#   executor (vps)  →  /opt/homelab/actions/deploy/<node>/<action_id>.json
#     → this runner rsync-pulls it (--remove-source-files: collected once)
#     → validates it (jobs/deploy-runner/action.py)
#     → scripts/deploy/deploy-service.sh --force-recreate  (same compose
#       invocation as a human deploy — identical project name)
#     → writes an action_result event and rsync-pushes it back to the VPS
#     → executor._reconcile_running_actions() → completed / failed
#
# It runs on the HOST, not in a container: compose then resolves relative bind
# mounts and the project name exactly as it does for a human `deploy.sh`, and
# no agent container needs a docker CLI. It is deliberately independent of
# node-agent (own rsync, own result push) so it can redeploy node-agent itself —
# the solaria case, where node-agent has been docker-blind for weeks.
#
# It NEVER runs `git pull`: a redeploy reconciles the node to the checkout it
# already has. Shipping new code stays a human `scripts/deploy/deploy.sh` action.
#
# Config comes from the environment (systemd EnvironmentFile) — see env.example.
# Install: see README.md.

# NOT set -e: one malformed or failing action must be reported and skipped, not
# abort the whole run.
set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
HELPER="${SCRIPT_DIR}/action.py"

NODE_NAME="${NODE_NAME:-}"
REPO_PATH="${REPO_PATH:-${HOME}/homelab-codex-ws}"
RUNTIME_PATH="${RUNTIME_PATH:-/opt/homelab}"
VPS_EVENTS_HOST="${VPS_EVENTS_HOST:-}"
VPS_EVENTS_USER="${VPS_EVENTS_USER:-oskar}"
VPS_EVENTS_PATH="${VPS_EVENTS_PATH:-/opt/homelab/events}"
VPS_DEPLOY_PATH="${VPS_DEPLOY_PATH:-/opt/homelab/actions/deploy}"
DEPLOY_TIMEOUT_SECS="${DEPLOY_TIMEOUT_SECS:-600}"

INBOX="${RUNTIME_PATH}/actions/deploy/${NODE_NAME}"
EVENTS_DIR="${RUNTIME_PATH}/events/${NODE_NAME}"
STATE_DIR="${RUNTIME_PATH}/state"
MARKER_DIR="${STATE_DIR}/processed-deploy-actions"
LOG_DIR="${RUNTIME_PATH}/logs/deploy-runner"
HOST_DIR="${REPO_PATH}/hosts/${NODE_NAME}"

# Same ssh settings node-agent uses for its rsync channel: no ~/.ssh/config, no
# host-key prompts, batch mode so a missing key fails fast instead of hanging.
SSH_CMD="ssh -F /dev/null -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 -o BatchMode=yes"
if [[ -n "${SSH_KEY:-}" ]]; then
    SSH_CMD="${SSH_CMD} -i ${SSH_KEY}"
fi

log() { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) [deploy-runner] $*"; }
die() { log "ERROR: $*"; exit 1; }

[[ -n "$NODE_NAME" ]] || die "NODE_NAME is not set (see env.example)"
[[ -d "$REPO_PATH" ]] || die "repo not found at ${REPO_PATH} (set REPO_PATH)"
[[ -d "$HOST_DIR" ]]  || die "no hosts/${NODE_NAME}/ directory in ${REPO_PATH}"
[[ -x "${REPO_PATH}/scripts/deploy/deploy-service.sh" ]] \
    || die "scripts/deploy/deploy-service.sh missing or not executable in ${REPO_PATH}"
command -v python3 >/dev/null || die "python3 not found"

mkdir -p "$INBOX" "$EVENTS_DIR" "$MARKER_DIR" "$LOG_DIR" || die "cannot create runtime dirs under ${RUNTIME_PATH}"

# Single instance: the timer fires every 60s but a deploy may take minutes.
exec 9>"${STATE_DIR}/deploy-runner.lock" || die "cannot open lock file"
if ! flock -n 9; then
    log "another deploy-runner run is in progress — exiting"
    exit 0
fi

# ── transport ────────────────────────────────────────────────────────────────

# Remote nodes fetch their inbox from the VPS. On the VPS itself the executor
# writes straight into the shared /opt/homelab mount, so there is nothing to
# pull and nothing to push — the executor reads the result event in place.
is_remote() { [[ -n "$VPS_EVENTS_HOST" ]]; }

pull_actions() {
    is_remote || return 0
    local rc=0
    rsync -az --remove-source-files \
        --omit-dir-times --no-perms --no-owner --no-group \
        -e "$SSH_CMD" \
        "${VPS_EVENTS_USER}@${VPS_EVENTS_HOST}:${VPS_DEPLOY_PATH}/${NODE_NAME}/" \
        "${INBOX}/" || rc=$?
    # 23/24 = "partial transfer"/"vanished source files": what an empty or
    # concurrently-drained remote inbox looks like. Not worth logging each run.
    if [[ $rc -ne 0 && $rc -ne 23 && $rc -ne 24 ]]; then
        log "WARN: dispatch pull failed (rsync rc=${rc})"
    fi
}

push_event() {
    local event_file="$1"
    is_remote || return 0
    local rc=0
    rsync -az --remove-source-files \
        --omit-dir-times --no-perms --no-owner --no-group \
        -e "$SSH_CMD" \
        "$event_file" \
        "${VPS_EVENTS_USER}@${VPS_EVENTS_HOST}:${VPS_EVENTS_PATH}/${NODE_NAME}/" || rc=$?
    if [[ $rc -ne 0 && $rc -ne 23 && $rc -ne 24 ]]; then
        # Left in place on failure: node-agent's own event shipping will pick it
        # up on its next cycle, so a result is delayed, never lost.
        log "WARN: result push failed (rsync rc=${rc}) — leaving ${event_file} for node-agent to ship"
    fi
}

report() {
    local action_id="$1" service="$2" success="$3" error="$4"
    local event_file
    event_file=$(python3 "$HELPER" emit-result \
        --events-dir "$EVENTS_DIR" \
        --node "$NODE_NAME" \
        --action-id "$action_id" \
        --service "$service" \
        --success "$success" \
        --error "$error") || { log "ERROR: could not write result event for ${action_id}"; return 1; }
    log "reported ${action_id}: success=${success}${error:+ error=${error}}"
    push_event "$event_file"
}

# ── main ─────────────────────────────────────────────────────────────────────

pull_actions

shopt -s nullglob
for action_file in "${INBOX}"/*.json; do
    OK="false"; ACTION_ID=""; SERVICE=""; ERROR=""
    eval "$(python3 "$HELPER" validate --file "$action_file" --node "$NODE_NAME" --repo "$REPO_PATH")"

    if [[ "$OK" != "true" ]]; then
        log "rejected $(basename "$action_file"): ${ERROR}"
        if [[ -n "$ACTION_ID" ]]; then
            report "$ACTION_ID" "$SERVICE" "false" "$ERROR"
        fi
        rm -f "$action_file"
        continue
    fi

    marker="${MARKER_DIR}/${ACTION_ID}.done"
    if [[ -e "$marker" ]]; then
        log "action ${ACTION_ID} already processed — skipping (idempotency)"
        rm -f "$action_file"
        continue
    fi

    log "deploying ${SERVICE} for action ${ACTION_ID}"
    action_log="${LOG_DIR}/${ACTION_ID}-$(date -u +%Y%m%dT%H%M%SZ).log"
    rc=0
    timeout "$DEPLOY_TIMEOUT_SECS" \
        "${REPO_PATH}/scripts/deploy/deploy-service.sh" \
        --repo "$REPO_PATH" \
        --host-dir "$HOST_DIR" \
        --service "$SERVICE" \
        --force-recreate \
        >"$action_log" 2>&1 || rc=$?

    case "$rc" in
        0)
            report "$ACTION_ID" "$SERVICE" "true" ""
            ;;
        3)
            report "$ACTION_ID" "$SERVICE" "false" \
                "${SERVICE} owns its deploy path (services/${SERVICE}/deploy-local.sh) — needs an operator deploy, not an automated redeploy"
            ;;
        124)
            report "$ACTION_ID" "$SERVICE" "false" \
                "deploy timed out after ${DEPLOY_TIMEOUT_SECS}s (log: ${action_log})"
            ;;
        *)
            # Last 20 lines are the useful part of a compose failure; the full
            # output stays in the per-action log on the node.
            err=$(tail -n 20 "$action_log" 2>/dev/null | tr -d '\000' | tail -c 2000)
            report "$ACTION_ID" "$SERVICE" "false" \
                "deploy-service.sh exited ${rc} (log: ${action_log}): ${err}"
            ;;
    esac

    touch "$marker" || log "WARN: could not write idempotency marker ${marker}"
    rm -f "$action_file"
done

exit 0