homelab-codex-ws/jobs/deploy-runner/tests/test_runner_integration.py

"""End-to-end test of deploy-runner.sh on a fake node.

Runs the real bash runner against temp directories with a stubbed `docker`, in
local (VPS) mode so no rsync/ssh is involved. Covers the full loop the control
plane depends on: dispatch file in → validate → deploy → action_result event
out → idempotency marker → inbox drained.
"""
from __future__ import annotations

import json
import os
import shutil
import subprocess
from pathlib import Path

import pytest

REPO = Path(__file__).resolve().parents[3]
RUNNER = REPO / "jobs" / "deploy-runner" / "deploy-runner.sh"


@pytest.fixture
def node(tmp_path: Path):
    """A fake node: repo checkout, /opt/homelab-alike runtime, docker stub."""
    repo = tmp_path / "repo"
    (repo / "scripts" / "deploy").mkdir(parents=True)
    shutil.copy(REPO / "scripts" / "deploy" / "deploy-service.sh",
                repo / "scripts" / "deploy" / "deploy-service.sh")
    os.chmod(repo / "scripts" / "deploy" / "deploy-service.sh", 0o755)

    (repo / "hosts" / "piha").mkdir(parents=True)
    (repo / "hosts" / "piha" / "services.yaml").write_text(
        "host: piha\nservices:\n  vikunja:\n    role: task-tracker\n"
    )
    (repo / "services" / "vikunja").mkdir(parents=True)
    (repo / "services" / "vikunja" / "docker-compose.yml").write_text("services: {}\n")

    runtime = tmp_path / "runtime"
    (runtime / "actions" / "deploy" / "piha").mkdir(parents=True)

    bindir = tmp_path / "bin"
    bindir.mkdir()
    argv_log = tmp_path / "docker-argv.txt"
    (bindir / "docker").write_text(
        f'#!/usr/bin/env bash\nprintf "%s " "$@" >> {argv_log}\necho >> {argv_log}\n'
        f'exit ${{DOCKER_STUB_EXIT:-0}}\n'
    )
    os.chmod(bindir / "docker", 0o755)

    return {"repo": repo, "runtime": runtime, "bindir": bindir, "argv_log": argv_log,
            "inbox": runtime / "actions" / "deploy" / "piha",
            "events": runtime / "events" / "piha"}


def dispatch(fake_node, action_id="redeploy-piha-vikunja", **overrides):
    action = {"action_id": action_id, "type": "redeploy", "node": "piha",
              "service": "vikunja", "dispatched_at": 0}
    action.update(overrides)
    (fake_node["inbox"] / f"{action_id}.json").write_text(json.dumps(action))


def run_runner(fake_node, **extra_env):
    env = dict(os.environ)
    env.update({
        "PATH": f"{fake_node['bindir']}:{os.environ['PATH']}",
        "NODE_NAME": "piha",
        "REPO_PATH": str(fake_node["repo"]),
        "RUNTIME_PATH": str(fake_node["runtime"]),
        "VPS_EVENTS_HOST": "",  # local mode: executor shares the same filesystem
    })
    env.update(extra_env)
    return subprocess.run([str(RUNNER)], capture_output=True, text=True, env=env)


def results(node) -> list[dict]:
    if not node["events"].exists():
        return []
    return [json.loads(p.read_text())
            for p in sorted(node["events"].glob("evt-*-action_result-*.json"))]


def test_successful_redeploy_reports_and_drains(node):
    dispatch(node)
    proc = run_runner(node)
    assert proc.returncode == 0, proc.stdout + proc.stderr

    assert "compose -f" in node["argv_log"].read_text()
    assert "--force-recreate" in node["argv_log"].read_text()

    events = results(node)
    assert len(events) == 1
    assert events[0]["payload"] == {
        "action_id": "redeploy-piha-vikunja", "success": True, "error": "",
        "node": "piha", "source": "deploy-runner",
    }

    assert list(node["inbox"].glob("*.json")) == []
    assert (node["runtime"] / "state" / "processed-deploy-actions"
            / "redeploy-piha-vikunja.done").exists()


def test_failed_deploy_reports_compose_output(node):
    dispatch(node)
    proc = run_runner(node, DOCKER_STUB_EXIT="1")
    assert proc.returncode == 0  # the runner itself succeeds; the action fails

    event = results(node)[0]
    assert event["payload"]["success"] is False
    assert "deploy-service.sh exited 1" in event["payload"]["error"]
    assert event["severity"] == "high"
    # Full output kept on the node for forensics
    assert list((node["runtime"] / "logs" / "deploy-runner").glob("*.log"))


def test_rejected_action_is_reported_not_silently_dropped(node):
    """An action for a service this node does not declare must come back as a
    failure — otherwise it sits in running/ until the executor times it out."""
    dispatch(node, service="gokapi")
    run_runner(node)

    event = results(node)[0]
    assert event["payload"]["success"] is False
    assert "not in hosts/piha/services.yaml" in event["payload"]["error"]
    assert not node["argv_log"].exists()  # docker never invoked
    assert list(node["inbox"].glob("*.json")) == []


def test_action_for_another_node_is_refused(node):
    dispatch(node, node="solaria")
    run_runner(node)

    event = results(node)[0]
    assert "addressed to node" in event["payload"]["error"]
    assert not node["argv_log"].exists()


def test_container_restart_is_not_executed_by_the_runner(node):
    """node-agent's job. The runner refuses anything but redeploy."""
    dispatch(node, type="container_restart")
    run_runner(node)

    event = results(node)[0]
    assert "not executable by the deploy runner" in event["payload"]["error"]
    assert not node["argv_log"].exists()


def test_already_processed_action_is_not_redeployed(node):
    dispatch(node)
    run_runner(node)
    first = len(results(node))

    dispatch(node)  # same action_id arrives again
    run_runner(node)

    assert len(results(node)) == first  # no second result
    assert len(node["argv_log"].read_text().splitlines()) == 1  # docker ran once
    assert list(node["inbox"].glob("*.json")) == []


def test_missing_node_name_fails_loudly(node):
    proc = run_runner(node, NODE_NAME="")
    assert proc.returncode == 1
    assert "NODE_NAME is not set" in proc.stdout + proc.stderr