269 lines
7.4 KiB
Bash
Executable file
269 lines
7.4 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# deploy.sh - Staged deployment framework for homelab nodes.
|
|
|
|
set -o pipefail
|
|
|
|
# --- Configuration ---
|
|
export RUNTIME_PATH="/opt/homelab"
|
|
export STATE_DIR="${RUNTIME_PATH}/state/deploy"
|
|
export LOG_DIR="${RUNTIME_PATH}/logs/deploy"
|
|
export REPO_PATH="${HOME}/homelab-codex-ws"
|
|
export TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
|
export LOG_FILE="${LOG_DIR}/deploy_${TIMESTAMP}.log"
|
|
|
|
# --- Initialization ---
|
|
mkdir -p "$STATE_DIR" "$LOG_DIR"
|
|
|
|
# Redirection for logging
|
|
exec > >(tee -a "$LOG_FILE") 2>&1
|
|
|
|
# --- Load Libraries ---
|
|
LIB_PATH="${REPO_PATH}/scripts/lib"
|
|
source "${LIB_PATH}/log.sh"
|
|
source "${LIB_PATH}/state.sh"
|
|
source "${LIB_PATH}/inventory.sh"
|
|
source "${LIB_PATH}/compose.sh"
|
|
source "${LIB_PATH}/diagnostics.sh"
|
|
|
|
# --- CLI Parsing ---
|
|
TARGET_HOST=$(hostname)
|
|
TARGET_SERVICE=""
|
|
RESUME=false
|
|
REQUESTED_STAGE=""
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case $1 in
|
|
--host)
|
|
TARGET_HOST="$2"
|
|
shift 2
|
|
;;
|
|
--service)
|
|
TARGET_SERVICE="$2"
|
|
shift 2
|
|
;;
|
|
--resume)
|
|
RESUME=true
|
|
shift
|
|
;;
|
|
--stage)
|
|
REQUESTED_STAGE="$2"
|
|
shift 2
|
|
;;
|
|
*)
|
|
if [[ "$1" =~ ^(prepare|validate|deploy|verify|diagnose|complete)$ ]]; then
|
|
REQUESTED_STAGE="$1"
|
|
fi
|
|
shift
|
|
;;
|
|
esac
|
|
done
|
|
|
|
# --- Stages ---
|
|
|
|
stage_prepare() {
|
|
local host=$1
|
|
if is_stage_complete "prepare" && [[ "$RESUME" == "true" ]]; then
|
|
log "INFO" "Skipping PREPARE (already complete)"
|
|
return 0
|
|
fi
|
|
|
|
log "INFO" "Stage: PREPARE ($host)"
|
|
set_stage "prepare"
|
|
|
|
cd "$REPO_PATH" || exit 1
|
|
log "INFO" "Pulling latest changes..."
|
|
if ! git pull; then
|
|
log "WARN" "Git pull failed, proceeding with local state (offline mode or network flap)"
|
|
fi
|
|
|
|
# Ensure runtime directories exist
|
|
mkdir -p "${RUNTIME_PATH}/config" "${RUNTIME_PATH}/data" "${RUNTIME_PATH}/state" "${RUNTIME_PATH}/logs"
|
|
|
|
struct_log "prepare" "$host" "all" "success" "repo_updated"
|
|
mark_stage_complete "prepare"
|
|
}
|
|
|
|
stage_validate() {
|
|
local host=$1
|
|
if is_stage_complete "validate" && [[ "$RESUME" == "true" ]]; then
|
|
log "INFO" "Skipping VALIDATE (already complete)"
|
|
return 0
|
|
fi
|
|
|
|
log "INFO" "Stage: VALIDATE ($host)"
|
|
set_stage "validate"
|
|
|
|
for service in "${SERVICES[@]}"; do
|
|
log "INFO" "Validating $service..."
|
|
if [[ ! -d "${REPO_PATH}/services/$service" ]]; then
|
|
log "ERROR" "Service definition not found: $service"
|
|
struct_log "validate" "$host" "$service" "fail" "not_found"
|
|
return 1
|
|
fi
|
|
done
|
|
|
|
struct_log "validate" "$host" "all" "success" "validated"
|
|
mark_stage_complete "validate"
|
|
}
|
|
|
|
stage_deploy() {
|
|
local host=$1
|
|
if is_stage_complete "deploy" && [[ "$RESUME" == "true" ]]; then
|
|
log "INFO" "Skipping DEPLOY (already complete)"
|
|
return 0
|
|
fi
|
|
|
|
log "INFO" "Stage: DEPLOY ($host)"
|
|
set_stage "deploy"
|
|
|
|
local last_s=$(get_last_service)
|
|
local skip=false
|
|
if [[ "$RESUME" == "true" && -n "$last_s" ]]; then
|
|
skip=true
|
|
fi
|
|
|
|
for service in "${SERVICES[@]}"; do
|
|
if [[ "$skip" == "true" ]]; then
|
|
if [[ "$service" == "$last_s" ]]; then
|
|
skip=false
|
|
log "INFO" "Resuming from $service..."
|
|
else
|
|
log "INFO" "Skipping $service (already processed)"
|
|
continue
|
|
fi
|
|
fi
|
|
|
|
log "INFO" "Deploying $service..."
|
|
set_last_service "$service"
|
|
|
|
if ! run_compose_up "$service"; then
|
|
struct_log "deploy" "$host" "$service" "fail" "docker_compose_failed"
|
|
collect_diagnostics "$host" "$service"
|
|
return 1
|
|
fi
|
|
|
|
struct_log "deploy" "$host" "$service" "success" "deployed"
|
|
done
|
|
|
|
set_last_service ""
|
|
mark_stage_complete "deploy"
|
|
}
|
|
|
|
stage_verify() {
|
|
local host=$1
|
|
if is_stage_complete "verify" && [[ "$RESUME" == "true" ]]; then
|
|
log "INFO" "Skipping VERIFY (already complete)"
|
|
return 0
|
|
fi
|
|
|
|
log "INFO" "Stage: VERIFY ($host)"
|
|
set_stage "verify"
|
|
|
|
for service in "${SERVICES[@]}"; do
|
|
log "INFO" "Verifying $service..."
|
|
local health_script="${REPO_PATH}/services/${service}/healthcheck.sh"
|
|
if [[ -f "$health_script" ]]; then
|
|
if ! bash "$health_script"; then
|
|
log "ERROR" "Healthcheck failed for $service"
|
|
struct_log "verify" "$host" "$service" "fail" "healthcheck_failed"
|
|
collect_diagnostics "$host" "$service"
|
|
return 1
|
|
fi
|
|
else
|
|
# Generic check if container is running
|
|
if ! docker ps --filter "name=$service" --filter "status=running" | grep -q "$service"; then
|
|
log "ERROR" "Container $service is not running"
|
|
struct_log "verify" "$host" "$service" "fail" "container_not_running"
|
|
collect_diagnostics "$host" "$service"
|
|
return 1
|
|
fi
|
|
fi
|
|
struct_log "verify" "$host" "$service" "success" "verified"
|
|
done
|
|
mark_stage_complete "verify"
|
|
}
|
|
|
|
stage_complete() {
|
|
local host=$1
|
|
log "INFO" "Stage: COMPLETE ($host)"
|
|
set_stage "complete"
|
|
struct_log "complete" "$host" "all" "success" "deployment_finished"
|
|
clear_deployment_state
|
|
}
|
|
|
|
# --- Execution Logic ---
|
|
|
|
run_deployment() {
|
|
local start_stage=$1
|
|
|
|
# Sequential execution from start_stage
|
|
case "$start_stage" in
|
|
prepare)
|
|
stage_prepare "$TARGET_HOST" || return 1
|
|
;&
|
|
validate)
|
|
stage_validate "$TARGET_HOST" || return 1
|
|
;&
|
|
deploy)
|
|
stage_deploy "$TARGET_HOST" || return 1
|
|
;&
|
|
verify)
|
|
stage_verify "$TARGET_HOST" || return 1
|
|
;&
|
|
complete)
|
|
stage_complete "$TARGET_HOST" || return 1
|
|
;;
|
|
*)
|
|
log "ERROR" "Invalid stage: $start_stage"
|
|
return 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# --- Main ---
|
|
|
|
log "INFO" "--- Homelab Deployment Started (Host: $TARGET_HOST, Service: ${TARGET_SERVICE:-all}) ---"
|
|
|
|
if ! load_inventory "$TARGET_HOST" "$TARGET_SERVICE"; then
|
|
log "ERROR" "Failed to load inventory"
|
|
exit 1
|
|
fi
|
|
|
|
EXIT_STATUS=0
|
|
if [[ "$RESUME" == "true" ]]; then
|
|
CURRENT=$(get_stage)
|
|
log "INFO" "Resuming from state: $CURRENT"
|
|
case "$CURRENT" in
|
|
prepare|validate|deploy|verify)
|
|
run_deployment "$CURRENT" || EXIT_STATUS=1
|
|
;;
|
|
complete|none)
|
|
log "INFO" "No interrupted deployment found. Starting from scratch..."
|
|
run_deployment "prepare" || EXIT_STATUS=1
|
|
;;
|
|
*)
|
|
log "INFO" "Unknown state. Starting from prepare..."
|
|
run_deployment "prepare" || EXIT_STATUS=1
|
|
;;
|
|
esac
|
|
elif [[ -n "$REQUESTED_STAGE" ]]; then
|
|
if [[ "$REQUESTED_STAGE" == "diagnose" ]]; then
|
|
collect_diagnostics "$TARGET_HOST" "$TARGET_SERVICE"
|
|
else
|
|
run_deployment "$REQUESTED_STAGE" || EXIT_STATUS=1
|
|
fi
|
|
else
|
|
# New deployment - clear previous state
|
|
clear_deployment_state
|
|
run_deployment "prepare" || EXIT_STATUS=1
|
|
fi
|
|
|
|
if [[ $EXIT_STATUS -eq 0 ]]; then
|
|
print_summary "$TARGET_HOST" "SUCCESS"
|
|
log "INFO" "--- Homelab Deployment Finished Successfully ---"
|
|
else
|
|
print_summary "$TARGET_HOST" "FAILED"
|
|
log "ERROR" "--- Homelab Deployment Failed ---"
|
|
exit 1
|
|
fi
|