From 4518b15f985e21c146c882abba50082aca1e50b4 Mon Sep 17 00:00:00 2001 From: oskar Date: Wed, 24 Jun 2026 17:13:44 +0200 Subject: [PATCH] feat(fleet-prometheus): scaffold fleet liveness Prometheus (VPS) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clean scaffold of a Prometheus instance dedicated to fleet liveness, deliberately separate from the home `prom` on PIHA. Scrapes only itself + the VPS-local node_exporter for now. - image pinned to prom/prometheus:v3.5.0 (LTS) — no :latest anti-pattern - TSDB retention 15d / 2GB on the tight VPS; --web.enable-lifecycle for reloads - mem_limit 512m, oom_score_adj +200 (sacrificial OOM victim before control-plane) - tailscale-internal exposure via plain published 9090, mirroring control-plane - node_exporter scraped via host.docker.internal:9100 (it runs network_mode: host) - secret-free prometheus.yml; alerting/rule_files left as commented placeholders - in-container healthcheck via busybox wget (present in the image; curl is not) Smoke: docker compose config OK; promtool check config SUCCESS; up -d -> /-/healthy + /-/ready OK; both targets (prometheus, fleet-node) report up. Co-Authored-By: Claude Opus 4.8 (1M context) --- services/fleet-prometheus/README.md | 82 ++++++++++++++++++++ services/fleet-prometheus/docker-compose.yml | 50 ++++++++++++ services/fleet-prometheus/env.example | 8 ++ services/fleet-prometheus/healthcheck.sh | 19 +++++ services/fleet-prometheus/prometheus.yml | 39 ++++++++++ services/fleet-prometheus/service.yaml | 25 ++++++ 6 files changed, 223 insertions(+) create mode 100644 services/fleet-prometheus/README.md create mode 100644 services/fleet-prometheus/docker-compose.yml create mode 100644 services/fleet-prometheus/env.example create mode 100755 services/fleet-prometheus/healthcheck.sh create mode 100644 services/fleet-prometheus/prometheus.yml create mode 100644 services/fleet-prometheus/service.yaml diff --git a/services/fleet-prometheus/README.md b/services/fleet-prometheus/README.md new file mode 100644 index 0000000..c1d3d3a --- /dev/null +++ b/services/fleet-prometheus/README.md @@ -0,0 +1,82 @@ +# fleet-prometheus + +Prometheus instance that is the **source of truth for fleet liveness**. Runs on +the **VPS** (ingress / control-plane host) and is reachable only over Tailscale. + +This is a **clean scaffold**: right now it scrapes only itself and the VPS-local +`node_exporter`. The fleet targets, liveness rules, and alerting integration are +deliberately separate, later steps (see *Next steps* below). + +## Why separate from the home `prom` + +The home `prom` on **PIHA** is LAN-only and monitors the home network. This +instance is intentionally a **distinct** Prometheus dedicated to the distributed +fleet — hence the `fleet-` naming. The split is made explicit in every series: + +- `external_labels: { fleet: "homelab-codex" }` +- job names prefixed/scoped for the fleet (`prometheus`, `fleet-node`) + +It also avoids two anti-patterns the home `prom` has: + +- **No `:latest`** — the image is pinned to `prom/prometheus:v3.5.0` (LTS). +- **No inline secrets** — `prometheus.yml` is secret-free by contract (the home + prom embeds a plaintext HAOS token; we do not). + +## Stack + +| Container | Image | Purpose | +|--------------------|-------------------------|--------------------------------| +| `fleet-prometheus` | `prom/prometheus:v3.5.0`| Fleet liveness Prometheus (9090) | + +## Placement & exposure + +- **Node:** VPS (`ubuntu-4gb-hel1-1`, Tailscale `100.95.58.48`). +- **Exposure:** `tailscale-internal` — exposed exactly like `control-plane`: a + plain published `9090:9090` port with **no** public npm reverse-proxy entry. + Reachability is constrained to the Tailscale mesh at the VPS firewall layer. + Do **not** add this to npm or public DNS. +- **Memory:** `mem_limit: 512m`. `oom_score_adj: 200` (positive) — fleet + monitoring is less critical than the control-plane (`-900`), so it is a + sacrificial OOM victim *before* the control-plane, never after. +- **Retention:** `15d` / `2GB` (whichever hits first) — tight on purpose. + +## Data + +TSDB lives in the named volume `fleet-prometheus_tsdb` → `/prometheus`. + +## Configuration + +- `prometheus.yml` — committed, **secret-free**. Global config + two scrape jobs. +- `env.example` — no secrets / no required env (kept only for layout parity). + +### Scrape targets (scaffold) + +| Job | Target | Notes | +|--------------|-----------------------------|-----------------------------------------| +| `prometheus` | `localhost:9090` | self | +| `fleet-node` | `host.docker.internal:9100` | VPS `node_exporter` (runs `network_mode: host`, listens on host `:9100`); reached via `host-gateway` | + +## Verify + +```bash +# Config is syntactically valid (uses promtool inside the image): +docker run --rm -v "$PWD/services/fleet-prometheus/prometheus.yml":/etc/prometheus/prometheus.yml:ro \ + prom/prometheus:v3.5.0 promtool check config /etc/prometheus/prometheus.yml + +# Compose renders: +docker compose -f services/fleet-prometheus/docker-compose.yml config + +# After `up -d`: +curl -sf http://localhost:9090/-/healthy +curl -s http://localhost:9090/api/v1/targets | jq '.data.activeTargets[].health' +./services/fleet-prometheus/healthcheck.sh +``` + +## Next steps (NOT done in this scaffold) + +- **Fleet targets:** add the `100.x` Tailscale `node_exporter`s for SATURN, + SOLARIA, PIHA, CHELSTY to the `fleet-node` job. +- **Liveness rules:** `rule_files` with `up == 0 for: 5m` (placeholder is + commented in `prometheus.yml`). +- **Alerting:** Alertmanager wiring, and/or `brain-watchdog` querying the + firing alerts API → Telegram (placeholder commented in `prometheus.yml`). diff --git a/services/fleet-prometheus/docker-compose.yml b/services/fleet-prometheus/docker-compose.yml new file mode 100644 index 0000000..7a404b4 --- /dev/null +++ b/services/fleet-prometheus/docker-compose.yml @@ -0,0 +1,50 @@ +services: + fleet-prometheus: + # Pinned to the Prometheus 3.5 LTS line — NEVER :latest. The home `prom` + # on PIHA runs :latest, which is the anti-pattern we are deliberately not + # repeating here: a fleet liveness source must be reproducible. + image: prom/prometheus:v3.5.0 + container_name: fleet-prometheus + restart: unless-stopped + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + # Tight VPS (3.7 GiB RAM + 4 GiB swap): keep the TSDB small. Whichever + # of time/size is hit first triggers compaction/deletion. + - '--storage.tsdb.retention.time=15d' + - '--storage.tsdb.retention.size=2GB' + # Allow `POST /-/reload` so liveness rules (added in a later step) can be + # hot-loaded without restarting and losing the scrape gap. + - '--web.enable-lifecycle' + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro + - fleet_prometheus_tsdb:/prometheus + ports: + # tailscale-internal: exposed exactly like control-plane (plain published + # HOST:CONTAINER mapping, no public npm reverse-proxy entry). Reachability + # is constrained to the Tailscale mesh at the VPS firewall layer, same as + # control-plane's 18180. Do NOT add this to npm / public DNS. + - "9090:9090" + # node_exporter on the VPS runs with network_mode: host, so it listens on + # the host's :9100. This bridged container reaches it via host-gateway. + extra_hosts: + - "host.docker.internal:host-gateway" + # Memory: VPS has 4 GiB RAM. 512m cgroup ceiling; Docker restarts the + # container on breach. oom_score_adj is POSITIVE on purpose — fleet + # monitoring is less critical than the control-plane (-900), so the host + # OOM-killer should sacrifice this container FIRST, never the control-plane. + mem_limit: 512m + oom_score_adj: 200 + # The prom/prometheus image is busybox-based and ships `wget` (but no curl), + # so an in-container HTTP check works here — unlike the vikunja image, which + # had neither and broke its healthcheck permanently. + healthcheck: + test: ["CMD", "wget", "-q", "-O", "-", "http://localhost:9090/-/healthy"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + +volumes: + fleet_prometheus_tsdb: + name: fleet-prometheus_tsdb diff --git a/services/fleet-prometheus/env.example b/services/fleet-prometheus/env.example new file mode 100644 index 0000000..286f42f --- /dev/null +++ b/services/fleet-prometheus/env.example @@ -0,0 +1,8 @@ +# fleet-prometheus has NO secrets and NO required environment. +# +# Configuration lives entirely in prometheus.yml (secret-free by contract) and +# the compose command flags. There is intentionally nothing to copy to a .env. +# +# This template exists only to keep the standard service file layout. If a +# future step needs env (e.g. an Alertmanager URL), add it here and wire an +# env_file into docker-compose.yml at that point. diff --git a/services/fleet-prometheus/healthcheck.sh b/services/fleet-prometheus/healthcheck.sh new file mode 100755 index 0000000..9eb0474 --- /dev/null +++ b/services/fleet-prometheus/healthcheck.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Host-side healthcheck for fleet-prometheus. +# Mirrors the vikunja pattern: confirm the container runs, then probe the API +# from the host. + +# Container must be running +if ! docker ps --filter "name=fleet-prometheus" --filter "status=running" | grep -qw "fleet-prometheus"; then + echo "[FAIL] fleet-prometheus container is not running" + exit 1 +fi + +# Prometheus must report itself healthy +if ! curl -sf http://localhost:9090/-/healthy > /dev/null; then + echo "[FAIL] Prometheus is not healthy on :9090" + exit 1 +fi + +echo "[OK] fleet-prometheus is healthy" +exit 0 diff --git a/services/fleet-prometheus/prometheus.yml b/services/fleet-prometheus/prometheus.yml new file mode 100644 index 0000000..c6039a8 --- /dev/null +++ b/services/fleet-prometheus/prometheus.yml @@ -0,0 +1,39 @@ +# fleet-prometheus — fleet liveness source of truth. +# +# Deliberately SEPARATE from the home `prom` on PIHA (LAN-only). The `fleet` +# external label and the `fleet-` job naming make it unambiguous in every +# scraped series that this instance monitors the distributed fleet, not the +# home LAN. This file is SECRET-FREE by contract — never inline tokens here +# (the home prom embedding a plaintext HAOS token is the anti-pattern we avoid). + +global: + scrape_interval: 15s + external_labels: + fleet: "homelab-codex" + +# alerting: +# # Alertmanager wiring is added in a separate step (fleet liveness alerting). +# alertmanagers: [] + +# rule_files: +# # Liveness rules (e.g. up == 0 for: 5m) are added in a separate step. +# - "rules/*.yml" + +scrape_configs: + # Self-monitoring — Prometheus scrapes its own /metrics. Bridged container + # listens on 0.0.0.0:9090, so localhost resolves inside the container. + - job_name: "prometheus" + static_configs: + - targets: ["localhost:9090"] + + # First and only fleet target for this scaffold: the VPS-local node_exporter, + # which runs network_mode: host and listens on the host's :9100. Reached via + # the host-gateway alias declared in docker-compose.yml. + - job_name: "fleet-node" + static_configs: + - targets: ["host.docker.internal:9100"] + labels: + node: "vps" + + # NOTE: Tailscale fleet targets (100.x node_exporters across SATURN, SOLARIA, + # PIHA, CHELSTY) are added in a separate step — not part of this scaffold. diff --git a/services/fleet-prometheus/service.yaml b/services/fleet-prometheus/service.yaml new file mode 100644 index 0000000..986de23 --- /dev/null +++ b/services/fleet-prometheus/service.yaml @@ -0,0 +1,25 @@ +service: + name: fleet-prometheus + owner_node: vps + role: fleet-liveness-source + exposure: tailscale-internal # reachable on Tailscale; NOT behind public npm proxy + dependencies: + - node_exporter # first scrape target (VPS-local host metrics) + ports: + - container: 9090 + host: 9090 + protocol: tcp + healthcheck: + type: http + endpoint: http://localhost:9090/-/healthy + interval: 30s + timeout: 10s + retries: 5 + restart_policy: unless-stopped + persistence: + paths: + - fleet-prometheus_tsdb # TSDB -> /prometheus (named volume) + runtime: + config_files: + - prometheus.yml # scrape config (secret-free, in git) + env_vars: [] # no secrets / no required env