From cbb910d85603af9557033dcdc690a8cd1f362519 Mon Sep 17 00:00:00 2001 From: oskar Date: Fri, 3 Jul 2026 13:41:44 +0200 Subject: [PATCH] feat(llm-gateway): wciagniecie shadow-serwisu z PIHA do GitOps Kod zyl tylko na dysku PIHA w /opt/llm-gateway (bez gita). Przeniesiony do services/llm-gateway/ pod pelny wzorzec homelaba: - app/main.py: kod 1:1 z PIHA + OLLAMA_URL/CHAT_MODEL/CODE_MODEL nadpisywalne przez env (defaulty bez zmian) - docker-compose.yml: bind TYLKO do Tailscale IP (${TAILSCALE_BIND_IP}, wzorzec fleet-prometheus), nie 0.0.0.0 jak w starym compose - service.yaml, env.example (bez sekretow), healthcheck.sh, README, testy - hosts/piha/runtime/llm-gateway: mem_limit 256m (PIHA jest RAM-bound) - rejestracja w hosts/piha/services.yaml i inventory/topology.yaml Zepsuty /opt/llm-gateway/docker-compose.yml (zduplikowany klucz ports) celowo NIE przeniesiony. DoD: pytest 4 passed; docker build + smoke run OK (GET / -> gateway ok). Co-Authored-By: Claude Fable 5 --- .../llm-gateway/docker-compose.override.yml | 10 ++ hosts/piha/services.yaml | 16 +++ inventory/topology.yaml | 1 + services/llm-gateway/.dockerignore | 6 ++ services/llm-gateway/Dockerfile | 12 +++ services/llm-gateway/README.md | 55 +++++++++++ services/llm-gateway/app/main.py | 99 +++++++++++++++++++ services/llm-gateway/docker-compose.yml | 24 +++++ services/llm-gateway/env.example | 10 ++ services/llm-gateway/healthcheck.sh | 27 +++++ services/llm-gateway/requirements.txt | 3 + services/llm-gateway/service.yaml | 26 +++++ services/llm-gateway/tests/test_main.py | 31 ++++++ 13 files changed, 320 insertions(+) create mode 100644 hosts/piha/runtime/llm-gateway/docker-compose.override.yml create mode 100644 services/llm-gateway/.dockerignore create mode 100644 services/llm-gateway/Dockerfile create mode 100644 services/llm-gateway/README.md create mode 100644 services/llm-gateway/app/main.py create mode 100644 services/llm-gateway/docker-compose.yml create mode 100644 services/llm-gateway/env.example create mode 100755 services/llm-gateway/healthcheck.sh create mode 100644 services/llm-gateway/requirements.txt create mode 100644 services/llm-gateway/service.yaml create mode 100644 services/llm-gateway/tests/test_main.py diff --git a/hosts/piha/runtime/llm-gateway/docker-compose.override.yml b/hosts/piha/runtime/llm-gateway/docker-compose.override.yml new file mode 100644 index 0000000..e323c7f --- /dev/null +++ b/hosts/piha/runtime/llm-gateway/docker-compose.override.yml @@ -0,0 +1,10 @@ +# PIHA-specific overrides for llm-gateway. +# +# RESOURCE CONTEXT: PIHA (Pi 5, 8 GB) is RAM-bound — HA, Immich and monitoring +# already hold ~6 GB. llm-gateway is a tiny stateless proxy (uvicorn + httpx); +# it must never grow into HA's memory. 256 MB is several times its normal +# resident set, so a breach means a leak — let the cgroup OOM killer restart it. + +services: + llm-gateway: + mem_limit: 256m diff --git a/hosts/piha/services.yaml b/hosts/piha/services.yaml index f7bc0fd..6cdafc4 100644 --- a/hosts/piha/services.yaml +++ b/hosts/piha/services.yaml @@ -58,6 +58,22 @@ services: config_path: services/vikunja # data is in Docker named volumes: vikunja_vikunja_db, vikunja_vikunja_files + llm-gateway: + role: llm-router # FastAPI proxy -> Ollama @ SOLARIA :11434 + deployment_model: docker-compose + exposure: private # Tailscale-only bind (TAILSCALE_BIND_IP); no public ingress + offline_required: false + depends_on: + local: [] + external: [ollama] # SOLARIA may be powered down -> routes 502, health stays ok + ports: + - name: http + container_port: 8080 + protocol: tcp + runtime: + # .env (TAILSCALE_BIND_IP) lives alongside the compose file; stateless, no data path + config_path: services/llm-gateway + kb-postgres: role: kb-database # KB spine: Postgres 16 + pgvector (always-on) deployment_model: docker-compose diff --git a/inventory/topology.yaml b/inventory/topology.yaml index 6632260..e1f88da 100644 --- a/inventory/topology.yaml +++ b/inventory/topology.yaml @@ -48,6 +48,7 @@ nodes: - brain-watchdog - vikunja # Task management (vikunja + postgres), public via npm - kb-postgres # KB spine: Postgres 16 + pgvector, port 5433 (always-on) + - llm-gateway # FastAPI router -> Ollama @ SOLARIA (Tailscale-internal :8080) solaria: roles: diff --git a/services/llm-gateway/.dockerignore b/services/llm-gateway/.dockerignore new file mode 100644 index 0000000..74a0c0c --- /dev/null +++ b/services/llm-gateway/.dockerignore @@ -0,0 +1,6 @@ +__pycache__/ +*.pyc +.venv/ +.git/ +tests/ +.env diff --git a/services/llm-gateway/Dockerfile b/services/llm-gateway/Dockerfile new file mode 100644 index 0000000..c6450d6 --- /dev/null +++ b/services/llm-gateway/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY app ./app + +EXPOSE 8080 +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/services/llm-gateway/README.md b/services/llm-gateway/README.md new file mode 100644 index 0000000..bc292e8 --- /dev/null +++ b/services/llm-gateway/README.md @@ -0,0 +1,55 @@ +# llm-gateway + +Small FastAPI router in front of Ollama on **SOLARIA**. Runs on the **PIHA** +node, bound to PIHA's Tailscale IP only (`http://piha:8080` on the mesh; no +public ingress). Stateless — no volumes, no secrets. + +Origin: migrated 2026-07-03 from an unmanaged shadow deployment at +`/opt/llm-gateway` on PIHA into GitOps. The old copy is kept on disk as +rollback until explicitly retired. + +## Endpoints + +| Endpoint | Method | Model | Purpose | +|---|---|---|---| +| `/` | GET | — | Health: `{"status":"gateway ok"}` | +| `/api/chat` | POST | `deepcoder:14b` | General chat completion | +| `/api/code` | POST | `deepseek-coder:latest` | Code/command generation ("return only code" system prompt) | + +Request body for both POST routes: `{"prompt": "...", "stream": false}`. + +`/api/code` quirk (by design): a single-line prompt whose first word is not a +natural-language starter (write/create/explain/…) is treated as an +already-formed shell command and echoed back verbatim as `response`. + +## Upstream dependency + +Calls Ollama at `http://solaria:11434/api/generate` (override via `OLLAMA_URL`). +SOLARIA is a GPU node that may be powered down — the gateway stays healthy +(`/` still answers) but proxied routes return **502** until Ollama is back. +`solaria` resolves via the host's Tailscale MagicDNS, which Docker's embedded +DNS forwards to. + +## Configuration + +- `.env` — **gitignored**, copy from `env.example`. Holds `TAILSCALE_BIND_IP` + (required for the mesh-only port bind) and optional `OLLAMA_URL`. +- No secrets anywhere in the stack. + +## Deploy (PIHA) + +1. `git pull` on PIHA (`~/homelab-codex-ws`). +2. `cp services/llm-gateway/env.example services/llm-gateway/.env` (defaults + are correct for PIHA). +3. ``` + docker compose -f services/llm-gateway/docker-compose.yml \ + -f hosts/piha/runtime/llm-gateway/docker-compose.override.yml up -d --build + ``` +4. Verify: `services/llm-gateway/healthcheck.sh`, then from any mesh node: + `curl http://piha:8080/` → `{"status":"gateway ok"}`. + +## Tests + +``` +cd services/llm-gateway && pytest +``` diff --git a/services/llm-gateway/app/main.py b/services/llm-gateway/app/main.py new file mode 100644 index 0000000..9fe649c --- /dev/null +++ b/services/llm-gateway/app/main.py @@ -0,0 +1,99 @@ +import os + +import httpx +from fastapi import FastAPI, HTTPException +from pydantic import BaseModel + + +class PromptRequest(BaseModel): + prompt: str + stream: bool = False + + +app = FastAPI() +# Ollama lives on SOLARIA (GPU node); resolved via Tailscale MagicDNS on the +# host, which Docker's embedded DNS forwards to. Overridable per node via env. +OLLAMA_URL = os.environ.get('OLLAMA_URL', 'http://solaria:11434/api/generate') +CHAT_MODEL = os.environ.get('CHAT_MODEL', 'deepcoder:14b') +CODE_MODEL = os.environ.get('CODE_MODEL', 'deepseek-coder:latest') +NATURAL_LANGUAGE_STARTERS = { + 'write', + 'create', + 'generate', + 'make', + 'say', + 'show', + 'give', + 'explain', + 'tell', + 'build', +} + + +@app.get('/') +async def root() -> dict[str, str]: + return {'status': 'gateway ok'} + + +async def generate(route: str, model: str, request: PromptRequest) -> dict[str, object]: + payload = { + 'model': model, + 'prompt': request.prompt, + 'stream': False, + } + try: + async with httpx.AsyncClient(timeout=120.0) as client: + ollama_response = await client.post(OLLAMA_URL, json=payload) + ollama_response.raise_for_status() + except httpx.HTTPStatusError as exc: + raise HTTPException(status_code=502, detail=f'Ollama error: {exc.response.text}') from exc + except httpx.HTTPError as exc: + raise HTTPException(status_code=502, detail=f'Ollama unreachable: {exc}') from exc + + raw = ollama_response.json() + return { + 'route': route, + 'model': model, + 'response': raw.get('response', ''), + 'raw': raw, + } + + +def looks_like_shell_command(prompt: str) -> bool: + stripped = prompt.strip() + if not stripped or '\n' in stripped: + return False + first_token = stripped.split()[0].lower() + if first_token in NATURAL_LANGUAGE_STARTERS: + return False + return True + + +@app.post('/api/chat') +async def chat(request: PromptRequest) -> dict[str, object]: + return await generate('chat', CHAT_MODEL, request) + + +@app.post('/api/code') +async def code(request: PromptRequest) -> dict[str, object]: + prompt = request.prompt + code_prompt = f""" +You are a coding and command assistant. + +Return ONLY the final answer. +Do NOT explain. +Do NOT use markdown. +Do NOT mention that you are an AI. +Do NOT add prose. + +If the task is a shell command request, return only the shell command. +If the task is a programming request, return only the code. + +TASK: +{prompt} +""".strip() + code_request = PromptRequest(prompt=code_prompt, stream=request.stream) + result = await generate('code', CODE_MODEL, code_request) + if looks_like_shell_command(prompt): + result['response'] = prompt.strip() + return result diff --git a/services/llm-gateway/docker-compose.yml b/services/llm-gateway/docker-compose.yml new file mode 100644 index 0000000..c652f16 --- /dev/null +++ b/services/llm-gateway/docker-compose.yml @@ -0,0 +1,24 @@ +services: + llm-gateway: + build: . + container_name: llm-gateway + restart: unless-stopped + ports: + # tailscale-internal: the listen socket is bound ONLY to the node's + # Tailscale interface IP (TAILSCALE_BIND_IP), never 0.0.0.0 — same + # convention as fleet-prometheus. Reachability is enforced in the bind + # itself; consumers reach it as http://piha:8080 over the mesh. + # Requires .env (from env.example) next to this file at deploy. + - "${TAILSCALE_BIND_IP}:8080:8080" + environment: + # Ollama upstream on SOLARIA; "solaria" resolves via the host's + # Tailscale MagicDNS, forwarded by Docker's embedded DNS. + - OLLAMA_URL=${OLLAMA_URL:-http://solaria:11434/api/generate} + # python:3.12-slim has no curl/wget, but it does have python — same + # in-container check pattern as control-plane. + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/', timeout=3).read()"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 10s diff --git a/services/llm-gateway/env.example b/services/llm-gateway/env.example new file mode 100644 index 0000000..3b97140 --- /dev/null +++ b/services/llm-gateway/env.example @@ -0,0 +1,10 @@ +# llm-gateway has NO secrets, but the Tailscale-only port bind needs one +# host-local value. Copy this file to .env (gitignored) next to +# docker-compose.yml; docker compose picks it up automatically. + +# Tailscale IP of the PIHA node. Bind the listen socket ONLY to the mesh — +# never 0.0.0.0. Verify when rebuilding the host: tailscale ip -4. +TAILSCALE_BIND_IP=100.108.208.3 + +# Optional: override the Ollama upstream (defaults to SOLARIA over MagicDNS). +# OLLAMA_URL=http://solaria:11434/api/generate diff --git a/services/llm-gateway/healthcheck.sh b/services/llm-gateway/healthcheck.sh new file mode 100755 index 0000000..9019be5 --- /dev/null +++ b/services/llm-gateway/healthcheck.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Healthcheck for llm-gateway (FastAPI router -> Ollama @ SOLARIA) + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# The port is bound to the Tailscale interface only, so localhost won't answer. +# Read the bind IP from .env (same file compose uses for the port mapping). +if [ -f "$SCRIPT_DIR/.env" ]; then + # shellcheck disable=SC1091 + source "$SCRIPT_DIR/.env" +fi +BIND_IP="${TAILSCALE_BIND_IP:-127.0.0.1}" + +# Container must be running +if ! docker ps --filter "name=llm-gateway" --filter "status=running" | grep -qw "llm-gateway"; then + echo "[FAIL] llm-gateway container is not running" + exit 1 +fi + +# Health endpoint must answer with the gateway-ok marker +if ! curl -sf "http://${BIND_IP}:8080/" | grep -q "gateway ok"; then + echo "[FAIL] llm-gateway is not responding on ${BIND_IP}:8080" + exit 1 +fi + +echo "[OK] llm-gateway is healthy" +exit 0 diff --git a/services/llm-gateway/requirements.txt b/services/llm-gateway/requirements.txt new file mode 100644 index 0000000..08f3b72 --- /dev/null +++ b/services/llm-gateway/requirements.txt @@ -0,0 +1,3 @@ +fastapi==0.115.6 +httpx==0.28.1 +uvicorn[standard]==0.34.0 diff --git a/services/llm-gateway/service.yaml b/services/llm-gateway/service.yaml new file mode 100644 index 0000000..25b565b --- /dev/null +++ b/services/llm-gateway/service.yaml @@ -0,0 +1,26 @@ +service: + name: llm-gateway + owner_node: piha + role: llm-router + exposure: private # Tailscale-only bind (TAILSCALE_BIND_IP); no public ingress, no npm + dependencies: + - ollama # upstream inference @ SOLARIA :11434 (external node; SOLARIA may be powered down) + ports: + - container: 8080 + host: 8080 + protocol: tcp + healthcheck: + type: http + endpoint: http://localhost:8080/ # GET / -> {"status":"gateway ok"} + interval: 30s + timeout: 10s + retries: 5 + restart_policy: unless-stopped + persistence: + paths: [] # stateless — nothing to back up + runtime: + config_files: + - .env # TAILSCALE_BIND_IP (+ optional OLLAMA_URL); gitignored, from env.example + env_vars: + - TAILSCALE_BIND_IP # required — compose port-bind interpolation + - OLLAMA_URL # optional — defaults to http://solaria:11434/api/generate diff --git a/services/llm-gateway/tests/test_main.py b/services/llm-gateway/tests/test_main.py new file mode 100644 index 0000000..1af2ec2 --- /dev/null +++ b/services/llm-gateway/tests/test_main.py @@ -0,0 +1,31 @@ +import pathlib +import sys + +from fastapi.testclient import TestClient + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) + +from app.main import app, looks_like_shell_command # noqa: E402 + + +def test_root_health(): + client = TestClient(app) + response = client.get('/') + assert response.status_code == 200 + assert response.json() == {'status': 'gateway ok'} + + +def test_shell_command_heuristic_detects_commands(): + assert looks_like_shell_command('ls -la /opt') + assert looks_like_shell_command('docker ps') + + +def test_shell_command_heuristic_rejects_natural_language(): + assert not looks_like_shell_command('write a python function') + assert not looks_like_shell_command('Explain what this does') + + +def test_shell_command_heuristic_rejects_multiline_and_empty(): + assert not looks_like_shell_command('') + assert not looks_like_shell_command(' ') + assert not looks_like_shell_command('ls -la\necho done')