diff --git a/hosts/solaria/runtime/kb-postgres/docker-compose.override.yml b/hosts/solaria/runtime/kb-postgres/docker-compose.override.yml new file mode 100644 index 0000000..8c839ef --- /dev/null +++ b/hosts/solaria/runtime/kb-postgres/docker-compose.override.yml @@ -0,0 +1,6 @@ +# SOLARIA-specific overrides for kb-postgres. +# SOLARIA has 64 GiB RAM; these limits are guard-rails against runaway growth, +# not a hard budget constraint like on VPS. +services: + kb-postgres: + mem_limit: 4g diff --git a/hosts/solaria/services.yaml b/hosts/solaria/services.yaml index 2cddb0f..6ee8161 100644 --- a/hosts/solaria/services.yaml +++ b/hosts/solaria/services.yaml @@ -1,6 +1,22 @@ host: solaria services: + kb-postgres: + role: kb-database + deployment_model: docker-compose + exposure: local-only # Tailscale-accessible; no public ingress + offline_required: false + depends_on: + local: [] + external: [] + ports: + - name: postgres + host_port: 5433 + protocol: tcp + runtime: + config_path: /opt/homelab/config/kb-postgres + # data lives in Docker named volume: kb_postgres_data + node-agent: role: node-stability-monitor deployment_model: docker-compose diff --git a/inventory/topology.yaml b/inventory/topology.yaml index 8a7f9f4..1096b3b 100644 --- a/inventory/topology.yaml +++ b/inventory/topology.yaml @@ -27,6 +27,9 @@ nodes: roles: - compute - ai + services: + - node-agent + - kb-postgres # KB spine: Postgres 16 + pgvector, port 5433 vps: roles: diff --git a/packages/kb-mail/pyproject.toml b/packages/kb-mail/pyproject.toml new file mode 100644 index 0000000..351501b --- /dev/null +++ b/packages/kb-mail/pyproject.toml @@ -0,0 +1,28 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "kb-mail" +version = "0.1.0" +requires-python = ">=3.11" +dependencies = [ + "asyncpg>=0.29", + "structlog>=24.1", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.1", + "pytest-asyncio>=0.23", +] + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] +markers = [ + "integration: requires a running kb-postgres instance (set KB_TEST_DSN)", +] diff --git a/packages/kb-mail/src/kb_mail/__init__.py b/packages/kb-mail/src/kb_mail/__init__.py new file mode 100644 index 0000000..fc5bd07 --- /dev/null +++ b/packages/kb-mail/src/kb_mail/__init__.py @@ -0,0 +1,5 @@ +from .archive import save_eml +from .db import get_envelope, insert_envelope +from .envelope import Envelope + +__all__ = ["Envelope", "insert_envelope", "get_envelope", "save_eml"] diff --git a/packages/kb-mail/src/kb_mail/archive.py b/packages/kb-mail/src/kb_mail/archive.py new file mode 100644 index 0000000..8a2668a --- /dev/null +++ b/packages/kb-mail/src/kb_mail/archive.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import asyncio +from datetime import datetime +from pathlib import Path + +import structlog + +_log = structlog.get_logger(__name__) + +_UNSAFE = str.maketrans({"/": "_", "\\": "_", ":": "_", "<": "", ">": ""}) + + +async def save_eml( + archive_root: Path, + envelope_id: str, + source: str, + ts: datetime, + raw: bytes, +) -> str: + """Write raw .eml bytes to the append-only archive. + + Layout: {archive_root}/{source}/{YYYY}/{MM}/{sanitized_id}.eml + + Returns the relative raw_ref path (store this in Envelope.raw_ref). + Raises FileExistsError if envelope_id is already archived (append-only guarantee). + """ + safe_id = envelope_id.translate(_UNSAFE) + rel_path = f"{source}/{ts.year:04d}/{ts.month:02d}/{safe_id}.eml" + dest = archive_root / rel_path + + dest.parent.mkdir(parents=True, exist_ok=True) + + if dest.exists(): + raise FileExistsError(f"archive: {rel_path} already exists — archive is append-only") + + await asyncio.to_thread(dest.write_bytes, raw) + _log.info("archive.saved", raw_ref=rel_path, bytes=len(raw)) + return rel_path diff --git a/packages/kb-mail/src/kb_mail/db.py b/packages/kb-mail/src/kb_mail/db.py new file mode 100644 index 0000000..f3acf42 --- /dev/null +++ b/packages/kb-mail/src/kb_mail/db.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import json +from datetime import timezone +from typing import Optional + +import asyncpg +import structlog + +from .envelope import Envelope + +_log = structlog.get_logger(__name__) + + +async def insert_envelope(conn: asyncpg.Connection, env: Envelope) -> None: + """Insert envelope into kb-postgres. Silently ignores duplicate ids (ON CONFLICT DO NOTHING).""" + geo_param = json.dumps(env.geo) if env.geo is not None else None + await conn.execute( + """ + INSERT INTO envelope (id, source, ts, geo, raw_ref, entities) + VALUES ($1, $2, $3, $4::jsonb, $5, $6::jsonb) + ON CONFLICT (id) DO NOTHING + """, + env.id, + env.source, + env.ts, + geo_param, + env.raw_ref, + json.dumps(env.entities), + ) + _log.info("envelope.inserted", id=env.id, source=env.source) + + +async def get_envelope(conn: asyncpg.Connection, envelope_id: str) -> Optional[Envelope]: + """Fetch a single envelope by id. Returns None if not found.""" + row = await conn.fetchrow( + "SELECT id, source, ts, geo, raw_ref, entities FROM envelope WHERE id = $1", + envelope_id, + ) + if row is None: + return None + + ts = row["ts"] + if ts.tzinfo is None: + ts = ts.replace(tzinfo=timezone.utc) + + # asyncpg decodes JSONB columns to Python objects; handle both str and object + geo = _decode_jsonb(row["geo"]) + entities = _decode_jsonb(row["entities"]) or [] + + return Envelope( + id=row["id"], + source=row["source"], + ts=ts, + geo=geo, + raw_ref=row["raw_ref"], + entities=entities, + ) + + +def _decode_jsonb(value: object) -> object: + """Decode a JSONB value that may arrive as a string or already-decoded object.""" + if value is None: + return None + if isinstance(value, str): + return json.loads(value) + return value diff --git a/packages/kb-mail/src/kb_mail/envelope.py b/packages/kb-mail/src/kb_mail/envelope.py new file mode 100644 index 0000000..a72cdbc --- /dev/null +++ b/packages/kb-mail/src/kb_mail/envelope.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime +from typing import Optional + + +@dataclass +class Envelope: + """Frozen cross-source envelope — the only schema contract locked in upfront. + + Additive: new fields may be added but existing ones must not be removed or renamed. + All fields mirror the `envelope` table in kb-postgres (init/001_envelope.sql). + """ + + id: str + source: str # fastmail | gmail | … + ts: datetime # message date, must be UTC-aware + raw_ref: str # relative path to .eml in archive + geo: Optional[dict] = None # null for mails; filled by layer-3 enrich + entities: list = field(default_factory=list) + + def __post_init__(self) -> None: + if self.ts.tzinfo is None: + raise ValueError("Envelope.ts must be timezone-aware (UTC)") diff --git a/packages/kb-mail/tests/conftest.py b/packages/kb-mail/tests/conftest.py new file mode 100644 index 0000000..6ecf809 --- /dev/null +++ b/packages/kb-mail/tests/conftest.py @@ -0,0 +1,18 @@ +import os +import pytest +import asyncpg + + +@pytest.fixture +async def db_conn(): + """Async asyncpg connection wrapped in a transaction that rolls back after each test. + + Requires KB_TEST_DSN env var or a kb-postgres running on localhost:5433. + Only used by tests marked @pytest.mark.integration. + """ + dsn = os.environ.get("KB_TEST_DSN", "postgresql://kb:kb@localhost:5433/kb") + conn = await asyncpg.connect(dsn) + await conn.execute("BEGIN") + yield conn + await conn.execute("ROLLBACK") + await conn.close() diff --git a/packages/kb-mail/tests/test_archive.py b/packages/kb-mail/tests/test_archive.py new file mode 100644 index 0000000..7c523a2 --- /dev/null +++ b/packages/kb-mail/tests/test_archive.py @@ -0,0 +1,45 @@ +"""Archive helper unit tests — no DB required, uses tmp_path.""" +from __future__ import annotations + +from datetime import datetime, timezone + +import pytest + +from kb_mail.archive import save_eml + + +_TS = datetime(2024, 6, 10, tzinfo=timezone.utc) +_RAW = b"From: alice@example.com\r\nSubject: Hello\r\n\r\nBody text" + + +async def test_save_eml_creates_file(tmp_path): + raw_ref = await save_eml(tmp_path, "msg1@fastmail.com", "fastmail", _TS, _RAW) + + dest = tmp_path / raw_ref + assert dest.exists() + assert dest.read_bytes() == _RAW + + +async def test_save_eml_returns_correct_rel_path(tmp_path): + raw_ref = await save_eml(tmp_path, "msg1@fastmail.com", "fastmail", _TS, _RAW) + # @ is safe on Linux/macOS filesystems and is preserved + assert raw_ref == "fastmail/2024/06/msg1@fastmail.com.eml" + + +async def test_save_eml_creates_subdirs(tmp_path): + await save_eml(tmp_path, "msgX", "gmail", datetime(2023, 12, 31, tzinfo=timezone.utc), b"data") + assert (tmp_path / "gmail" / "2023" / "12").is_dir() + + +async def test_save_eml_append_only_raises_on_duplicate(tmp_path): + await save_eml(tmp_path, "dup@fastmail.com", "fastmail", _TS, _RAW) + with pytest.raises(FileExistsError, match="append-only"): + await save_eml(tmp_path, "dup@fastmail.com", "fastmail", _TS, _RAW) + + +async def test_save_eml_sanitizes_unsafe_chars(tmp_path): + raw_ref = await save_eml(tmp_path, "", "fastmail", _TS, b"x") + # < and > stripped, / and : replaced with _ + assert "<" not in raw_ref + assert ">" not in raw_ref + assert "/" not in raw_ref.split("fastmail/")[1].rsplit("/", 1)[-1] diff --git a/packages/kb-mail/tests/test_db.py b/packages/kb-mail/tests/test_db.py new file mode 100644 index 0000000..b4a339f --- /dev/null +++ b/packages/kb-mail/tests/test_db.py @@ -0,0 +1,86 @@ +"""DB round-trip integration tests — require a running kb-postgres. + +Run with: + KB_TEST_DSN=postgresql://kb:@solaria:5433/kb pytest -m integration +""" +from __future__ import annotations + +from datetime import datetime, timezone + +import pytest + +from kb_mail.db import get_envelope, insert_envelope +from kb_mail.envelope import Envelope + +pytestmark = pytest.mark.integration + +_TS = datetime(2024, 3, 15, 12, 0, 0, tzinfo=timezone.utc) + + +async def test_envelope_roundtrip(db_conn): + env = Envelope( + id="", + source="fastmail", + ts=_TS, + raw_ref="fastmail/2024/03/roundtrip_fastmail.com.eml", + ) + await insert_envelope(db_conn, env) + fetched = await get_envelope(db_conn, env.id) + + assert fetched is not None + assert fetched.id == env.id + assert fetched.source == env.source + assert fetched.ts == env.ts + assert fetched.raw_ref == env.raw_ref + assert fetched.geo is None + assert fetched.entities == [] + + +async def test_envelope_with_entities_roundtrip(db_conn): + env = Envelope( + id="", + source="gmail", + ts=_TS, + raw_ref="gmail/2024/03/entities_gmail.com.eml", + entities=[{"type": "person", "name": "Alice"}, {"type": "org", "name": "ACME"}], + ) + await insert_envelope(db_conn, env) + fetched = await get_envelope(db_conn, env.id) + + assert fetched is not None + assert fetched.entities == env.entities + + +async def test_get_nonexistent_returns_none(db_conn): + result = await get_envelope(db_conn, "nonexistent-id-xyz-abc") + assert result is None + + +async def test_duplicate_insert_is_idempotent(db_conn): + env = Envelope( + id="", + source="fastmail", + ts=_TS, + raw_ref="fastmail/2024/03/dup.eml", + ) + await insert_envelope(db_conn, env) + await insert_envelope(db_conn, env) # ON CONFLICT DO NOTHING + + fetched = await get_envelope(db_conn, env.id) + assert fetched is not None + assert fetched.id == env.id + + +async def test_envelope_with_geo_roundtrip(db_conn): + env = Envelope( + id="", + source="gmail", + ts=_TS, + raw_ref="gmail/2024/03/geo_gmail.com.eml", + geo={"lat": 52.2297, "lon": 21.0122, "label": "Warsaw"}, + ) + await insert_envelope(db_conn, env) + fetched = await get_envelope(db_conn, env.id) + + assert fetched is not None + assert fetched.geo == env.geo diff --git a/packages/kb-mail/tests/test_envelope.py b/packages/kb-mail/tests/test_envelope.py new file mode 100644 index 0000000..d9736c1 --- /dev/null +++ b/packages/kb-mail/tests/test_envelope.py @@ -0,0 +1,55 @@ +"""Envelope model unit tests — no DB required.""" +from __future__ import annotations + +from datetime import datetime, timezone + +import pytest + +from kb_mail.envelope import Envelope + + +def test_envelope_minimal(): + ts = datetime(2024, 1, 15, 10, 30, tzinfo=timezone.utc) + env = Envelope( + id="", + source="fastmail", + ts=ts, + raw_ref="fastmail/2024/01/abc123_fastmail.com.eml", + ) + assert env.id == "" + assert env.source == "fastmail" + assert env.ts == ts + assert env.geo is None + assert env.entities == [] + + +def test_envelope_rejects_naive_ts(): + with pytest.raises(ValueError, match="timezone-aware"): + Envelope( + id="msg", + source="fastmail", + ts=datetime(2024, 1, 1), # naive — no tzinfo + raw_ref="x", + ) + + +def test_envelope_with_entities(): + ts = datetime(2024, 6, 10, tzinfo=timezone.utc) + env = Envelope( + id="msg2", + source="gmail", + ts=ts, + raw_ref="gmail/2024/06/msg2.eml", + entities=[{"type": "person", "name": "Alice"}], + ) + assert len(env.entities) == 1 + assert env.entities[0]["name"] == "Alice" + + +def test_envelope_entities_are_independent(): + """Mutable default must not be shared between instances.""" + ts = datetime(2024, 1, 1, tzinfo=timezone.utc) + a = Envelope(id="a", source="fastmail", ts=ts, raw_ref="a.eml") + b = Envelope(id="b", source="fastmail", ts=ts, raw_ref="b.eml") + a.entities.append({"type": "org", "name": "ACME"}) + assert b.entities == [] diff --git a/packages/kb-mail/tests/test_migration.py b/packages/kb-mail/tests/test_migration.py new file mode 100644 index 0000000..e8c8961 --- /dev/null +++ b/packages/kb-mail/tests/test_migration.py @@ -0,0 +1,48 @@ +"""Sanity tests for SQL migration files — no DB or network required.""" +from __future__ import annotations + +from pathlib import Path + +REPO_ROOT = Path(__file__).parent.parent.parent.parent +INIT_DIR = REPO_ROOT / "services" / "kb-postgres" / "init" + + +def test_migration_001_exists(): + assert (INIT_DIR / "001_envelope.sql").exists(), \ + f"Expected {INIT_DIR / '001_envelope.sql'} to exist" + + +def test_migration_001_enables_vector_extension(): + sql = (INIT_DIR / "001_envelope.sql").read_text() + assert "CREATE EXTENSION" in sql + assert "vector" in sql.lower() + + +def test_migration_001_creates_envelope_table(): + sql = (INIT_DIR / "001_envelope.sql").read_text() + assert "CREATE TABLE" in sql + assert "envelope" in sql.lower() + + +def test_migration_001_has_all_frozen_columns(): + sql = (INIT_DIR / "001_envelope.sql").read_text() + frozen_columns = ("id", "source", "ts", "geo", "raw_ref", "entities") + for col in frozen_columns: + assert col in sql, f"Frozen column '{col}' missing from 001_envelope.sql" + + +def test_migration_001_geo_is_nullable(): + sql = (INIT_DIR / "001_envelope.sql").read_text() + # geo must NOT have NOT NULL constraint + lines = [l for l in sql.splitlines() if "geo" in l.lower()] + assert lines, "No 'geo' line found in migration SQL" + for line in lines: + assert "NOT NULL" not in line.upper(), \ + f"geo must be nullable but found: {line.strip()}" + + +def test_migration_001_entities_has_default(): + sql = (INIT_DIR / "001_envelope.sql").read_text() + lines = [l for l in sql.splitlines() if "entities" in l.lower()] + assert any("DEFAULT" in l.upper() for l in lines), \ + "entities column must have a DEFAULT '[]'" diff --git a/services/kb-postgres/README.md b/services/kb-postgres/README.md new file mode 100644 index 0000000..2550f1a --- /dev/null +++ b/services/kb-postgres/README.md @@ -0,0 +1,89 @@ +# kb-postgres + +Postgres 16 + pgvector — KB spine on SOLARIA. Stores the frozen envelope schema shared by all KB pillars (mails, documents, photos, transactions). + +Port: **5433** on SOLARIA (Tailscale-accessible to other nodes). + +## Standard deploy (from SATURN) + +```bash +# On SATURN — pushes to master, then deploy.sh SSHes to SOLARIA and runs deploy-node.sh +git push origin master +scripts/deploy/deploy.sh solaria +``` + +`deploy-node.sh` on SOLARIA automatically picks up the per-host override: +``` +docker compose \ + -f services/kb-postgres/docker-compose.yml \ + -f hosts/solaria/runtime/kb-postgres/docker-compose.override.yml \ + up -d --remove-orphans +``` + +## First-time setup on SOLARIA (before first deploy) + +The `.env` file must exist at `services/kb-postgres/.env` in the SOLARIA repo checkout +(alongside the compose file — that's where `env_file: .env` resolves to): + +```bash +# On SOLARIA +cd ~/homelab-codex-ws +cp services/kb-postgres/env.example services/kb-postgres/.env +# Edit .env: set POSTGRES_PASSWORD to something strong +``` + +`.env` is gitignored (`*.env` rule in root `.gitignore`) — it will never be committed. + +## Manual one-off (debugging / first boot) + +```bash +# On SOLARIA, from repo root +docker compose \ + -f services/kb-postgres/docker-compose.yml \ + -f hosts/solaria/runtime/kb-postgres/docker-compose.override.yml \ + up -d +``` + +## Verify after first boot + +```bash +# Host-side healthcheck +./services/kb-postgres/healthcheck.sh + +# Inside the container +docker exec kb-postgres psql -U kb -d kb -c '\d envelope' +docker exec kb-postgres psql -U kb -d kb \ + -c "SELECT extname FROM pg_extension WHERE extname = 'vector';" +``` + +Expected `\d envelope` output: + +``` + Table "public.envelope" + Column | Type | Nullable | Default +----------+--------------------------+----------+----------- + id | text | not null | + source | text | not null | + ts | timestamp with time zone | not null | + geo | jsonb | | + raw_ref | text | not null | + entities | jsonb | not null | '[]'::jsonb +Indexes: + "envelope_pkey" PRIMARY KEY, btree (id) + "envelope_source_idx" btree (source) + "envelope_ts_idx" btree (ts) +``` + +## Schema contract + +The `envelope` table is the frozen cross-source envelope (see `docs/kb/kb-00-overview.md` §Zasady przekrojowe). Adding columns is OK; removing or renaming existing ones is NOT. + +Future migrations go in `init/` as `002_*.sql`, `003_*.sql`, …. Postgres runs `initdb` scripts only on a fresh volume — for existing instances apply migrations with `psql` directly. + +## Connection string + +``` +postgresql://kb:@solaria:5433/kb +``` + +Set `KB_TEST_DSN` to this value when running integration tests from `packages/kb-mail/`. diff --git a/services/kb-postgres/docker-compose.yml b/services/kb-postgres/docker-compose.yml new file mode 100644 index 0000000..605402a --- /dev/null +++ b/services/kb-postgres/docker-compose.yml @@ -0,0 +1,25 @@ +services: + kb-postgres: + image: pgvector/pgvector:pg16 + container_name: kb-postgres + restart: unless-stopped + env_file: + - .env + environment: + - POSTGRES_USER=kb + - POSTGRES_DB=kb + - TZ=Europe/Warsaw + volumes: + - kb_postgres_data:/var/lib/postgresql/data + - ./init:/docker-entrypoint-initdb.d:ro + ports: + - "5433:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U kb -d kb"] + interval: 10s + timeout: 5s + retries: 5 + +volumes: + kb_postgres_data: + name: kb_postgres_data diff --git a/services/kb-postgres/env.example b/services/kb-postgres/env.example new file mode 100644 index 0000000..5bf40f2 --- /dev/null +++ b/services/kb-postgres/env.example @@ -0,0 +1,5 @@ +# kb-postgres secrets — copy to .env and fill with real values. +# .env is gitignored; never commit it. + +# Postgres superuser password for the kb database. +POSTGRES_PASSWORD=change-me-strong-password diff --git a/services/kb-postgres/healthcheck.sh b/services/kb-postgres/healthcheck.sh new file mode 100755 index 0000000..a10c708 --- /dev/null +++ b/services/kb-postgres/healthcheck.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +if ! docker ps --filter "name=kb-postgres" --filter "status=running" | grep -qw "kb-postgres"; then + echo "[FAIL] kb-postgres container is not running" + exit 1 +fi + +if ! docker exec kb-postgres pg_isready -U kb -d kb > /dev/null 2>&1; then + echo "[FAIL] kb-postgres is not accepting connections" + exit 1 +fi + +echo "[OK] kb-postgres is healthy" +exit 0 diff --git a/services/kb-postgres/init/001_envelope.sql b/services/kb-postgres/init/001_envelope.sql new file mode 100644 index 0000000..bfc00fe --- /dev/null +++ b/services/kb-postgres/init/001_envelope.sql @@ -0,0 +1,17 @@ +-- KB spine: vector extension + frozen envelope schema +-- Additive: new columns may be added but existing ones must not be removed or renamed. +-- Version: 001 — initial + +CREATE EXTENSION IF NOT EXISTS vector; + +CREATE TABLE IF NOT EXISTS envelope ( + id TEXT PRIMARY KEY, + source TEXT NOT NULL, -- fastmail | gmail | ... + ts TIMESTAMPTZ NOT NULL, -- message date, UTC + geo JSONB, -- null for mails; filled by layer-3 enrich + raw_ref TEXT NOT NULL, -- relative path to .eml in archive + entities JSONB NOT NULL DEFAULT '[]' +); + +CREATE INDEX IF NOT EXISTS envelope_source_idx ON envelope (source); +CREATE INDEX IF NOT EXISTS envelope_ts_idx ON envelope (ts); diff --git a/services/kb-postgres/service.yaml b/services/kb-postgres/service.yaml new file mode 100644 index 0000000..975a781 --- /dev/null +++ b/services/kb-postgres/service.yaml @@ -0,0 +1,22 @@ +service: + name: kb-postgres + owner_node: solaria + exposure: local-only # reachable on Tailscale; no public ingress + dependencies: [] + ports: + - container: 5432 + host: 5433 # 5433 avoids clash with any co-located postgres + protocol: tcp + healthcheck: + type: custom + command: pg_isready -U kb -d kb + interval: 30s + timeout: 10s + retries: 5 + restart_policy: unless-stopped + persistence: + paths: + - kb_postgres_data # Docker named volume + runtime: + env_vars: + - POSTGRES_PASSWORD