feat(kb-mail): fundament — pgvector spine, koperta, archiwum, pakiet domeny

- services/kb-postgres: pgvector/pgvector:pg16 na SOLARIA (:5433), named
  volume, init/001_envelope.sql (CREATE EXTENSION vector + zamrożona tabela
  envelope: id/source/ts/geo/raw_ref/entities), service.yaml, healthcheck,
  README z poprawnym mechanizmem deploy (deploy-node.sh składa dwa -f)
- hosts/solaria/runtime/kb-postgres/docker-compose.override.yml: mem_limit 4g
- inventory/topology.yaml + hosts/solaria/services.yaml: kb-postgres wpisany
- packages/kb-mail: nowa konwencja shared lib (pip install /repo/packages/<lib>/)
  envelope.py — @dataclass Envelope, walidacja tz-aware ts
  db.py       — insert_envelope / get_envelope (asyncpg, ON CONFLICT DO NOTHING)
  archive.py  — save_eml append-only (asyncio.to_thread, FileExistsError na dup)
  tests: 15 unit pass + 5 integration (@pytest.mark.integration, wymaga KB_TEST_DSN)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

hosts/solaria/runtime/kb-postgres/docker-compose.override.yml

@@ -0,0 +1,6 @@
+# SOLARIA-specific overrides for kb-postgres.
+# SOLARIA has 64 GiB RAM; these limits are guard-rails against runaway growth,
+# not a hard budget constraint like on VPS.
+services:
+  kb-postgres:
+    mem_limit: 4g

hosts/solaria/services.yaml

@@ -1,6 +1,22 @@
 host: solaria
 
 services:
+  kb-postgres:
+    role: kb-database
+    deployment_model: docker-compose
+    exposure: local-only           # Tailscale-accessible; no public ingress
+    offline_required: false
+    depends_on:
+      local: []
+      external: []
+    ports:
+      - name: postgres
+        host_port: 5433
+        protocol: tcp
+    runtime:
+      config_path: /opt/homelab/config/kb-postgres
+      # data lives in Docker named volume: kb_postgres_data
+
   node-agent:
     role: node-stability-monitor
     deployment_model: docker-compose

inventory/topology.yaml

@@ -27,6 +27,9 @@ nodes:
     roles:
       - compute
       - ai
+    services:
+      - node-agent
+      - kb-postgres             # KB spine: Postgres 16 + pgvector, port 5433
 
   vps:
     roles:

packages/kb-mail/pyproject.toml

@@ -0,0 +1,28 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "kb-mail"
+version = "0.1.0"
+requires-python = ">=3.11"
+dependencies = [
+    "asyncpg>=0.29",
+    "structlog>=24.1",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.1",
+    "pytest-asyncio>=0.23",
+]
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]
+markers = [
+    "integration: requires a running kb-postgres instance (set KB_TEST_DSN)",
+]

packages/kb-mail/src/kb_mail/__init__.py

@@ -0,0 +1,5 @@
+from .archive import save_eml
+from .db import get_envelope, insert_envelope
+from .envelope import Envelope
+
+__all__ = ["Envelope", "insert_envelope", "get_envelope", "save_eml"]

packages/kb-mail/src/kb_mail/archive.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+import asyncio
+from datetime import datetime
+from pathlib import Path
+
+import structlog
+
+_log = structlog.get_logger(__name__)
+
+_UNSAFE = str.maketrans({"/": "_", "\\": "_", ":": "_", "<": "", ">": ""})
+
+
+async def save_eml(
+    archive_root: Path,
+    envelope_id: str,
+    source: str,
+    ts: datetime,
+    raw: bytes,
+) -> str:
+    """Write raw .eml bytes to the append-only archive.
+
+    Layout: {archive_root}/{source}/{YYYY}/{MM}/{sanitized_id}.eml
+
+    Returns the relative raw_ref path (store this in Envelope.raw_ref).
+    Raises FileExistsError if envelope_id is already archived (append-only guarantee).
+    """
+    safe_id = envelope_id.translate(_UNSAFE)
+    rel_path = f"{source}/{ts.year:04d}/{ts.month:02d}/{safe_id}.eml"
+    dest = archive_root / rel_path
+
+    dest.parent.mkdir(parents=True, exist_ok=True)
+
+    if dest.exists():
+        raise FileExistsError(f"archive: {rel_path} already exists — archive is append-only")
+
+    await asyncio.to_thread(dest.write_bytes, raw)
+    _log.info("archive.saved", raw_ref=rel_path, bytes=len(raw))
+    return rel_path

packages/kb-mail/src/kb_mail/db.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+import json
+from datetime import timezone
+from typing import Optional
+
+import asyncpg
+import structlog
+
+from .envelope import Envelope
+
+_log = structlog.get_logger(__name__)
+
+
+async def insert_envelope(conn: asyncpg.Connection, env: Envelope) -> None:
+    """Insert envelope into kb-postgres. Silently ignores duplicate ids (ON CONFLICT DO NOTHING)."""
+    geo_param = json.dumps(env.geo) if env.geo is not None else None
+    await conn.execute(
+        """
+        INSERT INTO envelope (id, source, ts, geo, raw_ref, entities)
+        VALUES ($1, $2, $3, $4::jsonb, $5, $6::jsonb)
+        ON CONFLICT (id) DO NOTHING
+        """,
+        env.id,
+        env.source,
+        env.ts,
+        geo_param,
+        env.raw_ref,
+        json.dumps(env.entities),
+    )
+    _log.info("envelope.inserted", id=env.id, source=env.source)
+
+
+async def get_envelope(conn: asyncpg.Connection, envelope_id: str) -> Optional[Envelope]:
+    """Fetch a single envelope by id. Returns None if not found."""
+    row = await conn.fetchrow(
+        "SELECT id, source, ts, geo, raw_ref, entities FROM envelope WHERE id = $1",
+        envelope_id,
+    )
+    if row is None:
+        return None
+
+    ts = row["ts"]
+    if ts.tzinfo is None:
+        ts = ts.replace(tzinfo=timezone.utc)
+
+    # asyncpg decodes JSONB columns to Python objects; handle both str and object
+    geo = _decode_jsonb(row["geo"])
+    entities = _decode_jsonb(row["entities"]) or []
+
+    return Envelope(
+        id=row["id"],
+        source=row["source"],
+        ts=ts,
+        geo=geo,
+        raw_ref=row["raw_ref"],
+        entities=entities,
+    )
+
+
+def _decode_jsonb(value: object) -> object:
+    """Decode a JSONB value that may arrive as a string or already-decoded object."""
+    if value is None:
+        return None
+    if isinstance(value, str):
+        return json.loads(value)
+    return value

packages/kb-mail/src/kb_mail/envelope.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Optional
+
+
+@dataclass
+class Envelope:
+    """Frozen cross-source envelope — the only schema contract locked in upfront.
+
+    Additive: new fields may be added but existing ones must not be removed or renamed.
+    All fields mirror the `envelope` table in kb-postgres (init/001_envelope.sql).
+    """
+
+    id: str
+    source: str                              # fastmail | gmail | …
+    ts: datetime                             # message date, must be UTC-aware
+    raw_ref: str                             # relative path to .eml in archive
+    geo: Optional[dict] = None              # null for mails; filled by layer-3 enrich
+    entities: list = field(default_factory=list)
+
+    def __post_init__(self) -> None:
+        if self.ts.tzinfo is None:
+            raise ValueError("Envelope.ts must be timezone-aware (UTC)")

packages/kb-mail/tests/conftest.py

@@ -0,0 +1,18 @@
+import os
+import pytest
+import asyncpg
+
+
+@pytest.fixture
+async def db_conn():
+    """Async asyncpg connection wrapped in a transaction that rolls back after each test.
+
+    Requires KB_TEST_DSN env var or a kb-postgres running on localhost:5433.
+    Only used by tests marked @pytest.mark.integration.
+    """
+    dsn = os.environ.get("KB_TEST_DSN", "postgresql://kb:kb@localhost:5433/kb")
+    conn = await asyncpg.connect(dsn)
+    await conn.execute("BEGIN")
+    yield conn
+    await conn.execute("ROLLBACK")
+    await conn.close()

packages/kb-mail/tests/test_archive.py

@@ -0,0 +1,45 @@
+"""Archive helper unit tests — no DB required, uses tmp_path."""
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+import pytest
+
+from kb_mail.archive import save_eml
+
+
+_TS = datetime(2024, 6, 10, tzinfo=timezone.utc)
+_RAW = b"From: alice@example.com\r\nSubject: Hello\r\n\r\nBody text"
+
+
+async def test_save_eml_creates_file(tmp_path):
+    raw_ref = await save_eml(tmp_path, "msg1@fastmail.com", "fastmail", _TS, _RAW)
+
+    dest = tmp_path / raw_ref
+    assert dest.exists()
+    assert dest.read_bytes() == _RAW
+
+
+async def test_save_eml_returns_correct_rel_path(tmp_path):
+    raw_ref = await save_eml(tmp_path, "msg1@fastmail.com", "fastmail", _TS, _RAW)
+    # @ is safe on Linux/macOS filesystems and is preserved
+    assert raw_ref == "fastmail/2024/06/msg1@fastmail.com.eml"
+
+
+async def test_save_eml_creates_subdirs(tmp_path):
+    await save_eml(tmp_path, "msgX", "gmail", datetime(2023, 12, 31, tzinfo=timezone.utc), b"data")
+    assert (tmp_path / "gmail" / "2023" / "12").is_dir()
+
+
+async def test_save_eml_append_only_raises_on_duplicate(tmp_path):
+    await save_eml(tmp_path, "dup@fastmail.com", "fastmail", _TS, _RAW)
+    with pytest.raises(FileExistsError, match="append-only"):
+        await save_eml(tmp_path, "dup@fastmail.com", "fastmail", _TS, _RAW)
+
+
+async def test_save_eml_sanitizes_unsafe_chars(tmp_path):
+    raw_ref = await save_eml(tmp_path, "<foo/bar:baz>", "fastmail", _TS, b"x")
+    # < and > stripped, / and : replaced with _
+    assert "<" not in raw_ref
+    assert ">" not in raw_ref
+    assert "/" not in raw_ref.split("fastmail/")[1].rsplit("/", 1)[-1]

packages/kb-mail/tests/test_db.py

@@ -0,0 +1,86 @@
+"""DB round-trip integration tests — require a running kb-postgres.
+
+Run with:
+    KB_TEST_DSN=postgresql://kb:<pw>@solaria:5433/kb pytest -m integration
+"""
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+import pytest
+
+from kb_mail.db import get_envelope, insert_envelope
+from kb_mail.envelope import Envelope
+
+pytestmark = pytest.mark.integration
+
+_TS = datetime(2024, 3, 15, 12, 0, 0, tzinfo=timezone.utc)
+
+
+async def test_envelope_roundtrip(db_conn):
+    env = Envelope(
+        id="<roundtrip@fastmail.com>",
+        source="fastmail",
+        ts=_TS,
+        raw_ref="fastmail/2024/03/roundtrip_fastmail.com.eml",
+    )
+    await insert_envelope(db_conn, env)
+    fetched = await get_envelope(db_conn, env.id)
+
+    assert fetched is not None
+    assert fetched.id == env.id
+    assert fetched.source == env.source
+    assert fetched.ts == env.ts
+    assert fetched.raw_ref == env.raw_ref
+    assert fetched.geo is None
+    assert fetched.entities == []
+
+
+async def test_envelope_with_entities_roundtrip(db_conn):
+    env = Envelope(
+        id="<entities@gmail.com>",
+        source="gmail",
+        ts=_TS,
+        raw_ref="gmail/2024/03/entities_gmail.com.eml",
+        entities=[{"type": "person", "name": "Alice"}, {"type": "org", "name": "ACME"}],
+    )
+    await insert_envelope(db_conn, env)
+    fetched = await get_envelope(db_conn, env.id)
+
+    assert fetched is not None
+    assert fetched.entities == env.entities
+
+
+async def test_get_nonexistent_returns_none(db_conn):
+    result = await get_envelope(db_conn, "nonexistent-id-xyz-abc")
+    assert result is None
+
+
+async def test_duplicate_insert_is_idempotent(db_conn):
+    env = Envelope(
+        id="<dup@fastmail.com>",
+        source="fastmail",
+        ts=_TS,
+        raw_ref="fastmail/2024/03/dup.eml",
+    )
+    await insert_envelope(db_conn, env)
+    await insert_envelope(db_conn, env)   # ON CONFLICT DO NOTHING
+
+    fetched = await get_envelope(db_conn, env.id)
+    assert fetched is not None
+    assert fetched.id == env.id
+
+
+async def test_envelope_with_geo_roundtrip(db_conn):
+    env = Envelope(
+        id="<geo@gmail.com>",
+        source="gmail",
+        ts=_TS,
+        raw_ref="gmail/2024/03/geo_gmail.com.eml",
+        geo={"lat": 52.2297, "lon": 21.0122, "label": "Warsaw"},
+    )
+    await insert_envelope(db_conn, env)
+    fetched = await get_envelope(db_conn, env.id)
+
+    assert fetched is not None
+    assert fetched.geo == env.geo

packages/kb-mail/tests/test_envelope.py

@@ -0,0 +1,55 @@
+"""Envelope model unit tests — no DB required."""
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+import pytest
+
+from kb_mail.envelope import Envelope
+
+
+def test_envelope_minimal():
+    ts = datetime(2024, 1, 15, 10, 30, tzinfo=timezone.utc)
+    env = Envelope(
+        id="<abc123@fastmail.com>",
+        source="fastmail",
+        ts=ts,
+        raw_ref="fastmail/2024/01/abc123_fastmail.com.eml",
+    )
+    assert env.id == "<abc123@fastmail.com>"
+    assert env.source == "fastmail"
+    assert env.ts == ts
+    assert env.geo is None
+    assert env.entities == []
+
+
+def test_envelope_rejects_naive_ts():
+    with pytest.raises(ValueError, match="timezone-aware"):
+        Envelope(
+            id="msg",
+            source="fastmail",
+            ts=datetime(2024, 1, 1),   # naive — no tzinfo
+            raw_ref="x",
+        )
+
+
+def test_envelope_with_entities():
+    ts = datetime(2024, 6, 10, tzinfo=timezone.utc)
+    env = Envelope(
+        id="msg2",
+        source="gmail",
+        ts=ts,
+        raw_ref="gmail/2024/06/msg2.eml",
+        entities=[{"type": "person", "name": "Alice"}],
+    )
+    assert len(env.entities) == 1
+    assert env.entities[0]["name"] == "Alice"
+
+
+def test_envelope_entities_are_independent():
+    """Mutable default must not be shared between instances."""
+    ts = datetime(2024, 1, 1, tzinfo=timezone.utc)
+    a = Envelope(id="a", source="fastmail", ts=ts, raw_ref="a.eml")
+    b = Envelope(id="b", source="fastmail", ts=ts, raw_ref="b.eml")
+    a.entities.append({"type": "org", "name": "ACME"})
+    assert b.entities == []

packages/kb-mail/tests/test_migration.py

@@ -0,0 +1,48 @@
+"""Sanity tests for SQL migration files — no DB or network required."""
+from __future__ import annotations
+
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).parent.parent.parent.parent
+INIT_DIR = REPO_ROOT / "services" / "kb-postgres" / "init"
+
+
+def test_migration_001_exists():
+    assert (INIT_DIR / "001_envelope.sql").exists(), \
+        f"Expected {INIT_DIR / '001_envelope.sql'} to exist"
+
+
+def test_migration_001_enables_vector_extension():
+    sql = (INIT_DIR / "001_envelope.sql").read_text()
+    assert "CREATE EXTENSION" in sql
+    assert "vector" in sql.lower()
+
+
+def test_migration_001_creates_envelope_table():
+    sql = (INIT_DIR / "001_envelope.sql").read_text()
+    assert "CREATE TABLE" in sql
+    assert "envelope" in sql.lower()
+
+
+def test_migration_001_has_all_frozen_columns():
+    sql = (INIT_DIR / "001_envelope.sql").read_text()
+    frozen_columns = ("id", "source", "ts", "geo", "raw_ref", "entities")
+    for col in frozen_columns:
+        assert col in sql, f"Frozen column '{col}' missing from 001_envelope.sql"
+
+
+def test_migration_001_geo_is_nullable():
+    sql = (INIT_DIR / "001_envelope.sql").read_text()
+    # geo must NOT have NOT NULL constraint
+    lines = [l for l in sql.splitlines() if "geo" in l.lower()]
+    assert lines, "No 'geo' line found in migration SQL"
+    for line in lines:
+        assert "NOT NULL" not in line.upper(), \
+            f"geo must be nullable but found: {line.strip()}"
+
+
+def test_migration_001_entities_has_default():
+    sql = (INIT_DIR / "001_envelope.sql").read_text()
+    lines = [l for l in sql.splitlines() if "entities" in l.lower()]
+    assert any("DEFAULT" in l.upper() for l in lines), \
+        "entities column must have a DEFAULT '[]'"

services/kb-postgres/README.md

@@ -0,0 +1,89 @@
+# kb-postgres
+
+Postgres 16 + pgvector — KB spine on SOLARIA. Stores the frozen envelope schema shared by all KB pillars (mails, documents, photos, transactions).
+
+Port: **5433** on SOLARIA (Tailscale-accessible to other nodes).
+
+## Standard deploy (from SATURN)
+
+```bash
+# On SATURN — pushes to master, then deploy.sh SSHes to SOLARIA and runs deploy-node.sh
+git push origin master
+scripts/deploy/deploy.sh solaria
+```
+
+`deploy-node.sh` on SOLARIA automatically picks up the per-host override:
+```
+docker compose \
+  -f services/kb-postgres/docker-compose.yml \
+  -f hosts/solaria/runtime/kb-postgres/docker-compose.override.yml \
+  up -d --remove-orphans
+```
+
+## First-time setup on SOLARIA (before first deploy)
+
+The `.env` file must exist at `services/kb-postgres/.env` in the SOLARIA repo checkout
+(alongside the compose file — that's where `env_file: .env` resolves to):
+
+```bash
+# On SOLARIA
+cd ~/homelab-codex-ws
+cp services/kb-postgres/env.example services/kb-postgres/.env
+# Edit .env: set POSTGRES_PASSWORD to something strong
+```
+
+`.env` is gitignored (`*.env` rule in root `.gitignore`) — it will never be committed.
+
+## Manual one-off (debugging / first boot)
+
+```bash
+# On SOLARIA, from repo root
+docker compose \
+  -f services/kb-postgres/docker-compose.yml \
+  -f hosts/solaria/runtime/kb-postgres/docker-compose.override.yml \
+  up -d
+```
+
+## Verify after first boot
+
+```bash
+# Host-side healthcheck
+./services/kb-postgres/healthcheck.sh
+
+# Inside the container
+docker exec kb-postgres psql -U kb -d kb -c '\d envelope'
+docker exec kb-postgres psql -U kb -d kb \
+  -c "SELECT extname FROM pg_extension WHERE extname = 'vector';"
+```
+
+Expected `\d envelope` output:
+
+```
+                    Table "public.envelope"
+  Column  |           Type           | Nullable |  Default
+----------+--------------------------+----------+-----------
+ id       | text                     | not null |
+ source   | text                     | not null |
+ ts       | timestamp with time zone | not null |
+ geo      | jsonb                    |          |
+ raw_ref  | text                     | not null |
+ entities | jsonb                    | not null | '[]'::jsonb
+Indexes:
+    "envelope_pkey" PRIMARY KEY, btree (id)
+    "envelope_source_idx" btree (source)
+    "envelope_ts_idx" btree (ts)
+```
+
+## Schema contract
+
+The `envelope` table is the frozen cross-source envelope (see `docs/kb/kb-00-overview.md` §Zasady przekrojowe). Adding columns is OK; removing or renaming existing ones is NOT.
+
+Future migrations go in `init/` as `002_*.sql`, `003_*.sql`, …. Postgres runs `initdb` scripts only on a fresh volume — for existing instances apply migrations with `psql` directly.
+
+## Connection string
+
+```
+postgresql://kb:<POSTGRES_PASSWORD>@solaria:5433/kb
+```
+
+Set `KB_TEST_DSN` to this value when running integration tests from `packages/kb-mail/`.

services/kb-postgres/docker-compose.yml

@@ -0,0 +1,25 @@
+services:
+  kb-postgres:
+    image: pgvector/pgvector:pg16
+    container_name: kb-postgres
+    restart: unless-stopped
+    env_file:
+      - .env
+    environment:
+      - POSTGRES_USER=kb
+      - POSTGRES_DB=kb
+      - TZ=Europe/Warsaw
+    volumes:
+      - kb_postgres_data:/var/lib/postgresql/data
+      - ./init:/docker-entrypoint-initdb.d:ro
+    ports:
+      - "5433:5432"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U kb -d kb"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+volumes:
+  kb_postgres_data:
+    name: kb_postgres_data

services/kb-postgres/env.example

@@ -0,0 +1,5 @@
+# kb-postgres secrets — copy to .env and fill with real values.
+# .env is gitignored; never commit it.
+
+# Postgres superuser password for the kb database.
+POSTGRES_PASSWORD=change-me-strong-password

services/kb-postgres/healthcheck.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if ! docker ps --filter "name=kb-postgres" --filter "status=running" | grep -qw "kb-postgres"; then
+    echo "[FAIL] kb-postgres container is not running"
+    exit 1
+fi
+
+if ! docker exec kb-postgres pg_isready -U kb -d kb > /dev/null 2>&1; then
+    echo "[FAIL] kb-postgres is not accepting connections"
+    exit 1
+fi
+
+echo "[OK] kb-postgres is healthy"
+exit 0

services/kb-postgres/init/001_envelope.sql

@@ -0,0 +1,17 @@
+-- KB spine: vector extension + frozen envelope schema
+-- Additive: new columns may be added but existing ones must not be removed or renamed.
+-- Version: 001 — initial
+
+CREATE EXTENSION IF NOT EXISTS vector;
+
+CREATE TABLE IF NOT EXISTS envelope (
+    id          TEXT        PRIMARY KEY,
+    source      TEXT        NOT NULL,           -- fastmail | gmail | ...
+    ts          TIMESTAMPTZ NOT NULL,           -- message date, UTC
+    geo         JSONB,                          -- null for mails; filled by layer-3 enrich
+    raw_ref     TEXT        NOT NULL,           -- relative path to .eml in archive
+    entities    JSONB       NOT NULL DEFAULT '[]'
+);
+
+CREATE INDEX IF NOT EXISTS envelope_source_idx ON envelope (source);
+CREATE INDEX IF NOT EXISTS envelope_ts_idx     ON envelope (ts);

services/kb-postgres/service.yaml

@@ -0,0 +1,22 @@
+service:
+  name: kb-postgres
+  owner_node: solaria
+  exposure: local-only           # reachable on Tailscale; no public ingress
+  dependencies: []
+  ports:
+    - container: 5432
+      host: 5433                 # 5433 avoids clash with any co-located postgres
+      protocol: tcp
+  healthcheck:
+    type: custom
+    command: pg_isready -U kb -d kb
+    interval: 30s
+    timeout: 10s
+    retries: 5
+  restart_policy: unless-stopped
+  persistence:
+    paths:
+      - kb_postgres_data         # Docker named volume
+  runtime:
+    env_vars:
+      - POSTGRES_PASSWORD