From e5fcc2d4d911134f1076f050283aab928e979c63 Mon Sep 17 00:00:00 2001 From: oskar Date: Mon, 13 Jul 2026 21:08:41 +0200 Subject: [PATCH] =?UTF-8?q?feat(kb-postgres):=20migration=20002=5Fchunks.s?= =?UTF-8?q?ql=20=E2=80=94=20document=5Fchunk=20table?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Krok 1 planu docs/kb/modules/05-faza2-plan.md §3/§6 (chunk-level embeddings, 1:N do envelope). Addytywna — 001_envelope.sql nietknięta (zweryfikowane \d envelope po migracji: identyczny schemat + FK jako "Referenced by"). Schemat wg rekomendacji recon (§2 decyzja 1+2): osobna tabela (nie kolumna w envelope, bo N-wartościowy chunking jest obowiązkowy przy dokumentach >8k tokenów), embedding VECTOR(1024) pod bge-m3 (dense), HNSW cosine index, kolumna `model` do trywialnego re-indexu przy zmianie modelu (kb-00 zasada #1: indeks odtwarzalny). Idempotentna (CREATE TABLE/INDEX IF NOT EXISTS, zweryfikowane podwójnym uruchomieniem na kb-postgres@PIHA — drugi run same NOTICE "already exists, skipping"). Zastosowana na żywej bazie: ssh piha docker exec kb-postgres psql, po potwierdzeniu SQL przez Oskara. \dt + \d document_chunk + \d envelope zweryfikowane po migracji. Testy: dopisane sanity-testy 002 do packages/kb-mail/tests/test_migration.py (wzorzec 001 — statyczne assercje na treści SQL, bez DB), 13/13 zielone. Co NIE jest częścią tego kroku (§3 planu, odłożone): entity/entity_link (graf encji) — szkic na przyszłość, nie blokuje domknięcia modułu 5. Co dalej (plan §6, poza zakresem tego kroku): ollama pull bge-m3 na SOLARII, token API Paperless, jobs/gmail-header-backfill/, adapter Paperless→koperta, chunking+embed job. Co-Authored-By: Claude Sonnet 5 --- packages/kb-mail/tests/test_migration.py | 41 ++++++++++++++++++++++++ services/kb-postgres/init/002_chunks.sql | 18 +++++++++++ 2 files changed, 59 insertions(+) create mode 100644 services/kb-postgres/init/002_chunks.sql diff --git a/packages/kb-mail/tests/test_migration.py b/packages/kb-mail/tests/test_migration.py index e8c8961..f58a5d3 100644 --- a/packages/kb-mail/tests/test_migration.py +++ b/packages/kb-mail/tests/test_migration.py @@ -46,3 +46,44 @@ def test_migration_001_entities_has_default(): lines = [l for l in sql.splitlines() if "entities" in l.lower()] assert any("DEFAULT" in l.upper() for l in lines), \ "entities column must have a DEFAULT '[]'" + + +def test_migration_002_exists(): + assert (INIT_DIR / "002_chunks.sql").exists(), \ + f"Expected {INIT_DIR / '002_chunks.sql'} to exist" + + +def test_migration_002_does_not_touch_envelope(): + sql = (INIT_DIR / "002_chunks.sql").read_text() + assert "ALTER TABLE envelope" not in sql + assert "DROP TABLE envelope" not in sql.upper() + + +def test_migration_002_creates_document_chunk_table(): + sql = (INIT_DIR / "002_chunks.sql").read_text() + assert "CREATE TABLE IF NOT EXISTS document_chunk" in sql + + +def test_migration_002_references_envelope_with_cascade(): + sql = (INIT_DIR / "002_chunks.sql").read_text() + assert "REFERENCES envelope(id) ON DELETE CASCADE" in sql + + +def test_migration_002_embedding_dimension_matches_bge_m3(): + sql = (INIT_DIR / "002_chunks.sql").read_text() + # bge-m3 dense embedding dimension is 1024 (docs/kb/modules/05-faza2-plan.md §1.4) + assert "VECTOR(1024)" in sql.upper() + + +def test_migration_002_has_unique_envelope_chunk_index(): + sql = (INIT_DIR / "002_chunks.sql").read_text() + assert "UNIQUE (envelope_id, chunk_index)" in sql + + +def test_migration_002_uses_idempotent_ddl(): + sql = (INIT_DIR / "002_chunks.sql").read_text() + assert "CREATE TABLE IF NOT EXISTS" in sql + for line in sql.splitlines(): + if line.strip().upper().startswith("CREATE INDEX"): + assert "IF NOT EXISTS" in line.upper(), \ + f"CREATE INDEX must be idempotent: {line.strip()}" diff --git a/services/kb-postgres/init/002_chunks.sql b/services/kb-postgres/init/002_chunks.sql new file mode 100644 index 0000000..f0296ec --- /dev/null +++ b/services/kb-postgres/init/002_chunks.sql @@ -0,0 +1,18 @@ +-- KB spine: document_chunk — chunk-level embeddings, 1:N against envelope +-- Additive: does not modify 001_envelope.sql. envelope stays untouched. +-- Version: 002 — chunks (bge-m3 dense embeddings, dim=1024) + +CREATE TABLE IF NOT EXISTS document_chunk ( + id BIGSERIAL PRIMARY KEY, + envelope_id TEXT NOT NULL REFERENCES envelope(id) ON DELETE CASCADE, + chunk_index INT NOT NULL, -- order within envelope, 0-based + text TEXT NOT NULL, + embedding VECTOR(1024), -- bge-m3 dense; NULL until embedded + model TEXT NOT NULL, -- e.g. 'bge-m3' — tracks re-index on model change + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (envelope_id, chunk_index) +); + +CREATE INDEX IF NOT EXISTS document_chunk_envelope_idx ON document_chunk (envelope_id); +CREATE INDEX IF NOT EXISTS document_chunk_embedding_hnsw_idx + ON document_chunk USING hnsw (embedding vector_cosine_ops);