2026-07-13 20:00:00 +02:00
|
|
|
[build-system]
|
|
|
|
|
requires = ["setuptools>=68"]
|
|
|
|
|
build-backend = "setuptools.build_meta"
|
|
|
|
|
|
|
|
|
|
[project]
|
|
|
|
|
name = "documents-ingest"
|
|
|
|
|
version = "0.1.0"
|
|
|
|
|
requires-python = ">=3.11"
|
|
|
|
|
dependencies = [
|
|
|
|
|
"asyncpg>=0.29",
|
|
|
|
|
"structlog>=24.1",
|
feat(documents-ingest): Paperless -> envelope adapter (module 5 phase 2 step 5)
Adds documents-ingest-paperless: paginated GET /api/documents/, maps each
doc to a source='paperless' envelope per plan §4.2-4.3, reusing
kb_mail.Envelope/insert_envelope unchanged (packages/kb-mail not touched).
Cross-source link (source_mail entity) is a deterministic join of
original_file_name against the phase-1 registry.json consume_name index —
no heuristics, no correspondent guessing (plan decision 4). Stats always
balance (fetched = already_in_db + inserted + errors) and main() now also
exits non-zero on imbalance, not just on errors>0, matching the exit-code
convention already established in gmail-bulk-import.
Verified live on PIHA (rsync to /tmp, ~/kb/venv, PIHA checkout untouched):
dry-run then --apply inserted 186/186 paperless envelopes (0 errors,
180 source_mail links), a second --apply reported inserted=0/already_in_db=186
(idempotent), gmail rows stayed at 225030 and document_chunk stayed empty.
Rotated the kb-ingest Paperless API token after it was accidentally
partially echoed during recon (old token now dead).
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-15 15:06:56 +02:00
|
|
|
"aiohttp>=3.9",
|
|
|
|
|
"kb-mail",
|
2026-07-13 20:00:00 +02:00
|
|
|
]
|
|
|
|
|
|
|
|
|
|
[project.scripts]
|
|
|
|
|
documents-ingest = "documents_ingest.extractor:main"
|
feat(documents-ingest): Paperless -> envelope adapter (module 5 phase 2 step 5)
Adds documents-ingest-paperless: paginated GET /api/documents/, maps each
doc to a source='paperless' envelope per plan §4.2-4.3, reusing
kb_mail.Envelope/insert_envelope unchanged (packages/kb-mail not touched).
Cross-source link (source_mail entity) is a deterministic join of
original_file_name against the phase-1 registry.json consume_name index —
no heuristics, no correspondent guessing (plan decision 4). Stats always
balance (fetched = already_in_db + inserted + errors) and main() now also
exits non-zero on imbalance, not just on errors>0, matching the exit-code
convention already established in gmail-bulk-import.
Verified live on PIHA (rsync to /tmp, ~/kb/venv, PIHA checkout untouched):
dry-run then --apply inserted 186/186 paperless envelopes (0 errors,
180 source_mail links), a second --apply reported inserted=0/already_in_db=186
(idempotent), gmail rows stayed at 225030 and document_chunk stayed empty.
Rotated the kb-ingest Paperless API token after it was accidentally
partially echoed during recon (old token now dead).
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-15 15:06:56 +02:00
|
|
|
documents-ingest-paperless = "documents_ingest.paperless_adapter:main"
|
feat(documents-ingest): chunk + embed job (module 5 phase 2 step 6)
Adds documents-ingest-embed: chunks source='paperless' envelope content
(paragraph-preferring, ~600 tok/chunk, ~150 tok overlap, hard char-fallback
for oversized paragraphs per plan §2 decision 3), embeds each chunk via
Ollama (bge-m3, dim validated against document_chunk's VECTOR(1024) on
every response) and inserts into document_chunk. Lives in documents-ingest
per the plan's own recommendation (§6 step 6) rather than a new package —
reuses the job family's existing idempotency/stats-balance/dry-run
conventions (paperless_adapter.py, gmail-header-backfill).
A 5-angle multi-agent code review of the initial implementation surfaced
three real bugs, fixed here: hard_split() could infinite-loop if
--chunk-overlap >= --chunk-size (now guarded in both hard_split() and
main()); insert_chunk() wasn't error-isolated like embed_chunk(), so a DB
write failure would crash the whole run instead of being counted and
skipped; and ON CONFLICT DO NOTHING's outcome was discarded, so a silently
skipped row (the known gap where document_chunk's UNIQUE constraint
doesn't include `model`) would have been miscounted as a successful insert
- now tracked separately as chunks_conflict_skipped and treated as a
run failure.
Smoke-tested and run to completion live on SOLARIA against the real Ollama
instance and kb-postgres@PIHA: dry-run matched the known phase-2-step-5
figures exactly (186 fetched, 26 empty_content, 2684 chunks planned), a
--limit 10 apply + idempotent re-run + DB/distance sanity checks all
passed, and the full 186-document run inserted 2683/2684 chunks (1 isolated
error - Ollama's runtime context window rejected one pathological
dot-leader table-of-contents chunk that tokenized far more densely than
estimated; documented as a known limitation, not fixed here given it's a
single-chunk edge case). Timing: ~0.79s/chunk average on CPU (SOLARIA's
Ollama runs GPU-less per the recent GPU-reservation-disabled fix), ~35 min
wall-clock for the full pilot - the real input for scoping the later
mail-corpus embedding phase (plan §7's GPU-based estimate doesn't hold
here).
pytest: 101 passed.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-15 20:41:00 +02:00
|
|
|
documents-ingest-embed = "documents_ingest.chunk_embed:main"
|
2026-07-13 20:00:00 +02:00
|
|
|
|
|
|
|
|
[tool.setuptools.packages.find]
|
|
|
|
|
where = ["src"]
|
|
|
|
|
|
|
|
|
|
[tool.pytest.ini_options]
|
|
|
|
|
asyncio_mode = "auto"
|
|
|
|
|
testpaths = ["tests"]
|