e06cc10571
Second full pass over the gmail .eml archive (gmail-bulk-import's first pass skipped inline text/plain and text/html on purpose). Per envelope: typed parse with compat32 fallback -> body extraction (inline text/plain preferred, HTML->text via a small stdlib HTMLParser otherwise) -> quote-strip (reply markers + `>`-quoted lines, EN/PL/Outlook patterns) -> newsletter classification (List-Unsubscribe/List-Id/Precedence, chunked but not embedded, excluded_reason='newsletter') -> Temat/Od/Data prefix from the already-backfilled entities[type=headers] (zero header re-parse) -> chunk via kb_mail.chunking -> batched embed_batch (64) -> INSERT document_chunk. In-Reply-To/References are appended as entities[type=threading] during the same read (idempotent WHERE NOT EXISTS append, 1:1 with gmail-header-backfill) -- the only DB writes are document_chunk INSERTs and an additive envelope.entities UPDATE; the .eml archive stays read-only. --dsn/KB_DSN, --archive-root, --since/--limit/--offset, --batch-size, --apply (dry-run default). Idempotency keys on (envelope_id, chunk_index) pre-fetched scoped to --model, built correctly from the start per the plan's flagged chunk_embed.py precedent. A failed embed batch is isolated (chunks_errors, no abort) for Ollama's documented instability; a wrong embedding dimension aborts the whole run. 48 tests, DoD smoke run against live kb-postgres@PIHA confirmed wiring (archive not yet rsync'd to SOLARIA, so all 5 rows correctly reported missing_file). docs/kb/modules/05-faza-mailowa-plan.md, §5.
32 lines
564 B
TOML
32 lines
564 B
TOML
[build-system]
|
|
requires = ["setuptools>=68"]
|
|
build-backend = "setuptools.build_meta"
|
|
|
|
[project]
|
|
name = "mail-body-ingest"
|
|
version = "0.1.0"
|
|
requires-python = ">=3.11"
|
|
dependencies = [
|
|
"asyncpg>=0.29",
|
|
"aiohttp>=3.9",
|
|
"structlog>=24.1",
|
|
"kb-mail",
|
|
"kb-retrieval",
|
|
]
|
|
|
|
[project.optional-dependencies]
|
|
dev = [
|
|
"pytest>=8.1",
|
|
"pytest-asyncio>=0.23",
|
|
]
|
|
|
|
[project.scripts]
|
|
mail-body-ingest = "mail_body_ingest.ingest:main"
|
|
|
|
[tool.setuptools.packages.find]
|
|
where = ["src"]
|
|
|
|
[tool.pytest.ini_options]
|
|
asyncio_mode = "auto"
|
|
testpaths = ["tests"]
|