"""Message-level derivations shared by every job that turns `.eml` bytes into an `Envelope`: the envelope id, the timestamp, the attachment manifest, and the archive path they imply. Extracted from `gmail_bulk_import.importer` when `jobs/mail-imap-sync` needed the identical four derivations (recon `kb/audits/mail-sync-2026-08-06.md` §2.1) — the poller's dedup rests entirely on producing byte-identical ids to the 225 030 rows the bulk import already wrote, so a second implementation of `message_id()` is the one thing that must not exist. `gmail_bulk_import.importer` re-exports these under its old private names, unchanged. """ from __future__ import annotations import email.message import hashlib from datetime import datetime, timezone from email.utils import parsedate_to_datetime import structlog from .text import sanitize_surrogates as _sanitize _log = structlog.get_logger(__name__) EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc) # Mirrors archive._UNSAFE — used to reconstruct raw_ref for already-archived messages # without writing them again. _ARCHIVE_UNSAFE = str.maketrans({"/": "_", "\\": "_", ":": "_", "<": "", ">": ""}) def eml_ref(envelope_id: str, source: str, ts: datetime) -> str: """The archive-relative `.eml` path `save_eml` would write, computed without writing. Kept in lockstep with `kb_mail.archive.save_eml` — callers use it on `FileExistsError` ("already archived") to fill `Envelope.raw_ref` for a message they did not just write. """ safe_id = envelope_id.translate(_ARCHIVE_UNSAFE) return f"{source}/{ts.year:04d}/{ts.month:02d}/{safe_id}.eml" def message_id(msg: email.message.Message) -> str: """Return a stable envelope id from the Message-ID header, or a SHA-256 content hash. This IS the dedup key (`envelope.id`, `ON CONFLICT (id) DO NOTHING`), so its output must stay byte-identical across jobs and years — 9 of the 225 030 existing rows carry the `sha256-` form, the rest a bare Message-ID. compat32 `.get()` returns an `email.header.Header` (not str) when the raw value holds 8-bit bytes — `str()` + `_sanitize` keeps the id jsonb-safe and gives `.strip()` a real string to work on. An unguarded `.strip()` on a Header raised AttributeError and killed a whole bulk-import run (the archive has proven 8-bit header bytes). """ mid = _sanitize(str(msg.get("Message-ID", ""))).strip().strip("<>") if mid: return mid return "sha256-" + hashlib.sha256(msg.as_bytes()).hexdigest()[:32] def parse_date(msg: email.message.Message) -> datetime: """Parse the Date header to a UTC-aware datetime. Returns `EPOCH` as last resort. `str()` first: a compat32 8-bit Date surfaces as an `email.header.Header`, and `parsedate_to_datetime()` raises on a Header (→ needless epoch fallback) even when `str(header)` is perfectly parseable. Note this is the SENDER's clock, which is why it must never be used as a sync cursor — 2 559 envelopes in the live corpus sit at epoch 1970 (recon §2.2 wariant C). """ date_str = str(msg.get("Date", "")) if date_str: try: dt = parsedate_to_datetime(date_str) if dt.tzinfo is None: dt = dt.replace(tzinfo=timezone.utc) return dt.astimezone(timezone.utc) except Exception: pass return EPOCH def parse_attachments(msg: email.message.Message) -> list[dict]: """Return `entities[type=attachment]` descriptors from the MIME structure. A part is an attachment if it has a filename, `Content-Disposition: attachment`, or is not inline `text/plain`/`text/html`. Errors on individual parts are logged and skipped — never propagated, since one unwalkable part must not cost the whole message. """ attachments = [] for part in msg.walk(): if part.get_content_maintype() == "multipart": continue content_type = part.get_content_type() disposition = part.get_content_disposition() or "" filename = part.get_filename() if ( content_type in ("text/plain", "text/html") and disposition != "attachment" and not filename ): continue try: payload = part.get_payload(decode=True) if payload is None: continue attachments.append({ "type": "attachment", "filename": filename, "content_type": content_type, "size": len(payload), "sha256": hashlib.sha256(payload).hexdigest(), }) except Exception: _log.warning("attachment_parse_failed", filename=filename, content_type=content_type) return attachments