homelab-codex-ws/packages/kb-mail/src/kb_mail/message.py

"""Message-level derivations shared by every job that turns `.eml` bytes into an `Envelope`:
the envelope id, the timestamp, the attachment manifest, and the archive path they imply.

Extracted from `gmail_bulk_import.importer` when `jobs/mail-imap-sync` needed the identical
four derivations (recon `kb/audits/mail-sync-2026-08-06.md` §2.1) — the poller's dedup rests
entirely on producing byte-identical ids to the 225 030 rows the bulk import already wrote, so
a second implementation of `message_id()` is the one thing that must not exist.
`gmail_bulk_import.importer` re-exports these under its old private names, unchanged.
"""
from __future__ import annotations

import email.message
import hashlib
from datetime import datetime, timezone
from email.utils import parsedate_to_datetime

import structlog

from .text import sanitize_surrogates as _sanitize

_log = structlog.get_logger(__name__)

EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc)

# Mirrors archive._UNSAFE — used to reconstruct raw_ref for already-archived messages
# without writing them again.
_ARCHIVE_UNSAFE = str.maketrans({"/": "_", "\\": "_", ":": "_", "<": "", ">": ""})


def eml_ref(envelope_id: str, source: str, ts: datetime) -> str:
    """The archive-relative `.eml` path `save_eml` would write, computed without writing.

    Kept in lockstep with `kb_mail.archive.save_eml` — callers use it on `FileExistsError`
    ("already archived") to fill `Envelope.raw_ref` for a message they did not just write.
    """
    safe_id = envelope_id.translate(_ARCHIVE_UNSAFE)
    return f"{source}/{ts.year:04d}/{ts.month:02d}/{safe_id}.eml"


def message_id(msg: email.message.Message) -> str:
    """Return a stable envelope id from the Message-ID header, or a SHA-256 content hash.

    This IS the dedup key (`envelope.id`, `ON CONFLICT (id) DO NOTHING`), so its output must
    stay byte-identical across jobs and years — 9 of the 225 030 existing rows carry the
    `sha256-` form, the rest a bare Message-ID.

    compat32 `.get()` returns an `email.header.Header` (not str) when the raw value holds
    8-bit bytes — `str()` + `_sanitize` keeps the id jsonb-safe and gives `.strip()` a real
    string to work on. An unguarded `.strip()` on a Header raised AttributeError and killed a
    whole bulk-import run (the archive has proven 8-bit header bytes).
    """
    mid = _sanitize(str(msg.get("Message-ID", ""))).strip().strip("<>")
    if mid:
        return mid
    return "sha256-" + hashlib.sha256(msg.as_bytes()).hexdigest()[:32]


def parse_date(msg: email.message.Message) -> datetime:
    """Parse the Date header to a UTC-aware datetime. Returns `EPOCH` as last resort.

    `str()` first: a compat32 8-bit Date surfaces as an `email.header.Header`, and
    `parsedate_to_datetime()` raises on a Header (→ needless epoch fallback) even when
    `str(header)` is perfectly parseable.

    Note this is the SENDER's clock, which is why it must never be used as a sync cursor —
    2 559 envelopes in the live corpus sit at epoch 1970 (recon §2.2 wariant C).
    """
    date_str = str(msg.get("Date", ""))
    if date_str:
        try:
            dt = parsedate_to_datetime(date_str)
            if dt.tzinfo is None:
                dt = dt.replace(tzinfo=timezone.utc)
            return dt.astimezone(timezone.utc)
        except Exception:
            pass
    return EPOCH


def parse_attachments(msg: email.message.Message) -> list[dict]:
    """Return `entities[type=attachment]` descriptors from the MIME structure.

    A part is an attachment if it has a filename, `Content-Disposition: attachment`, or is not
    inline `text/plain`/`text/html`. Errors on individual parts are logged and skipped — never
    propagated, since one unwalkable part must not cost the whole message.
    """
    attachments = []
    for part in msg.walk():
        if part.get_content_maintype() == "multipart":
            continue
        content_type = part.get_content_type()
        disposition = part.get_content_disposition() or ""
        filename = part.get_filename()

        if (
            content_type in ("text/plain", "text/html")
            and disposition != "attachment"
            and not filename
        ):
            continue

        try:
            payload = part.get_payload(decode=True)
            if payload is None:
                continue
            attachments.append({
                "type": "attachment",
                "filename": filename,
                "content_type": content_type,
                "size": len(payload),
                "sha256": hashlib.sha256(payload).hexdigest(),
            })
        except Exception:
            _log.warning("attachment_parse_failed",
                         filename=filename, content_type=content_type)
    return attachments