homelab-codex-ws/packages/kb-mail/src/kb_mail/sync_state.py

"""Per-folder IMAP sync cursor: the `mail_sync_state` table (migration 005) plus the pure
logic that decides what a tick fetches and how far the cursor is allowed to move.

The rules encoded here are the whole correctness story of the poller, so they live as pure
functions with their own tests rather than inline in the job's loop:

* **UIDVALIDITY governs everything.** When the server changes it, every stored UID becomes
  meaningless. The only safe response is to sweep the folder and lean on Message-ID dedup —
  a poller that trusts `last_uid` across a UIDVALIDITY change loses mail with no error
  anywhere (recon `kb/audits/mail-sync-2026-08-06.md` §2.2).
* **The cursor moves only over messages that are fully durable**, archive file and envelope
  row both. It moves over a CONTIGUOUS prefix of successes, so a message that failed is
  re-fetched next tick instead of being stepped over silently.
* **A first tick is a policy decision, not a default.** `new-only` starts the corpus from
  now; `since` closes a known gap (gmail's corpus stops at 2026-06-19); `full` pulls a
  mailbox's whole history. Which one is right depends on a measurement the runbook takes.
"""
from __future__ import annotations

from dataclasses import dataclass
from datetime import date, datetime
from typing import Iterable, Optional

import asyncpg
import structlog

from .imap import FolderStatus, ImapAccount

_log = structlog.get_logger(__name__)

# What a plan tells the caller to run against the server.
SEARCH_NONE = "none"          # fetch nothing this tick; just record the high-water mark
SEARCH_ALL = "all"            # UID SEARCH ALL — full sweep, dedup does the rest
SEARCH_SINCE = "since"        # UID SEARCH SINCE <date>
SEARCH_FROM_UID = "from-uid"  # UID SEARCH UID <start>:*


@dataclass(frozen=True)
class FolderSyncState:
    """One row of `mail_sync_state`."""

    account: str
    folder: str
    uidvalidity: int
    last_uid: int
    last_sync_ts: Optional[datetime] = None


@dataclass(frozen=True)
class SyncPlan:
    """What this tick will do to one folder.

    `baseline_uid` is the cursor to persist when the plan turns up NO messages — never a
    floor to combine with partial results, because on a full sweep it equals `uidnext - 1`
    and combining it with a partially-failed batch would step over everything that failed.
    """

    mode: str
    search: str
    baseline_uid: int
    start_uid: Optional[int] = None
    since: Optional[date] = None
    uidvalidity_reset: bool = False


def plan_folder_sync(
    account: ImapAccount,
    status: FolderStatus,
    state: Optional[FolderSyncState],
) -> SyncPlan:
    """Decide this tick's work for one folder from the stored cursor and the server's state."""
    high_water = max(status.uidnext - 1, 0)

    if state is None:
        if account.initial_mode == "new-only":
            # Start the corpus here: record the high-water mark, fetch nothing. Everything
            # already in the mailbox stays out of KB unless the operator re-runs with a
            # different initial_mode after clearing the row.
            return SyncPlan(mode="initial-new-only", search=SEARCH_NONE, baseline_uid=high_water)
        if account.initial_mode == "since":
            return SyncPlan(mode="initial-since", search=SEARCH_SINCE,
                            baseline_uid=high_water, since=account.initial_since)
        return SyncPlan(mode="initial-full", search=SEARCH_ALL, baseline_uid=high_water)

    if state.uidvalidity != status.uidvalidity:
        _log.warning("imap.uidvalidity_reset", account=account.name, folder=status.name,
                     stored=state.uidvalidity, server=status.uidvalidity)
        return SyncPlan(mode="uidvalidity-reset", search=SEARCH_ALL,
                        baseline_uid=high_water, uidvalidity_reset=True)

    return SyncPlan(mode="incremental", search=SEARCH_FROM_UID,
                    baseline_uid=state.last_uid, start_uid=state.last_uid + 1)


def contiguous_last_uid(floor: int, attempted: Iterable[int], succeeded: set[int]) -> int:
    """How far the cursor may move: the last UID of the unbroken run of successes.

    A failure stops the cursor at the message before it. The consequences are deliberate and
    worth naming: nothing is ever lost, the re-fetch costs nothing (dedup), and a message
    that fails *permanently* stalls its folder — visibly, as a non-zero error counter every
    tick and a cursor that stops moving, with the offending UID in the log. That is strictly
    better than the alternative of stepping over it, which drops mail while reporting success.
    The runbook documents the manual `UPDATE mail_sync_state` escape hatch for that case.
    """
    last = floor
    for uid in sorted(attempted):
        if uid not in succeeded:
            break
        last = max(last, uid)
    return last


async def get_folder_state(
    conn: asyncpg.Connection, account: str, folder: str
) -> Optional[FolderSyncState]:
    row = await conn.fetchrow(
        "SELECT account, folder, uidvalidity, last_uid, last_sync_ts "
        "FROM mail_sync_state WHERE account = $1 AND folder = $2",
        account, folder,
    )
    if row is None:
        return None
    return FolderSyncState(
        account=row["account"],
        folder=row["folder"],
        uidvalidity=int(row["uidvalidity"]),
        last_uid=int(row["last_uid"]),
        last_sync_ts=row["last_sync_ts"],
    )


async def upsert_folder_state(
    conn: asyncpg.Connection, account: str, folder: str, uidvalidity: int, last_uid: int
) -> None:
    """Persist the cursor and stamp `last_sync_ts`.

    Called on every tick including empty ones: `last_sync_ts` is how one answers "is this
    folder being polled at all", which stays true when nothing has arrived for a week.
    """
    await conn.execute(
        """
        INSERT INTO mail_sync_state (account, folder, uidvalidity, last_uid, last_sync_ts)
        VALUES ($1, $2, $3, $4, now())
        ON CONFLICT (account, folder) DO UPDATE
        SET uidvalidity  = EXCLUDED.uidvalidity,
            last_uid     = EXCLUDED.last_uid,
            last_sync_ts = EXCLUDED.last_sync_ts
        """,
        account, folder, uidvalidity, last_uid,
    )
    _log.info("sync_state.saved", account=account, folder=folder,
              uidvalidity=uidvalidity, last_uid=last_uid)


async def fetch_all_state(conn: asyncpg.Connection) -> list[FolderSyncState]:
    """Every cursor, for the runbook's inspection queries and the job's summary line."""
    rows = await conn.fetch(
        "SELECT account, folder, uidvalidity, last_uid, last_sync_ts "
        "FROM mail_sync_state ORDER BY account, folder"
    )
    return [
        FolderSyncState(
            account=r["account"], folder=r["folder"], uidvalidity=int(r["uidvalidity"]),
            last_uid=int(r["last_uid"]), last_sync_ts=r["last_sync_ts"],
        )
        for r in rows
    ]