"""Per-folder IMAP sync cursor: the `mail_sync_state` table (migration 005) plus the pure logic that decides what a tick fetches and how far the cursor is allowed to move. The rules encoded here are the whole correctness story of the poller, so they live as pure functions with their own tests rather than inline in the job's loop: * **UIDVALIDITY governs everything.** When the server changes it, every stored UID becomes meaningless. The only safe response is to sweep the folder and lean on Message-ID dedup — a poller that trusts `last_uid` across a UIDVALIDITY change loses mail with no error anywhere (recon `kb/audits/mail-sync-2026-08-06.md` §2.2). * **The cursor moves only over messages that are fully durable**, archive file and envelope row both. It moves over a CONTIGUOUS prefix of successes, so a message that failed is re-fetched next tick instead of being stepped over silently. * **A first tick is a policy decision, not a default.** `new-only` starts the corpus from now; `since` closes a known gap (gmail's corpus stops at 2026-06-19); `full` pulls a mailbox's whole history. Which one is right depends on a measurement the runbook takes. """ from __future__ import annotations from dataclasses import dataclass from datetime import date, datetime from typing import Iterable, Optional import asyncpg import structlog from .imap import FolderStatus, ImapAccount _log = structlog.get_logger(__name__) # What a plan tells the caller to run against the server. SEARCH_NONE = "none" # fetch nothing this tick; just record the high-water mark SEARCH_ALL = "all" # UID SEARCH ALL — full sweep, dedup does the rest SEARCH_SINCE = "since" # UID SEARCH SINCE SEARCH_FROM_UID = "from-uid" # UID SEARCH UID :* @dataclass(frozen=True) class FolderSyncState: """One row of `mail_sync_state`.""" account: str folder: str uidvalidity: int last_uid: int last_sync_ts: Optional[datetime] = None @dataclass(frozen=True) class SyncPlan: """What this tick will do to one folder. `baseline_uid` is the cursor to persist when the plan turns up NO messages — never a floor to combine with partial results, because on a full sweep it equals `uidnext - 1` and combining it with a partially-failed batch would step over everything that failed. """ mode: str search: str baseline_uid: int start_uid: Optional[int] = None since: Optional[date] = None uidvalidity_reset: bool = False def plan_folder_sync( account: ImapAccount, status: FolderStatus, state: Optional[FolderSyncState], ) -> SyncPlan: """Decide this tick's work for one folder from the stored cursor and the server's state.""" high_water = max(status.uidnext - 1, 0) if state is None: if account.initial_mode == "new-only": # Start the corpus here: record the high-water mark, fetch nothing. Everything # already in the mailbox stays out of KB unless the operator re-runs with a # different initial_mode after clearing the row. return SyncPlan(mode="initial-new-only", search=SEARCH_NONE, baseline_uid=high_water) if account.initial_mode == "since": return SyncPlan(mode="initial-since", search=SEARCH_SINCE, baseline_uid=high_water, since=account.initial_since) return SyncPlan(mode="initial-full", search=SEARCH_ALL, baseline_uid=high_water) if state.uidvalidity != status.uidvalidity: _log.warning("imap.uidvalidity_reset", account=account.name, folder=status.name, stored=state.uidvalidity, server=status.uidvalidity) return SyncPlan(mode="uidvalidity-reset", search=SEARCH_ALL, baseline_uid=high_water, uidvalidity_reset=True) return SyncPlan(mode="incremental", search=SEARCH_FROM_UID, baseline_uid=state.last_uid, start_uid=state.last_uid + 1) def contiguous_last_uid(floor: int, attempted: Iterable[int], succeeded: set[int]) -> int: """How far the cursor may move: the last UID of the unbroken run of successes. A failure stops the cursor at the message before it. The consequences are deliberate and worth naming: nothing is ever lost, the re-fetch costs nothing (dedup), and a message that fails *permanently* stalls its folder — visibly, as a non-zero error counter every tick and a cursor that stops moving, with the offending UID in the log. That is strictly better than the alternative of stepping over it, which drops mail while reporting success. The runbook documents the manual `UPDATE mail_sync_state` escape hatch for that case. """ last = floor for uid in sorted(attempted): if uid not in succeeded: break last = max(last, uid) return last async def get_folder_state( conn: asyncpg.Connection, account: str, folder: str ) -> Optional[FolderSyncState]: row = await conn.fetchrow( "SELECT account, folder, uidvalidity, last_uid, last_sync_ts " "FROM mail_sync_state WHERE account = $1 AND folder = $2", account, folder, ) if row is None: return None return FolderSyncState( account=row["account"], folder=row["folder"], uidvalidity=int(row["uidvalidity"]), last_uid=int(row["last_uid"]), last_sync_ts=row["last_sync_ts"], ) async def upsert_folder_state( conn: asyncpg.Connection, account: str, folder: str, uidvalidity: int, last_uid: int ) -> None: """Persist the cursor and stamp `last_sync_ts`. Called on every tick including empty ones: `last_sync_ts` is how one answers "is this folder being polled at all", which stays true when nothing has arrived for a week. """ await conn.execute( """ INSERT INTO mail_sync_state (account, folder, uidvalidity, last_uid, last_sync_ts) VALUES ($1, $2, $3, $4, now()) ON CONFLICT (account, folder) DO UPDATE SET uidvalidity = EXCLUDED.uidvalidity, last_uid = EXCLUDED.last_uid, last_sync_ts = EXCLUDED.last_sync_ts """, account, folder, uidvalidity, last_uid, ) _log.info("sync_state.saved", account=account, folder=folder, uidvalidity=uidvalidity, last_uid=last_uid) async def fetch_all_state(conn: asyncpg.Connection) -> list[FolderSyncState]: """Every cursor, for the runbook's inspection queries and the job's summary line.""" rows = await conn.fetch( "SELECT account, folder, uidvalidity, last_uid, last_sync_ts " "FROM mail_sync_state ORDER BY account, folder" ) return [ FolderSyncState( account=r["account"], folder=r["folder"], uidvalidity=int(r["uidvalidity"]), last_uid=int(r["last_uid"]), last_sync_ts=r["last_sync_ts"], ) for r in rows ]