Nowy job jobs/mail-imap-sync — jedyny wlasciwy nowy kod przyrostowki (recon kb/audits/mail-sync-2026-08-06.md §6 poz. 1). Jeden tick, per konto i folder: EXAMINE -> plan -> UID SEARCH -> FETCH BODY.PEEK[] -> save_eml -> insert_envelope(entities=[headers, attachment...]) -> UPDATE mail_sync_state. Wlasciwosci, ktore latwo zgubic po cichu: - Job NIE embeduje i NIE chunkuje (recon §3.2). Pobieranie jest sieciowe i chodzi na PIHA 24/7; chunk+embed potrzebuje Ollamy na SOLARII, wylaczanej ~16 h/dobe. Spoiwem jest kolejka wynikajaca z danych: koperta bez chunkow JEST elementem kolejki, ktora drenuje mail-body-ingest --only-unchunked. - Koperta dostaje entities[type=headers] juz przy INSERCIE. Bez tego kazdy nowy mail mialby prefiks "Temat: (brak tematu) | Od: ?" — bez bledu, tylko z gorszym retrievalem (recon §2.5 i). - Kolizja Message-ID miedzy kontami jest liczona (envelopes_conflict_other_source), nie ukryta w zwyklych duplikatach (recon §2.4). - Kursor przesuwa sie tylko po nieprzerwanym ciagu w pelni trwalych wiadomosci. Bledna wiadomosc jest ponawiana (dedup czyni to darmowym), nie przeskakiwana; trwale zatrucie widac jako niezerowy licznik bledow i stojacy kursor. - Poswiadczenia wylacznie ze srodowiska — brak flagi --password/--user (Decyzja (c)). - Tryb --measure (STATUS MESSAGES/UIDNEXT/UIDVALIDITY): pomiar, na ktorym operator oprze decyzje o historii Fastmaila (Decyzja (e), celowo nieodgadywana). - Metryki .prom per konto; last_success_timestamp przenoszony przez nieudany run. Wpiecie w istniejacy tor (recon §6 poz. 4-6): - mail_body_ingest.fetch_envelopes: --sources (domyslnie gmail,fastmail) + --only-unchunked. - fetch_existing_chunk_keys zawezone do zbioru roboczego — bez tego kazdy tick czyta wszystkie 389 012 kluczy chunkow (~26 MB, ~1,0 s) na nodzie z 2,4 GB available (recon §2.5 iv). - DEFAULT_SUMMARYLESS_SOURCES += "fastmail" (Decyzja (g)) w TYM SAMYM commicie, ktory wprowadza zrodlo: bez tego koperty fastmail zaindeksowalyby sie poprawnie i byly niewidoczne w /search, bez zadnego bledu. Testy: 80 dla nowego joba (mock IMAP na poziomie imaplib, wiec testowane jest prawdziwe parsowanie protokolu) — nowe wiadomosci, uniewaznienie UIDVALIDITY, dedup, wznowienie po przerwaniu, izolacja kont, dry-run bez sieci, metryki; + 47 mail-body-ingest/kb-retrieval. Smoke: --help, blad konfiguracji -> exit 2. Zero polaczen z zywymi kontami — pierwszy zywy sync robi operator wg runbooka. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
200 lines
8 KiB
Python
200 lines
8 KiB
Python
"""Fakes shared by the job's tests — no network, no DB, no live mailbox.
|
|
|
|
The IMAP fake sits at the `imaplib` level rather than at `ImapClient`, so every test runs the
|
|
adapter's REAL protocol parsing (LIST attributes, EXAMINE untagged responses, `n:*` range
|
|
trailers, `(prefix, literal)` FETCH tuples). The DB fake models the two tables the job touches
|
|
closely enough that dedup, cross-account collisions and cursor persistence are exercised as
|
|
behaviour rather than asserted as SQL strings.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime, timezone
|
|
|
|
|
|
class FakeIMAP:
|
|
"""Multi-folder `imaplib.IMAP4_SSL` stand-in.
|
|
|
|
`folders` maps mailbox name -> {uid: raw bytes}. `uidvalidity` may be a per-folder dict so
|
|
a test can flip one folder's value to simulate a server-side renumbering.
|
|
"""
|
|
|
|
def __init__(self, folders, *, uidvalidity=42, list_attrs=None, login_error=None,
|
|
fetch_errors=(), vanished=()):
|
|
self.folders = folders
|
|
self._uidvalidity = uidvalidity
|
|
self.list_attrs = list_attrs or {}
|
|
self.login_error = login_error
|
|
self.fetch_errors = set(fetch_errors)
|
|
self.vanished = set(vanished)
|
|
self.commands: list[tuple] = []
|
|
self.selected = None
|
|
self.logged_out = False
|
|
|
|
# -- helpers -----------------------------------------------------------------
|
|
|
|
def uidvalidity_of(self, folder: str) -> int:
|
|
if isinstance(self._uidvalidity, dict):
|
|
return self._uidvalidity.get(folder, 42)
|
|
return self._uidvalidity
|
|
|
|
def uidnext_of(self, folder: str) -> int:
|
|
uids = self.folders.get(folder) or {}
|
|
return (max(uids) + 1) if uids else 1
|
|
|
|
def fetch_commands(self):
|
|
return [c for c in self.commands if c[:2] == ("UID", "FETCH")]
|
|
|
|
# -- imaplib surface ---------------------------------------------------------
|
|
|
|
def login(self, user, password):
|
|
self.commands.append(("LOGIN", user))
|
|
if self.login_error:
|
|
raise RuntimeError(self.login_error)
|
|
return ("OK", [b"ok"])
|
|
|
|
def logout(self):
|
|
self.logged_out = True
|
|
return ("BYE", [b"bye"])
|
|
|
|
def list(self, directory='""', pattern="*"):
|
|
self.commands.append(("LIST",))
|
|
lines = []
|
|
for name in self.folders:
|
|
attrs = " ".join(self.list_attrs.get(name, ["\\HasNoChildren"]))
|
|
lines.append(f'({attrs}) "/" "{name}"'.encode())
|
|
return ("OK", lines)
|
|
|
|
def select(self, mailbox, readonly=False):
|
|
name = mailbox.strip('"')
|
|
self.commands.append(("SELECT", name, readonly))
|
|
if name not in self.folders:
|
|
return ("NO", [b"no such mailbox"])
|
|
self.selected = name
|
|
return ("OK", [str(len(self.folders[name])).encode()])
|
|
|
|
def status(self, mailbox, names):
|
|
name = mailbox.strip('"')
|
|
self.commands.append(("STATUS", name, names))
|
|
line = (
|
|
f'"{name}" (MESSAGES {len(self.folders.get(name) or {})} '
|
|
f"UIDNEXT {self.uidnext_of(name)} UIDVALIDITY {self.uidvalidity_of(name)})"
|
|
).encode()
|
|
return ("OK", [line])
|
|
|
|
def response(self, key):
|
|
if self.selected is None:
|
|
return (key, [None])
|
|
if key == "UIDVALIDITY":
|
|
return (key, [str(self.uidvalidity_of(self.selected)).encode()])
|
|
if key == "UIDNEXT":
|
|
return (key, [str(self.uidnext_of(self.selected)).encode()])
|
|
return (key, [None])
|
|
|
|
def uid(self, command, *args):
|
|
self.commands.append(("UID", command) + args)
|
|
uids = sorted(self.folders.get(self.selected) or {})
|
|
if command == "SEARCH":
|
|
criteria = args[1:]
|
|
if criteria and criteria[0] == "UID":
|
|
start = int(criteria[1].split(":")[0])
|
|
hits = [u for u in uids if u >= start]
|
|
if not hits and uids:
|
|
hits = [uids[-1]] # real servers do this; the client must filter it out
|
|
elif criteria and criteria[0] == "SINCE":
|
|
hits = [u for u in uids if u >= getattr(self, "since_uid_floor", 0)]
|
|
else:
|
|
hits = uids
|
|
return ("OK", [" ".join(str(u) for u in hits).encode()])
|
|
if command == "FETCH":
|
|
uid = int(args[0])
|
|
if uid in self.fetch_errors:
|
|
raise RuntimeError(f"fetch of {uid} exploded")
|
|
if uid in self.vanished:
|
|
return ("OK", [None])
|
|
raw = (self.folders.get(self.selected) or {}).get(uid)
|
|
if raw is None:
|
|
return ("OK", [None])
|
|
return ("OK", [(f"1 (UID {uid} BODY[] {{{len(raw)}}}".encode(), raw), b")"])
|
|
raise AssertionError(f"unexpected UID command {command}")
|
|
|
|
|
|
class FakeConn:
|
|
"""In-memory `envelope` + `mail_sync_state`, dispatched on SQL keywords."""
|
|
|
|
def __init__(self, envelopes=None, state=None, insert_errors=()):
|
|
self.envelopes = dict(envelopes or {}) # id -> {"source", "ts", "entities"}
|
|
self.state = dict(state or {}) # (account, folder) -> (uidvalidity, last_uid)
|
|
self.insert_errors = set(insert_errors) # envelope ids whose INSERT raises
|
|
self.queries: list[tuple] = []
|
|
self.closed = False
|
|
|
|
async def fetchrow(self, query, *params):
|
|
self.queries.append((query, params))
|
|
if "FROM mail_sync_state" in query:
|
|
row = self.state.get((params[0], params[1]))
|
|
if row is None:
|
|
return None
|
|
return {"account": params[0], "folder": params[1], "uidvalidity": row[0],
|
|
"last_uid": row[1], "last_sync_ts": None}
|
|
raise AssertionError(f"unexpected fetchrow: {query}")
|
|
|
|
async def fetchval(self, query, *params):
|
|
self.queries.append((query, params))
|
|
if "SELECT source FROM envelope" in query:
|
|
row = self.envelopes.get(params[0])
|
|
return row["source"] if row else None
|
|
raise AssertionError(f"unexpected fetchval: {query}")
|
|
|
|
async def fetch(self, query, *params):
|
|
self.queries.append((query, params))
|
|
if "max(ts)" in query:
|
|
newest: dict = {}
|
|
for row in self.envelopes.values():
|
|
if row["source"] in params[0]:
|
|
current = newest.get(row["source"])
|
|
if current is None or row["ts"] > current:
|
|
newest[row["source"]] = row["ts"]
|
|
return [{"source": s, "newest": ts} for s, ts in newest.items()]
|
|
if "FROM mail_sync_state" in query:
|
|
return [
|
|
{"account": a, "folder": f, "uidvalidity": v[0], "last_uid": v[1],
|
|
"last_sync_ts": None}
|
|
for (a, f), v in sorted(self.state.items())
|
|
]
|
|
raise AssertionError(f"unexpected fetch: {query}")
|
|
|
|
async def execute(self, query, *params):
|
|
self.queries.append((query, params))
|
|
if "INSERT INTO envelope" in query:
|
|
envelope_id = params[0]
|
|
if envelope_id in self.insert_errors:
|
|
raise RuntimeError(f"insert of {envelope_id} exploded")
|
|
if envelope_id in self.envelopes:
|
|
return "INSERT 0 0"
|
|
self.envelopes[envelope_id] = {"source": params[1], "ts": params[2],
|
|
"entities": params[5]}
|
|
return "INSERT 0 1"
|
|
if "INSERT INTO mail_sync_state" in query:
|
|
self.state[(params[0], params[1])] = (params[2], params[3])
|
|
return "INSERT 0 1"
|
|
raise AssertionError(f"unexpected execute: {query}")
|
|
|
|
async def close(self):
|
|
self.closed = True
|
|
|
|
|
|
def eml(message_id: str, *, subject="Temat", sender="Alice <a@example.com>",
|
|
date="Mon, 15 Jun 2026 14:00:00 +0200", body="tresc wiadomosci") -> bytes:
|
|
return (
|
|
f"Message-ID: <{message_id}>\r\n"
|
|
f"From: {sender}\r\n"
|
|
f"To: Oskar <o@example.com>\r\n"
|
|
f"Subject: {subject}\r\n"
|
|
f"Date: {date}\r\n"
|
|
f"Content-Type: text/plain; charset=utf-8\r\n"
|
|
f"\r\n{body}\r\n"
|
|
).encode("utf-8")
|
|
|
|
|
|
JUNE_15 = datetime(2026, 6, 15, 12, 0, tzinfo=timezone.utc)
|