homelab-codex-ws/jobs/mail-imap-sync/tests/conftest.py
oskar f056b08574 feat(mail-imap-sync): job przyrostowki + wpiecie w tor body-ingest
Nowy job jobs/mail-imap-sync — jedyny wlasciwy nowy kod przyrostowki
(recon kb/audits/mail-sync-2026-08-06.md §6 poz. 1). Jeden tick, per konto
i folder: EXAMINE -> plan -> UID SEARCH -> FETCH BODY.PEEK[] -> save_eml ->
insert_envelope(entities=[headers, attachment...]) -> UPDATE mail_sync_state.

Wlasciwosci, ktore latwo zgubic po cichu:
- Job NIE embeduje i NIE chunkuje (recon §3.2). Pobieranie jest sieciowe i chodzi
  na PIHA 24/7; chunk+embed potrzebuje Ollamy na SOLARII, wylaczanej ~16 h/dobe.
  Spoiwem jest kolejka wynikajaca z danych: koperta bez chunkow JEST elementem
  kolejki, ktora drenuje mail-body-ingest --only-unchunked.
- Koperta dostaje entities[type=headers] juz przy INSERCIE. Bez tego kazdy nowy
  mail mialby prefiks "Temat: (brak tematu) | Od: ?" — bez bledu, tylko z gorszym
  retrievalem (recon §2.5 i).
- Kolizja Message-ID miedzy kontami jest liczona (envelopes_conflict_other_source),
  nie ukryta w zwyklych duplikatach (recon §2.4).
- Kursor przesuwa sie tylko po nieprzerwanym ciagu w pelni trwalych wiadomosci.
  Bledna wiadomosc jest ponawiana (dedup czyni to darmowym), nie przeskakiwana;
  trwale zatrucie widac jako niezerowy licznik bledow i stojacy kursor.
- Poswiadczenia wylacznie ze srodowiska — brak flagi --password/--user (Decyzja (c)).
- Tryb --measure (STATUS MESSAGES/UIDNEXT/UIDVALIDITY): pomiar, na ktorym operator
  oprze decyzje o historii Fastmaila (Decyzja (e), celowo nieodgadywana).
- Metryki .prom per konto; last_success_timestamp przenoszony przez nieudany run.

Wpiecie w istniejacy tor (recon §6 poz. 4-6):
- mail_body_ingest.fetch_envelopes: --sources (domyslnie gmail,fastmail) +
  --only-unchunked.
- fetch_existing_chunk_keys zawezone do zbioru roboczego — bez tego kazdy tick
  czyta wszystkie 389 012 kluczy chunkow (~26 MB, ~1,0 s) na nodzie z 2,4 GB
  available (recon §2.5 iv).
- DEFAULT_SUMMARYLESS_SOURCES += "fastmail" (Decyzja (g)) w TYM SAMYM commicie,
  ktory wprowadza zrodlo: bez tego koperty fastmail zaindeksowalyby sie poprawnie
  i byly niewidoczne w /search, bez zadnego bledu.

Testy: 80 dla nowego joba (mock IMAP na poziomie imaplib, wiec testowane jest
prawdziwe parsowanie protokolu) — nowe wiadomosci, uniewaznienie UIDVALIDITY,
dedup, wznowienie po przerwaniu, izolacja kont, dry-run bez sieci, metryki;
+ 47 mail-body-ingest/kb-retrieval. Smoke: --help, blad konfiguracji -> exit 2.
Zero polaczen z zywymi kontami — pierwszy zywy sync robi operator wg runbooka.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-06 15:30:57 +02:00

200 lines
8 KiB
Python

"""Fakes shared by the job's tests — no network, no DB, no live mailbox.
The IMAP fake sits at the `imaplib` level rather than at `ImapClient`, so every test runs the
adapter's REAL protocol parsing (LIST attributes, EXAMINE untagged responses, `n:*` range
trailers, `(prefix, literal)` FETCH tuples). The DB fake models the two tables the job touches
closely enough that dedup, cross-account collisions and cursor persistence are exercised as
behaviour rather than asserted as SQL strings.
"""
from __future__ import annotations
from datetime import datetime, timezone
class FakeIMAP:
"""Multi-folder `imaplib.IMAP4_SSL` stand-in.
`folders` maps mailbox name -> {uid: raw bytes}. `uidvalidity` may be a per-folder dict so
a test can flip one folder's value to simulate a server-side renumbering.
"""
def __init__(self, folders, *, uidvalidity=42, list_attrs=None, login_error=None,
fetch_errors=(), vanished=()):
self.folders = folders
self._uidvalidity = uidvalidity
self.list_attrs = list_attrs or {}
self.login_error = login_error
self.fetch_errors = set(fetch_errors)
self.vanished = set(vanished)
self.commands: list[tuple] = []
self.selected = None
self.logged_out = False
# -- helpers -----------------------------------------------------------------
def uidvalidity_of(self, folder: str) -> int:
if isinstance(self._uidvalidity, dict):
return self._uidvalidity.get(folder, 42)
return self._uidvalidity
def uidnext_of(self, folder: str) -> int:
uids = self.folders.get(folder) or {}
return (max(uids) + 1) if uids else 1
def fetch_commands(self):
return [c for c in self.commands if c[:2] == ("UID", "FETCH")]
# -- imaplib surface ---------------------------------------------------------
def login(self, user, password):
self.commands.append(("LOGIN", user))
if self.login_error:
raise RuntimeError(self.login_error)
return ("OK", [b"ok"])
def logout(self):
self.logged_out = True
return ("BYE", [b"bye"])
def list(self, directory='""', pattern="*"):
self.commands.append(("LIST",))
lines = []
for name in self.folders:
attrs = " ".join(self.list_attrs.get(name, ["\\HasNoChildren"]))
lines.append(f'({attrs}) "/" "{name}"'.encode())
return ("OK", lines)
def select(self, mailbox, readonly=False):
name = mailbox.strip('"')
self.commands.append(("SELECT", name, readonly))
if name not in self.folders:
return ("NO", [b"no such mailbox"])
self.selected = name
return ("OK", [str(len(self.folders[name])).encode()])
def status(self, mailbox, names):
name = mailbox.strip('"')
self.commands.append(("STATUS", name, names))
line = (
f'"{name}" (MESSAGES {len(self.folders.get(name) or {})} '
f"UIDNEXT {self.uidnext_of(name)} UIDVALIDITY {self.uidvalidity_of(name)})"
).encode()
return ("OK", [line])
def response(self, key):
if self.selected is None:
return (key, [None])
if key == "UIDVALIDITY":
return (key, [str(self.uidvalidity_of(self.selected)).encode()])
if key == "UIDNEXT":
return (key, [str(self.uidnext_of(self.selected)).encode()])
return (key, [None])
def uid(self, command, *args):
self.commands.append(("UID", command) + args)
uids = sorted(self.folders.get(self.selected) or {})
if command == "SEARCH":
criteria = args[1:]
if criteria and criteria[0] == "UID":
start = int(criteria[1].split(":")[0])
hits = [u for u in uids if u >= start]
if not hits and uids:
hits = [uids[-1]] # real servers do this; the client must filter it out
elif criteria and criteria[0] == "SINCE":
hits = [u for u in uids if u >= getattr(self, "since_uid_floor", 0)]
else:
hits = uids
return ("OK", [" ".join(str(u) for u in hits).encode()])
if command == "FETCH":
uid = int(args[0])
if uid in self.fetch_errors:
raise RuntimeError(f"fetch of {uid} exploded")
if uid in self.vanished:
return ("OK", [None])
raw = (self.folders.get(self.selected) or {}).get(uid)
if raw is None:
return ("OK", [None])
return ("OK", [(f"1 (UID {uid} BODY[] {{{len(raw)}}}".encode(), raw), b")"])
raise AssertionError(f"unexpected UID command {command}")
class FakeConn:
"""In-memory `envelope` + `mail_sync_state`, dispatched on SQL keywords."""
def __init__(self, envelopes=None, state=None, insert_errors=()):
self.envelopes = dict(envelopes or {}) # id -> {"source", "ts", "entities"}
self.state = dict(state or {}) # (account, folder) -> (uidvalidity, last_uid)
self.insert_errors = set(insert_errors) # envelope ids whose INSERT raises
self.queries: list[tuple] = []
self.closed = False
async def fetchrow(self, query, *params):
self.queries.append((query, params))
if "FROM mail_sync_state" in query:
row = self.state.get((params[0], params[1]))
if row is None:
return None
return {"account": params[0], "folder": params[1], "uidvalidity": row[0],
"last_uid": row[1], "last_sync_ts": None}
raise AssertionError(f"unexpected fetchrow: {query}")
async def fetchval(self, query, *params):
self.queries.append((query, params))
if "SELECT source FROM envelope" in query:
row = self.envelopes.get(params[0])
return row["source"] if row else None
raise AssertionError(f"unexpected fetchval: {query}")
async def fetch(self, query, *params):
self.queries.append((query, params))
if "max(ts)" in query:
newest: dict = {}
for row in self.envelopes.values():
if row["source"] in params[0]:
current = newest.get(row["source"])
if current is None or row["ts"] > current:
newest[row["source"]] = row["ts"]
return [{"source": s, "newest": ts} for s, ts in newest.items()]
if "FROM mail_sync_state" in query:
return [
{"account": a, "folder": f, "uidvalidity": v[0], "last_uid": v[1],
"last_sync_ts": None}
for (a, f), v in sorted(self.state.items())
]
raise AssertionError(f"unexpected fetch: {query}")
async def execute(self, query, *params):
self.queries.append((query, params))
if "INSERT INTO envelope" in query:
envelope_id = params[0]
if envelope_id in self.insert_errors:
raise RuntimeError(f"insert of {envelope_id} exploded")
if envelope_id in self.envelopes:
return "INSERT 0 0"
self.envelopes[envelope_id] = {"source": params[1], "ts": params[2],
"entities": params[5]}
return "INSERT 0 1"
if "INSERT INTO mail_sync_state" in query:
self.state[(params[0], params[1])] = (params[2], params[3])
return "INSERT 0 1"
raise AssertionError(f"unexpected execute: {query}")
async def close(self):
self.closed = True
def eml(message_id: str, *, subject="Temat", sender="Alice <a@example.com>",
date="Mon, 15 Jun 2026 14:00:00 +0200", body="tresc wiadomosci") -> bytes:
return (
f"Message-ID: <{message_id}>\r\n"
f"From: {sender}\r\n"
f"To: Oskar <o@example.com>\r\n"
f"Subject: {subject}\r\n"
f"Date: {date}\r\n"
f"Content-Type: text/plain; charset=utf-8\r\n"
f"\r\n{body}\r\n"
).encode("utf-8")
JUNE_15 = datetime(2026, 6, 15, 12, 0, tzinfo=timezone.utc)