f234280b3a
- Usuwa Dockerfile i docker-compose.yml; job odpalany lokalnie na PIHA (pip install -e)
- _parse_attachments: manifest MIME → entities[]{type,filename,content_type,size,sha256};
bajty zostają w .eml, wyciąganie/OCR = faza 2
- Batch inserty co 500 wpisów (executemany + ON CONFLICT DO NOTHING); idempotentny
na skipped przez _eml_ref (mirrors archive._UNSAFE); pełna wznawialność
- --limit N: ucina pętlę po N wiadomościach do testów na próbce
- epoch_fallback: licznik + WARNING gdy Date nieparsowalne/brak
- Nowe stats: msgs_with_attachments, total_attachments, total_attachment_bytes
- DSN w docstringu: localhost:5433/kb i piha:5433/kb; usunięto solaria:5433
- 9 nowych testów (24 razem), wszystkie zielone
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
284 lines
9.8 KiB
Python
284 lines
9.8 KiB
Python
"""Unit tests for Gmail bulk importer — no DB, no external services required."""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import mailbox
|
|
from datetime import datetime, timezone
|
|
from email import encoders
|
|
from email.mime.base import MIMEBase
|
|
from email.mime.multipart import MIMEMultipart
|
|
from email.mime.text import MIMEText
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from gmail_bulk_import.importer import (
|
|
_message_id,
|
|
_parse_attachments,
|
|
_parse_date,
|
|
run_import,
|
|
)
|
|
|
|
|
|
def _make_msg(headers: dict) -> mailbox.mboxMessage:
|
|
msg = mailbox.mboxMessage()
|
|
for k, v in headers.items():
|
|
msg[k] = v
|
|
return msg
|
|
|
|
|
|
def _make_mbox(path: Path, msgs: list[mailbox.mboxMessage]) -> None:
|
|
mbox = mailbox.mbox(str(path), create=True)
|
|
for msg in msgs:
|
|
mbox.add(msg)
|
|
mbox.flush()
|
|
mbox.close()
|
|
|
|
|
|
def _msg_with_id(mid: str = "<test-unique@example.com>") -> mailbox.mboxMessage:
|
|
return _make_msg({
|
|
"From": "alice@example.com",
|
|
"Message-ID": mid,
|
|
"Date": "Tue, 10 Jun 2025 12:00:00 +0000",
|
|
"Subject": "Hello",
|
|
})
|
|
|
|
|
|
def _mime_with_attachments(
|
|
mid: str = "<attach-test@example.com>",
|
|
payloads: list | None = None,
|
|
) -> mailbox.mboxMessage:
|
|
"""Build a multipart mboxMessage with binary attachments.
|
|
|
|
payloads: list of (filename, content_type, bytes). Defaults to PDF + PNG.
|
|
"""
|
|
if payloads is None:
|
|
payloads = [
|
|
("report.pdf", "application/pdf", b"PDF content here"),
|
|
("photo.png", "image/png", b"PNG bytes"),
|
|
]
|
|
mime = MIMEMultipart()
|
|
mime["From"] = "alice@example.com"
|
|
mime["To"] = "bob@example.com"
|
|
mime["Subject"] = "Files attached"
|
|
mime["Message-ID"] = mid
|
|
mime["Date"] = "Tue, 10 Jun 2025 12:00:00 +0000"
|
|
mime.attach(MIMEText("See attachments", "plain"))
|
|
for filename, ctype, data in payloads:
|
|
maintype, subtype = ctype.split("/", 1)
|
|
att = MIMEBase(maintype, subtype)
|
|
att.set_payload(data)
|
|
encoders.encode_base64(att)
|
|
att.add_header("Content-Disposition", "attachment", filename=filename)
|
|
mime.attach(att)
|
|
return mailbox.mboxMessage(mime)
|
|
|
|
|
|
class TestMessageId:
|
|
def test_uses_message_id_header(self):
|
|
msg = _make_msg({"Message-ID": "<foo-123@example.com>"})
|
|
assert _message_id(msg) == "foo-123@example.com"
|
|
|
|
def test_strips_angle_brackets(self):
|
|
msg = _make_msg({"Message-ID": " <bar@baz.com> "})
|
|
assert _message_id(msg) == "bar@baz.com"
|
|
|
|
def test_fallback_to_sha256_when_missing(self):
|
|
msg = _make_msg({"From": "a@b.com", "Subject": "no id"})
|
|
result = _message_id(msg)
|
|
assert result.startswith("sha256-")
|
|
assert len(result) == len("sha256-") + 32
|
|
|
|
def test_sha256_is_stable(self):
|
|
msg = _make_msg({"From": "a@b.com", "Subject": "stable"})
|
|
assert _message_id(msg) == _message_id(msg)
|
|
|
|
|
|
class TestParseDate:
|
|
def test_parses_rfc2822(self):
|
|
msg = _make_msg({"Date": "Tue, 10 Jun 2025 12:00:00 +0000"})
|
|
ts = _parse_date(msg)
|
|
assert ts == datetime(2025, 6, 10, 12, 0, 0, tzinfo=timezone.utc)
|
|
|
|
def test_result_is_utc_aware(self):
|
|
msg = _make_msg({"Date": "Tue, 10 Jun 2025 12:00:00 +0000"})
|
|
assert _parse_date(msg).tzinfo is not None
|
|
|
|
def test_falls_back_to_epoch_on_bad_date(self):
|
|
msg = _make_msg({"Date": "not-a-date"})
|
|
assert _parse_date(msg) == datetime(1970, 1, 1, tzinfo=timezone.utc)
|
|
|
|
def test_falls_back_to_epoch_when_missing(self):
|
|
msg = _make_msg({"From": "a@b.com"})
|
|
assert _parse_date(msg) == datetime(1970, 1, 1, tzinfo=timezone.utc)
|
|
|
|
def test_converts_offset_to_utc(self):
|
|
msg = _make_msg({"Date": "Tue, 10 Jun 2025 14:00:00 +0200"})
|
|
ts = _parse_date(msg)
|
|
assert ts == datetime(2025, 6, 10, 12, 0, 0, tzinfo=timezone.utc)
|
|
|
|
|
|
class TestParseAttachments:
|
|
def test_plain_text_message_has_no_attachments(self):
|
|
assert _parse_attachments(_msg_with_id()) == []
|
|
|
|
def test_two_attachments_manifest(self):
|
|
atts = _parse_attachments(_mime_with_attachments())
|
|
assert len(atts) == 2
|
|
|
|
pdf = next(a for a in atts if a["filename"] == "report.pdf")
|
|
assert pdf["type"] == "attachment"
|
|
assert pdf["content_type"] == "application/pdf"
|
|
assert pdf["size"] == len(b"PDF content here")
|
|
assert pdf["sha256"] == hashlib.sha256(b"PDF content here").hexdigest()
|
|
|
|
png = next(a for a in atts if a["filename"] == "photo.png")
|
|
assert png["content_type"] == "image/png"
|
|
assert png["size"] == len(b"PNG bytes")
|
|
assert png["sha256"] == hashlib.sha256(b"PNG bytes").hexdigest()
|
|
|
|
def test_attachment_descriptors_have_required_keys(self):
|
|
for att in _parse_attachments(_mime_with_attachments()):
|
|
assert {"type", "filename", "content_type", "size", "sha256"} <= att.keys()
|
|
|
|
|
|
class TestRunImport:
|
|
async def test_dry_run_counts_without_writing(self, tmp_path):
|
|
mbox_path = tmp_path / "mail.mbox"
|
|
archive = tmp_path / "archive"
|
|
_make_mbox(mbox_path, [_msg_with_id("<a@e.com>"), _msg_with_id("<b@e.com>")])
|
|
|
|
stats = await run_import(mbox_path, archive, dry_run=True)
|
|
|
|
assert stats["processed"] == 2
|
|
assert stats["imported"] == 2
|
|
assert not archive.exists()
|
|
|
|
async def test_imports_eml_to_archive(self, tmp_path):
|
|
mbox_path = tmp_path / "mail.mbox"
|
|
archive = tmp_path / "archive"
|
|
_make_mbox(mbox_path, [_msg_with_id()])
|
|
|
|
stats = await run_import(mbox_path, archive)
|
|
|
|
assert stats["imported"] == 1
|
|
assert stats["skipped"] == 0
|
|
assert stats["errors"] == 0
|
|
assert len(list(archive.rglob("*.eml"))) == 1
|
|
|
|
async def test_eml_stored_under_gmail_source(self, tmp_path):
|
|
mbox_path = tmp_path / "mail.mbox"
|
|
archive = tmp_path / "archive"
|
|
_make_mbox(mbox_path, [_msg_with_id()])
|
|
|
|
await run_import(mbox_path, archive)
|
|
|
|
eml_files = list(archive.rglob("*.eml"))
|
|
assert eml_files[0].parts[len(archive.parts)] == "gmail"
|
|
|
|
async def test_idempotent_second_run_skips(self, tmp_path):
|
|
mbox_path = tmp_path / "mail.mbox"
|
|
archive = tmp_path / "archive"
|
|
_make_mbox(mbox_path, [_msg_with_id()])
|
|
|
|
await run_import(mbox_path, archive)
|
|
stats2 = await run_import(mbox_path, archive)
|
|
|
|
assert stats2["imported"] == 0
|
|
assert stats2["skipped"] == 1
|
|
assert stats2["errors"] == 0
|
|
|
|
async def test_two_distinct_messages_both_imported(self, tmp_path):
|
|
mbox_path = tmp_path / "mail.mbox"
|
|
archive = tmp_path / "archive"
|
|
_make_mbox(mbox_path, [_msg_with_id("<msg1@e.com>"), _msg_with_id("<msg2@e.com>")])
|
|
|
|
stats = await run_import(mbox_path, archive)
|
|
|
|
assert stats["imported"] == 2
|
|
assert stats["errors"] == 0
|
|
assert len(list(archive.rglob("*.eml"))) == 2
|
|
|
|
async def test_archive_only_mode_no_dsn(self, tmp_path):
|
|
mbox_path = tmp_path / "mail.mbox"
|
|
archive = tmp_path / "archive"
|
|
_make_mbox(mbox_path, [_msg_with_id()])
|
|
|
|
stats = await run_import(mbox_path, archive, dsn=None)
|
|
|
|
assert stats["imported"] == 1
|
|
assert list(archive.rglob("*.eml"))
|
|
|
|
async def test_limit_truncates_processing(self, tmp_path):
|
|
mbox_path = tmp_path / "mail.mbox"
|
|
archive = tmp_path / "archive"
|
|
_make_mbox(mbox_path, [_msg_with_id(f"<m{i}@e.com>") for i in range(5)])
|
|
|
|
stats = await run_import(mbox_path, archive, limit=3)
|
|
|
|
assert stats["processed"] == 3
|
|
assert stats["imported"] == 3
|
|
|
|
async def test_limit_with_dry_run(self, tmp_path):
|
|
mbox_path = tmp_path / "mail.mbox"
|
|
archive = tmp_path / "archive"
|
|
_make_mbox(mbox_path, [_msg_with_id(f"<m{i}@e.com>") for i in range(10)])
|
|
|
|
stats = await run_import(mbox_path, archive, dry_run=True, limit=4)
|
|
|
|
assert stats["processed"] == 4
|
|
assert stats["imported"] == 4
|
|
assert not archive.exists()
|
|
|
|
async def test_epoch_fallback_counted(self, tmp_path):
|
|
mbox_path = tmp_path / "mail.mbox"
|
|
archive = tmp_path / "archive"
|
|
_make_mbox(mbox_path, [
|
|
_make_msg({"Message-ID": "<bad@e.com>", "Date": "not-a-date"}),
|
|
_make_msg({"Message-ID": "<missing@e.com>"}),
|
|
_msg_with_id("<good@e.com>"),
|
|
])
|
|
|
|
stats = await run_import(mbox_path, archive)
|
|
|
|
assert stats["epoch_fallback"] == 2
|
|
assert stats["processed"] == 3
|
|
|
|
async def test_attachment_stats_aggregated(self, tmp_path):
|
|
mbox_path = tmp_path / "mail.mbox"
|
|
archive = tmp_path / "archive"
|
|
_make_mbox(mbox_path, [
|
|
_msg_with_id("<plain@e.com>"),
|
|
_mime_with_attachments("<att@e.com>"),
|
|
])
|
|
|
|
stats = await run_import(mbox_path, archive)
|
|
|
|
assert stats["msgs_with_attachments"] == 1
|
|
assert stats["total_attachments"] == 2
|
|
assert stats["total_attachment_bytes"] == (
|
|
len(b"PDF content here") + len(b"PNG bytes")
|
|
)
|
|
|
|
async def test_batch_flush_multiple_messages(self, tmp_path):
|
|
"""BATCH_SIZE=500; 3 messages stay in pending until end-of-loop flush."""
|
|
mbox_path = tmp_path / "mail.mbox"
|
|
archive = tmp_path / "archive"
|
|
_make_mbox(mbox_path, [_msg_with_id(f"<m{i}@e.com>") for i in range(3)])
|
|
|
|
stats = await run_import(mbox_path, archive)
|
|
|
|
assert stats["imported"] == 3
|
|
assert stats["errors"] == 0
|
|
assert len(list(archive.rglob("*.eml"))) == 3
|
|
|
|
async def test_attachment_entities_in_stats(self, tmp_path):
|
|
mbox_path = tmp_path / "mail.mbox"
|
|
archive = tmp_path / "archive"
|
|
_make_mbox(mbox_path, [_mime_with_attachments()])
|
|
|
|
stats = await run_import(mbox_path, archive)
|
|
|
|
assert stats["imported"] == 1
|
|
assert stats["total_attachments"] == 2
|