homelab-codex-ws/jobs/gmail-header-backfill/tests/test_backfill.py

"""Unit tests for the gmail header backfill job — no DB, no external services."""
from __future__ import annotations

import json
from datetime import datetime, timezone

import pytest

from gmail_header_backfill.backfill import (
    _decode_jsonb,
    _has_headers,
    fetch_batch,
    parse_headers,
    run,
)


def _eml(headers: dict, body: str = "body") -> bytes:
    lines = [f"{k}: {v}" for k, v in headers.items()]
    return ("\r\n".join(lines) + "\r\n\r\n" + body).encode("utf-8")


class TestParseHeadersBasics:
    def test_shape_has_all_expected_keys(self):
        raw = _eml({
            "From": "alice@example.com",
            "To": "bob@example.com",
            "Subject": "Hello",
            "Date": "Tue, 10 Jun 2025 12:00:00 +0000",
        })
        headers = parse_headers(raw)
        assert headers["type"] == "headers"
        assert {"from", "to", "cc", "delivered_to", "subject", "date_raw"} <= headers.keys()

    def test_from_parsed_to_name_and_address(self):
        raw = _eml({"From": "Alice Example <alice@example.com>", "Date": "Tue, 10 Jun 2025 12:00:00 +0000"})
        headers = parse_headers(raw)
        assert headers["from"] == {"name": "Alice Example", "address": "alice@example.com"}

    def test_from_without_display_name(self):
        raw = _eml({"From": "alice@example.com", "Date": "Tue, 10 Jun 2025 12:00:00 +0000"})
        headers = parse_headers(raw)
        assert headers["from"] == {"name": None, "address": "alice@example.com"}

    def test_missing_from_is_none(self):
        raw = _eml({"To": "bob@example.com", "Date": "Tue, 10 Jun 2025 12:00:00 +0000"})
        headers = parse_headers(raw)
        assert headers["from"] is None

    def test_subject_present(self):
        raw = _eml({"From": "a@b.com", "Subject": "Twoja polisa", "Date": "Tue, 10 Jun 2025 12:00:00 +0000"})
        assert parse_headers(raw)["subject"] == "Twoja polisa"

    def test_subject_missing_is_none(self):
        raw = _eml({"From": "a@b.com", "Date": "Tue, 10 Jun 2025 12:00:00 +0000"})
        assert parse_headers(raw)["subject"] is None

    def test_empty_to_and_cc_yield_empty_lists(self):
        raw = _eml({"From": "a@b.com", "Date": "Tue, 10 Jun 2025 12:00:00 +0000"})
        headers = parse_headers(raw)
        assert headers["to"] == []
        assert headers["cc"] == []
        assert headers["delivered_to"] == []


class TestParseHeadersMultiTo:
    def test_multiple_to_addresses_across_one_header(self):
        raw = _eml({
            "From": "a@b.com",
            "To": "bob@example.com, Carol Jones <carol@example.com>",
            "Date": "Tue, 10 Jun 2025 12:00:00 +0000",
        })
        headers = parse_headers(raw)
        assert headers["to"] == [
            {"name": None, "address": "bob@example.com"},
            {"name": "Carol Jones", "address": "carol@example.com"},
        ]

    def test_quoted_display_name_with_comma_not_split(self):
        raw = _eml({
            "From": '"Kowalski, Jan" <jan@example.com>',
            "Date": "Tue, 10 Jun 2025 12:00:00 +0000",
        })
        headers = parse_headers(raw)
        assert headers["from"] == {"name": "Kowalski, Jan", "address": "jan@example.com"}


class TestParseHeadersMultiDeliveredTo:
    def test_multiple_delivered_to_headers_all_kept(self):
        lines = (
            "From: a@b.com\r\n"
            "Delivered-To: oskar+alias@gmail.com\r\n"
            "Delivered-To: oskar@gmail.com\r\n"
            "Date: Tue, 10 Jun 2025 12:00:00 +0000\r\n"
            "\r\n"
            "body"
        )
        headers = parse_headers(lines.encode())
        assert headers["delivered_to"] == ["oskar+alias@gmail.com", "oskar@gmail.com"]

    def test_single_delivered_to(self):
        raw = _eml({
            "From": "a@b.com",
            "Delivered-To": "oskar@gmail.com",
            "Date": "Tue, 10 Jun 2025 12:00:00 +0000",
        })
        assert parse_headers(raw)["delivered_to"] == ["oskar@gmail.com"]


class TestParseHeadersRfc2047:
    def test_decodes_encoded_word_subject(self):
        raw = _eml({
            "From": "a@b.com",
            "Subject": "=?UTF-8?Q?Twoja_polisa?=",
            "Date": "Tue, 10 Jun 2025 12:00:00 +0000",
        })
        assert parse_headers(raw)["subject"] == "Twoja polisa"

    def test_decodes_encoded_word_display_name(self):
        raw = _eml({
            "From": "=?UTF-8?B?V2FydGE=?= <no-reply@warta.pl>",
            "Date": "Tue, 10 Jun 2025 12:00:00 +0000",
        })
        headers = parse_headers(raw)
        assert headers["from"] == {"name": "Warta", "address": "no-reply@warta.pl"}

    def test_decodes_encoded_word_with_polish_chars(self):
        raw = _eml({
            "From": "a@b.com",
            "To": "=?UTF-8?Q?Oskar_K=C4=85pa=C5=82a?= <oskar@gmail.com>",
            "Date": "Tue, 10 Jun 2025 12:00:00 +0000",
        })
        headers = parse_headers(raw)
        assert headers["to"] == [{"name": "Oskar Kąpała", "address": "oskar@gmail.com"}]

    def test_malformed_encoded_word_does_not_raise(self):
        raw = _eml({
            "From": "a@b.com",
            "Subject": "=?UTF-8?B?not-valid-base64!!!?=",
            "Date": "Tue, 10 Jun 2025 12:00:00 +0000",
        })
        headers = parse_headers(raw)  # must not raise
        assert isinstance(headers["subject"], str)


class TestParseHeadersMultipleFrom:
    def test_multiple_from_headers_uses_first_and_logs(self, caplog):
        raw = (
            b"From: alice@example.com\r\n"
            b"From: bob@example.com\r\n"
            b"Date: Tue, 10 Jun 2025 12:00:00 +0000\r\n"
            b"\r\nbody"
        )
        headers = parse_headers(raw)
        assert headers["from"] == {"name": None, "address": "alice@example.com"}


class TestParseHeadersDateRaw:
    def test_date_raw_preserves_literal_original_text(self):
        # policy.default's DateHeader reformats (e.g. corrects weekday, zero-pads day) —
        # date_raw must preserve the byte-for-byte original text instead (plan §4.1).
        raw = _eml({"From": "a@b.com", "Date": "Mon, 9 Jun 2026 12:34:56 +0200"})
        assert parse_headers(raw)["date_raw"] == "Mon, 9 Jun 2026 12:34:56 +0200"

    def test_date_raw_none_when_missing(self):
        raw = _eml({"From": "a@b.com"})
        assert parse_headers(raw)["date_raw"] is None

    def test_date_raw_preserved_even_when_unparseable(self):
        raw = _eml({"From": "a@b.com", "Date": "not-a-date-at-all"})
        assert parse_headers(raw)["date_raw"] == "not-a-date-at-all"


class TestDecodeJsonb:
    def test_passes_through_object(self):
        assert _decode_jsonb([{"type": "attachment"}]) == [{"type": "attachment"}]

    def test_decodes_string(self):
        assert _decode_jsonb('[{"type": "attachment"}]') == [{"type": "attachment"}]

    def test_none_stays_none(self):
        assert _decode_jsonb(None) is None


class TestHasHeaders:
    def test_true_when_present(self):
        assert _has_headers([{"type": "attachment"}, {"type": "headers"}]) is True

    def test_false_when_absent(self):
        assert _has_headers([{"type": "attachment"}]) is False

    def test_false_on_empty_list(self):
        assert _has_headers([]) is False


class _FakeConn:
    def __init__(self, rows):
        self._rows = rows
        self.executemany_calls: list[tuple[str, list]] = []

    async def fetch(self, query, *params):
        return self._rows

    async def executemany(self, query, rows):
        self.executemany_calls.append((query, list(rows)))

    async def close(self):
        pass


def _row(envelope_id, raw_ref, entities):
    return {"id": envelope_id, "raw_ref": raw_ref, "entities": json.dumps(entities)}


def _attachment_entity():
    return {"type": "attachment", "filename": "x.pdf", "content_type": "application/pdf",
            "size": 100, "sha256": "abc"}


class TestRun:
    def _setup_archive(self, tmp_path, raw_ref, raw_bytes):
        archive_root = tmp_path / "archive"
        eml_path = archive_root / raw_ref
        eml_path.parent.mkdir(parents=True, exist_ok=True)
        eml_path.write_bytes(raw_bytes)
        return archive_root

    def _patch_connect(self, monkeypatch, conn):
        async def _fake_connect(dsn):
            return conn
        monkeypatch.setattr("gmail_header_backfill.backfill.asyncpg.connect", _fake_connect)

    async def test_dry_run_does_not_call_executemany(self, tmp_path, monkeypatch):
        raw = _eml({"From": "a@b.com", "Subject": "Hi", "Date": "Tue, 10 Jun 2025 12:00:00 +0000"})
        archive_root = self._setup_archive(tmp_path, "gmail/2025/06/m1.eml", raw)
        conn = _FakeConn([_row("env1", "gmail/2025/06/m1.eml", [_attachment_entity()])])
        self._patch_connect(monkeypatch, conn)

        stats = await run(dsn="postgresql://fake", archive_root=archive_root, apply=False)

        assert stats["scanned"] == 1
        assert stats["updated"] == 1
        assert conn.executemany_calls == []

    async def test_apply_calls_executemany_with_headers_entity(self, tmp_path, monkeypatch):
        raw = _eml({"From": "a@b.com", "Subject": "Hi", "Date": "Tue, 10 Jun 2025 12:00:00 +0000"})
        archive_root = self._setup_archive(tmp_path, "gmail/2025/06/m1.eml", raw)
        conn = _FakeConn([_row("env1", "gmail/2025/06/m1.eml", [_attachment_entity()])])
        self._patch_connect(monkeypatch, conn)

        stats = await run(dsn="postgresql://fake", archive_root=archive_root, apply=True)

        assert stats["updated"] == 1
        assert len(conn.executemany_calls) == 1
        query, rows = conn.executemany_calls[0]
        assert "NOT EXISTS" in query
        assert rows[0][0] == "env1"
        patch = json.loads(rows[0][1])
        assert patch == [{
            "type": "headers",
            "from": {"name": None, "address": "a@b.com"},
            "to": [],
            "cc": [],
            "delivered_to": [],
            "subject": "Hi",
            "date_raw": "Tue, 10 Jun 2025 12:00:00 +0000",
        }]

    async def test_existing_entities_are_not_touched_client_side(self, tmp_path, monkeypatch):
        # The job never rewrites the existing manifest — it only appends a new patch row
        # for the UPDATE (entities || $2::jsonb happens in SQL, not here).
        raw = _eml({"From": "a@b.com", "Date": "Tue, 10 Jun 2025 12:00:00 +0000"})
        archive_root = self._setup_archive(tmp_path, "gmail/2025/06/m1.eml", raw)
        original_entities = [_attachment_entity()]
        conn = _FakeConn([_row("env1", "gmail/2025/06/m1.eml", original_entities)])
        self._patch_connect(monkeypatch, conn)

        await run(dsn="postgresql://fake", archive_root=archive_root, apply=True)

        query, rows = conn.executemany_calls[0]
        patch = json.loads(rows[0][1])
        assert len(patch) == 1
        assert patch[0]["type"] == "headers"

    async def test_idempotent_skips_rows_already_backfilled(self, tmp_path, monkeypatch):
        archive_root = tmp_path / "archive"
        archive_root.mkdir()
        entities = [_attachment_entity(), {"type": "headers", "from": None, "to": [],
                                            "cc": [], "delivered_to": [], "subject": None,
                                            "date_raw": None}]
        conn = _FakeConn([_row("env1", "gmail/2025/06/m1.eml", entities)])
        self._patch_connect(monkeypatch, conn)

        stats = await run(dsn="postgresql://fake", archive_root=archive_root, apply=True)

        assert stats["already_has_headers"] == 1
        assert stats["updated"] == 0
        assert conn.executemany_calls == []

    async def test_missing_eml_file_counted_as_read_error(self, tmp_path, monkeypatch):
        archive_root = tmp_path / "archive"
        archive_root.mkdir()
        conn = _FakeConn([_row("env1", "gmail/2025/06/missing.eml", [_attachment_entity()])])
        self._patch_connect(monkeypatch, conn)

        stats = await run(dsn="postgresql://fake", archive_root=archive_root, apply=True)

        assert stats["read_errors"] == 1
        assert stats["updated"] == 0

    async def test_limit_and_offset_are_passed_to_query(self, tmp_path, monkeypatch):
        captured = {}

        class _FakeConnCapturing(_FakeConn):
            async def fetch(self, query, *params):
                captured["params"] = params
                return self._rows

        conn = _FakeConnCapturing([])
        self._patch_connect(monkeypatch, conn)
        archive_root = tmp_path / "archive"
        archive_root.mkdir()

        await run(dsn="postgresql://fake", archive_root=archive_root, limit=50, offset=200, apply=False)

        assert captured["params"] == (50, 200)

    async def test_batch_flush_multiple_rows_in_one_executemany(self, tmp_path, monkeypatch):
        raw = _eml({"From": "a@b.com", "Date": "Tue, 10 Jun 2025 12:00:00 +0000"})
        archive_root = self._setup_archive(tmp_path, "gmail/2025/06/m1.eml", raw)
        rows = [_row(f"env{i}", "gmail/2025/06/m1.eml", [_attachment_entity()]) for i in range(3)]
        conn = _FakeConn(rows)
        self._patch_connect(monkeypatch, conn)

        stats = await run(dsn="postgresql://fake", archive_root=archive_root, apply=True)

        assert stats["updated"] == 3
        assert len(conn.executemany_calls) == 1  # under UPDATE_BATCH_SIZE, one flush at end
        _, flushed_rows = conn.executemany_calls[0]
        assert len(flushed_rows) == 3


class TestFetchBatchQueryShape:
    async def test_selects_only_gmail_source_ordered_by_id(self, monkeypatch):
        captured = {}

        class _FakeConnCapturing(_FakeConn):
            async def fetch(self, query, *params):
                captured["query"] = query
                captured["params"] = params
                return []

        conn = _FakeConnCapturing([])
        await fetch_batch(conn, limit=10, offset=5)

        assert "source = 'gmail'" in captured["query"]
        assert "ORDER BY id" in captured["query"]
        assert captured["params"] == (10, 5)