diff --git a/docs/kb/modules/05-fallback-dedup-raport.md b/docs/kb/modules/05-fallback-dedup-raport.md index 175c2b2..a4ea3d7 100644 --- a/docs/kb/modules/05-fallback-dedup-raport.md +++ b/docs/kb/modules/05-fallback-dedup-raport.md @@ -1,7 +1,14 @@ # Raport dedup: fallback embed SOLARIA→PIHA — e7625cd (master) vs 3d4ee38 (task/kb-f4-fallback) -**Data**: 2026-07-30 · **Worktree**: `task/kb-fallback-dedup` · **Status**: recon zakończony, -czeka na decyzję operatora co do listy salvage (sekcja „Rekomendacja zbiorcza"). +**Data**: 2026-07-30 · **Worktree**: `task/kb-fallback-dedup` · **Status**: salvage **wykonany** +2026-07-30 — operator zatwierdził pełną listę S1–S4 (sekcja „Rekomendacja zbiorcza"); +zrealizowane na tym branchu: S1 (cherry-pick `--transport http` + README), S2 (session log +2026-07-27 z dopiskiem redakcyjnym), S3 (testy T1/T2/T3 w `test_embed_router.py`; T4 +pominięty zgodnie z raportem), S4 (wynik kalibracji w override + README ollama-piha). +Weryfikacja: pytest kb-query 42/42 PASS; `retrieval_eval.py --transport http` przepuszczony +end-to-end przeciwko stubowi `/search` (raport + werdykt bramki generują się poprawnie; +kryterium 4 na stubie nie przechodzi wyłącznie dlatego, że `mail_queries` w `queries.yaml` +mają `expected_envelope: null` — placeholdery, artefakt danych, nie kodu). Dwie równoległe sesje zaimplementowały ten sam krok planu (moduł 5 faza 4, §2 decyzja 2 / §5): diff --git a/hosts/piha/runtime/ollama-piha/docker-compose.override.yml b/hosts/piha/runtime/ollama-piha/docker-compose.override.yml index 99d6b97..33681c6 100644 --- a/hosts/piha/runtime/ollama-piha/docker-compose.override.yml +++ b/hosts/piha/runtime/ollama-piha/docker-compose.override.yml @@ -10,8 +10,11 @@ services: ollama-piha: # Plan §2 D2 starting value — the cgroup OOM killer restarts this container # on breach instead of the host OOM killer picking a victim (which could be - # Home Assistant). Confirm or trim after live calibration (plan §5 step 4: - # a few embeds + docker stats at a normal-load hour). + # Home Assistant). Confirmed by live calibration 2026-07-27 (plan §5 step 4, + # docs/sessions/2026-07-27-kb-f4-fallback.md §4): peak ~983 MiB during an + # embed burst at a normal-load hour, ~66 MiB idle — comfortably inside this + # ceiling, verdict GO. Same container config as measured (image, + # OLLAMA_KEEP_ALIVE=0, this limit), so the number carries over. mem_limit: 2560m # Deliberately no mem_reservation: the working set is a transient spike and # the idle daemon is ~100 MB — soft-reserving gigabytes would permanently diff --git a/services/kb-query/tests/test_embed_router.py b/services/kb-query/tests/test_embed_router.py index 5430913..60ebe18 100644 --- a/services/kb-query/tests/test_embed_router.py +++ b/services/kb-query/tests/test_embed_router.py @@ -3,6 +3,7 @@ real Ollama on either side. Fake backends are keyed by URL so one fake session c both legs; the clock is injected so the 30 s TTL is tested without sleeping.""" from __future__ import annotations +import asyncio import pathlib import sys @@ -18,12 +19,15 @@ FALLBACK = "http://192.168.31.5:11434" class _FakeResponse: - def __init__(self, payload=None, exc=None, status=200): + def __init__(self, payload=None, exc=None, status=200, delay=0.0): self._payload = payload self._exc = exc self.status = status # check_ollama_health reads resp.status directly + self._delay = delay async def __aenter__(self): + if self._delay: + await asyncio.sleep(self._delay) if self._exc is not None: raise self._exc return self @@ -43,11 +47,13 @@ class _FakeBackend: `embed_exc` makes /api/embeddings fail while /api/tags still answers (the plan's step-3b scenario: probe says up, the real embed then dies mid-query).""" - def __init__(self, models=("bge-m3:latest",), down=False, embed_exc=None, vector_value=0.01): + def __init__(self, models=("bge-m3:latest",), down=False, embed_exc=None, vector_value=0.01, + embed_delay=0.0): self.models = list(models) self.down = down self.embed_exc = embed_exc self.vector_value = vector_value + self.embed_delay = embed_delay # seconds the /api/embeddings answer hangs before serving self.tags_calls = 0 self.embed_calls = 0 @@ -76,7 +82,8 @@ class _FakeSession: return _FakeResponse(exc=aiohttp.ClientConnectionError("connection refused")) if backend.embed_exc is not None: return _FakeResponse(exc=backend.embed_exc) - return _FakeResponse({"embedding": [backend.vector_value] * 1024}) + return _FakeResponse({"embedding": [backend.vector_value] * 1024}, + delay=backend.embed_delay) class _FakeClock: @@ -148,6 +155,48 @@ class TestFailover: assert embedding[0] == 0.02 assert solaria.embed_calls == 1 + async def test_mid_embed_timeout_serves_same_query_from_fallback(self): + # The step-3b guarantee for the OTHER failure shape: SOLARIA hangs instead of + # refusing. This exercises the asyncio.wait_for path (a hang raises TimeoutError, + # not aiohttp.ClientError) -- the hard primary timeout must cut the hang off and + # the SAME request must still come back from the fallback. + solaria = _FakeBackend(embed_delay=0.2) + piha = _FakeBackend(vector_value=0.02) + session = _FakeSession({PRIMARY: solaria, FALLBACK: piha}) + router = _router(primary_embed_timeout_s=0.05) + embedding, backend = await router.embed(session, "q") + assert backend == "piha" + assert embedding[0] == 0.02 + assert solaria.embed_calls == 1 + + async def test_mid_embed_failure_opens_circuit_for_subsequent_requests(self): + # The one-shot flip must persist: after a mid-embed failure, requests inside the + # same TTL window go straight to the fallback -- no re-probe, no primary retry. + solaria = _FakeBackend(embed_exc=aiohttp.ClientConnectionError("died mid-embed")) + piha = _FakeBackend() + clock = _FakeClock() + router = _router(clock=clock) + session = _FakeSession({PRIMARY: solaria, FALLBACK: piha}) + await router.embed(session, "q1") # probe says up -> embed dies -> flip to down + tags_after_first = solaria.tags_calls + clock.now += 10 # still inside the TTL window the flip opened + _, backend = await router.embed(session, "q2") + assert backend == "piha" + assert solaria.embed_calls == 1 # primary never retried inside the window + assert solaria.tags_calls == tags_after_first # and never re-probed either + + async def test_fallback_embed_is_not_bounded_by_the_primary_timeout(self): + # Last-resort semantics: a Pi-5 CPU embed plus a cold model load is legitimately + # slow -- the fallback leg must NOT inherit the primary's hard timeout, or "slow + # but alive" would turn back into "dead". + solaria = _FakeBackend(down=True) + piha = _FakeBackend(embed_delay=0.2, vector_value=0.02) + session = _FakeSession({PRIMARY: solaria, FALLBACK: piha}) + router = _router(primary_embed_timeout_s=0.05) + embedding, backend = await router.embed(session, "q") + assert backend == "piha" + assert embedding[0] == 0.02 + async def test_down_verdict_is_cached_and_skips_primary_until_ttl_expires(self): solaria = _FakeBackend(down=True) piha = _FakeBackend() diff --git a/services/ollama-piha/README.md b/services/ollama-piha/README.md index 6336d93..6931f12 100644 --- a/services/ollama-piha/README.md +++ b/services/ollama-piha/README.md @@ -65,6 +65,21 @@ night-quiet) hour: run a few embeds as above while watching step 5: keep as default fallback / tune `mem_limit` / fall back to explicit 503 degradation. +**Calibration status: measured 2026-07-27 — verdict GO** (live PIHA under +normal load, 3 consecutive `/api/embeddings` calls after `ollama pull bge-m3`; +full protocol in `docs/sessions/2026-07-27-kb-f4-fallback.md` §4): + +- Latency: 5.25 s (cold start) / 4.41 s / 4.16 s — single seconds as expected, + no warm-up between calls by design (`OLLAMA_KEEP_ALIVE=0` releases the model + after every request, `ollama ps` shows nothing resident in between). +- RAM: idle ~66 MiB, burst peak ~983 MiB (`docker stats` sampled at 0.3 s) — + well inside the 2560m ceiling; host `available` never dropped below ~1.3 GiB. +- Kept as **default fallback** (no feature flag). The measurement was taken + against the same container configuration this repo deploys (image, + `OLLAMA_KEEP_ALIVE=0`, `mem_limit: 2560m`), so it carries over; only the + model storage differed (bind mount then, named volume now), which does not + affect RAM/latency. + ## Relation to kb-query kb-query's router (`services/kb-query/app/embed_router.py`) health-checks