Sync caption changes to Meili incrementally

This commit is contained in:
Sleeps
2026-10-07 14:24:37 +01:00
parent fb4f827cd9
commit 4ed8b386e7
+28 -5
View File
@@ -37,7 +37,7 @@ from .maintenance import (
from .meili import Meili
from .routing import ExplicitContentRouter, apply_context_routing
from .search_service import SearchService
from .sync import rebuild_meili
from .sync import CaptionMeiliSync, rebuild_meili
def format_ts(ms: int) -> str:
@@ -425,7 +425,8 @@ def _route_rows(
"""
UPDATE frames
SET caption_route = ?, caption_route_reason = ?, route_score = ?,
route_model = ?, route_policy = ?, routed_at = CURRENT_TIMESTAMP
route_model = ?, route_policy = ?, routed_at = CURRENT_TIMESTAMP,
meili_synced_at = NULL
WHERE id = ?
""",
(route, reason, score, route_model, policy, frame_id),
@@ -504,6 +505,7 @@ def _caption_queue(
total_frames: int | None = None,
frame_source: FrameSource | None = None,
before_frame: Callable[[], None] | None = None,
after_saved: Callable[[str], None] | None = None,
) -> tuple[int, int]:
if not routed_rows:
return 0, 0
@@ -583,13 +585,16 @@ def _caption_queue(
"""
UPDATE frames
SET caption = ?, caption_model = ?,
captioned_at = CURRENT_TIMESTAMP
captioned_at = CURRENT_TIMESTAMP,
meili_synced_at = NULL
WHERE id = ?
""",
(caption, model, row["id"]),
)
completed += 1
if after_saved is not None:
after_saved(str(row["id"]))
if progress is not None:
progress(
{
@@ -680,6 +685,18 @@ def cmd_caption(args: argparse.Namespace) -> None:
print(f"Specialist prompt: {settings.caption_specialist_prompt_file}")
print(f"Ollama: {settings.ollama_url}")
caption_sync = None
if args.sync:
caption_sync = CaptionMeiliSync(
[],
meili=Meili(
settings.meili_url,
settings.meili_master_key,
settings.meili_index,
),
batch_size=settings.caption_meili_batch_size,
)
if general_rows:
_unload_ollama_model(settings.caption_specialist_model)
general_done, general_failed = _caption_queue(
@@ -688,6 +705,7 @@ def cmd_caption(args: argparse.Namespace) -> None:
settings.caption_general_model,
general_prompt,
total_frames=len(general_rows) + len(specialist_rows),
after_saved=caption_sync.add if caption_sync is not None else None,
)
if specialist_rows:
@@ -699,14 +717,19 @@ def cmd_caption(args: argparse.Namespace) -> None:
specialist_prompt,
completed_offset=general_done,
total_frames=len(general_rows) + len(specialist_rows),
after_saved=caption_sync.add if caption_sync is not None else None,
)
completed = general_done + specialist_done
failed = general_failed + specialist_failed + missing
print(f"\nCaptioning complete: {completed} saved, {failed} failed/skipped.")
if args.sync:
cmd_sync_meili(args)
if args.sync and caption_sync is not None:
caption_sync.flush()
print(
f"Incremental Meilisearch caption updates complete: "
f"{caption_sync.updated_total} document(s) queued."
)
elif completed:
print("Next: python -m scene_search.cli sync-meili")