From 4ed8b386e7e37e2b36b48aa518e1cf39e1ecbcab Mon Sep 17 00:00:00 2001 From: Sleeps Date: Wed, 7 Oct 2026 14:24:37 +0100 Subject: [PATCH] Sync caption changes to Meili incrementally --- scene_search/cli.py | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/scene_search/cli.py b/scene_search/cli.py index 92e6ac6..eddd00d 100644 --- a/scene_search/cli.py +++ b/scene_search/cli.py @@ -37,7 +37,7 @@ from .maintenance import ( from .meili import Meili from .routing import ExplicitContentRouter, apply_context_routing from .search_service import SearchService -from .sync import rebuild_meili +from .sync import CaptionMeiliSync, rebuild_meili def format_ts(ms: int) -> str: @@ -425,7 +425,8 @@ def _route_rows( """ UPDATE frames SET caption_route = ?, caption_route_reason = ?, route_score = ?, - route_model = ?, route_policy = ?, routed_at = CURRENT_TIMESTAMP + route_model = ?, route_policy = ?, routed_at = CURRENT_TIMESTAMP, + meili_synced_at = NULL WHERE id = ? """, (route, reason, score, route_model, policy, frame_id), @@ -504,6 +505,7 @@ def _caption_queue( total_frames: int | None = None, frame_source: FrameSource | None = None, before_frame: Callable[[], None] | None = None, + after_saved: Callable[[str], None] | None = None, ) -> tuple[int, int]: if not routed_rows: return 0, 0 @@ -583,13 +585,16 @@ def _caption_queue( """ UPDATE frames SET caption = ?, caption_model = ?, - captioned_at = CURRENT_TIMESTAMP + captioned_at = CURRENT_TIMESTAMP, + meili_synced_at = NULL WHERE id = ? """, (caption, model, row["id"]), ) completed += 1 + if after_saved is not None: + after_saved(str(row["id"])) if progress is not None: progress( { @@ -680,6 +685,18 @@ def cmd_caption(args: argparse.Namespace) -> None: print(f"Specialist prompt: {settings.caption_specialist_prompt_file}") print(f"Ollama: {settings.ollama_url}") + caption_sync = None + if args.sync: + caption_sync = CaptionMeiliSync( + [], + meili=Meili( + settings.meili_url, + settings.meili_master_key, + settings.meili_index, + ), + batch_size=settings.caption_meili_batch_size, + ) + if general_rows: _unload_ollama_model(settings.caption_specialist_model) general_done, general_failed = _caption_queue( @@ -688,6 +705,7 @@ def cmd_caption(args: argparse.Namespace) -> None: settings.caption_general_model, general_prompt, total_frames=len(general_rows) + len(specialist_rows), + after_saved=caption_sync.add if caption_sync is not None else None, ) if specialist_rows: @@ -699,14 +717,19 @@ def cmd_caption(args: argparse.Namespace) -> None: specialist_prompt, completed_offset=general_done, total_frames=len(general_rows) + len(specialist_rows), + after_saved=caption_sync.add if caption_sync is not None else None, ) completed = general_done + specialist_done failed = general_failed + specialist_failed + missing print(f"\nCaptioning complete: {completed} saved, {failed} failed/skipped.") - if args.sync: - cmd_sync_meili(args) + if args.sync and caption_sync is not None: + caption_sync.flush() + print( + f"Incremental Meilisearch caption updates complete: " + f"{caption_sync.updated_total} document(s) queued." + ) elif completed: print("Next: python -m scene_search.cli sync-meili")