Sync caption changes to Meili incrementally

This commit is contained in:
Sleeps
2026-10-07 14:24:37 +01:00
parent fb4f827cd9
commit 4ed8b386e7
+28 -5
View File
@@ -37,7 +37,7 @@ from .maintenance import (
from .meili import Meili from .meili import Meili
from .routing import ExplicitContentRouter, apply_context_routing from .routing import ExplicitContentRouter, apply_context_routing
from .search_service import SearchService from .search_service import SearchService
from .sync import rebuild_meili from .sync import CaptionMeiliSync, rebuild_meili
def format_ts(ms: int) -> str: def format_ts(ms: int) -> str:
@@ -425,7 +425,8 @@ def _route_rows(
""" """
UPDATE frames UPDATE frames
SET caption_route = ?, caption_route_reason = ?, route_score = ?, SET caption_route = ?, caption_route_reason = ?, route_score = ?,
route_model = ?, route_policy = ?, routed_at = CURRENT_TIMESTAMP route_model = ?, route_policy = ?, routed_at = CURRENT_TIMESTAMP,
meili_synced_at = NULL
WHERE id = ? WHERE id = ?
""", """,
(route, reason, score, route_model, policy, frame_id), (route, reason, score, route_model, policy, frame_id),
@@ -504,6 +505,7 @@ def _caption_queue(
total_frames: int | None = None, total_frames: int | None = None,
frame_source: FrameSource | None = None, frame_source: FrameSource | None = None,
before_frame: Callable[[], None] | None = None, before_frame: Callable[[], None] | None = None,
after_saved: Callable[[str], None] | None = None,
) -> tuple[int, int]: ) -> tuple[int, int]:
if not routed_rows: if not routed_rows:
return 0, 0 return 0, 0
@@ -583,13 +585,16 @@ def _caption_queue(
""" """
UPDATE frames UPDATE frames
SET caption = ?, caption_model = ?, SET caption = ?, caption_model = ?,
captioned_at = CURRENT_TIMESTAMP captioned_at = CURRENT_TIMESTAMP,
meili_synced_at = NULL
WHERE id = ? WHERE id = ?
""", """,
(caption, model, row["id"]), (caption, model, row["id"]),
) )
completed += 1 completed += 1
if after_saved is not None:
after_saved(str(row["id"]))
if progress is not None: if progress is not None:
progress( progress(
{ {
@@ -680,6 +685,18 @@ def cmd_caption(args: argparse.Namespace) -> None:
print(f"Specialist prompt: {settings.caption_specialist_prompt_file}") print(f"Specialist prompt: {settings.caption_specialist_prompt_file}")
print(f"Ollama: {settings.ollama_url}") print(f"Ollama: {settings.ollama_url}")
caption_sync = None
if args.sync:
caption_sync = CaptionMeiliSync(
[],
meili=Meili(
settings.meili_url,
settings.meili_master_key,
settings.meili_index,
),
batch_size=settings.caption_meili_batch_size,
)
if general_rows: if general_rows:
_unload_ollama_model(settings.caption_specialist_model) _unload_ollama_model(settings.caption_specialist_model)
general_done, general_failed = _caption_queue( general_done, general_failed = _caption_queue(
@@ -688,6 +705,7 @@ def cmd_caption(args: argparse.Namespace) -> None:
settings.caption_general_model, settings.caption_general_model,
general_prompt, general_prompt,
total_frames=len(general_rows) + len(specialist_rows), total_frames=len(general_rows) + len(specialist_rows),
after_saved=caption_sync.add if caption_sync is not None else None,
) )
if specialist_rows: if specialist_rows:
@@ -699,14 +717,19 @@ def cmd_caption(args: argparse.Namespace) -> None:
specialist_prompt, specialist_prompt,
completed_offset=general_done, completed_offset=general_done,
total_frames=len(general_rows) + len(specialist_rows), total_frames=len(general_rows) + len(specialist_rows),
after_saved=caption_sync.add if caption_sync is not None else None,
) )
completed = general_done + specialist_done completed = general_done + specialist_done
failed = general_failed + specialist_failed + missing failed = general_failed + specialist_failed + missing
print(f"\nCaptioning complete: {completed} saved, {failed} failed/skipped.") print(f"\nCaptioning complete: {completed} saved, {failed} failed/skipped.")
if args.sync: if args.sync and caption_sync is not None:
cmd_sync_meili(args) caption_sync.flush()
print(
f"Incremental Meilisearch caption updates complete: "
f"{caption_sync.updated_total} document(s) queued."
)
elif completed: elif completed:
print("Next: python -m scene_search.cli sync-meili") print("Next: python -m scene_search.cli sync-meili")