From 6a24c53092bf2adca790008e9047e36129f6dd4c Mon Sep 17 00:00:00 2001 From: Sleeps Date: Wed, 7 Oct 2026 14:25:25 +0100 Subject: [PATCH] Replace final caption rebuild with incremental Meili updates --- scene_search/caption_jobs.py | 57 +++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/scene_search/caption_jobs.py b/scene_search/caption_jobs.py index f51b690..c0c4428 100644 --- a/scene_search/caption_jobs.py +++ b/scene_search/caption_jobs.py @@ -17,7 +17,7 @@ from .config import settings from .database import connect, init_db from .frame_source import FrameSource from .search_service import SearchService -from .sync import rebuild_meili +from .sync import CaptionMeiliSync class CaptionJobStopped(Exception): @@ -50,6 +50,9 @@ class CaptionJobManager: "paused": False, "pause_requested": False, "stop_requested": False, + "meili_documents_updated": 0, + "meili_sync_pending": 0, + "meili_sync_error": None, } @property @@ -177,6 +180,9 @@ class CaptionJobManager: "paused": False, "pause_requested": False, "stop_requested": False, + "meili_documents_updated": 0, + "meili_sync_pending": 0, + "meili_sync_error": None, } self._thread = threading.Thread( @@ -198,6 +204,12 @@ class CaptionJobManager: limit=None, library_ids=library_ids, ) + caption_sync = CaptionMeiliSync( + library_ids, + meili=self.search_service.meili, + batch_size=settings.caption_meili_batch_size, + progress=self._progress, + ) self._progress( { @@ -233,9 +245,11 @@ class CaptionJobManager: "already_captioned": already_captioned, "frames_selected": len(rows), "frames_total": len(rows), - "preparing": None, + "preparing": "Catching up saved captions in search", } ) + caption_sync.catch_up() + self._progress({"preparing": None}) if not rows: self._progress( @@ -248,9 +262,10 @@ class CaptionJobManager: ) return - self._run(args, rows) + self._run(args, rows, caption_sync) except CaptionJobStopped: self._set_gpu_busy(False) + caption_sync.flush() self._pause_requested.clear() self._stop_requested.clear() self._resume_event.set() @@ -268,6 +283,7 @@ class CaptionJobManager: ) except Exception as exc: self._set_gpu_busy(False) + caption_sync.flush() self._progress( { "running": False, @@ -301,7 +317,12 @@ class CaptionJobManager: with self._state_lock: self._gpu_busy = value - def _run(self, args: SimpleNamespace, rows: list) -> None: + def _run( + self, + args: SimpleNamespace, + rows: list, + caption_sync: CaptionMeiliSync, + ) -> None: missing = 0 try: self._progress({"stage": "routing"}) @@ -330,6 +351,8 @@ class CaptionJobManager: finally: self._set_gpu_busy(False) + caption_sync.catch_up() + if not routed: self._progress( { @@ -401,6 +424,7 @@ class CaptionJobManager: total_frames=frames_total, frame_source=frame_source, before_frame=lambda: self._wait_if_paused("captioning"), + after_saved=caption_sync.add, ) if specialist_rows: @@ -425,6 +449,7 @@ class CaptionJobManager: total_frames=frames_total, frame_source=frame_source, before_frame=lambda: self._wait_if_paused("captioning"), + after_saved=caption_sync.add, ) completed = general_done + specialist_done @@ -432,25 +457,8 @@ class CaptionJobManager: self._wait_if_paused("captioning") - self._progress( - { - "stage": "syncing", - "frames_completed": completed, - "failed": failed, - } - ) - - def sync_progress(update: dict) -> None: - self._progress( - { - **update, - "stage": "syncing", - "frames_completed": completed, - "failed": failed, - } - ) - - documents = rebuild_meili(progress=sync_progress) + caption_sync.flush() + caption_sync.catch_up() self._progress( { @@ -459,7 +467,8 @@ class CaptionJobManager: "finished_at": datetime.now(timezone.utc).isoformat(), "frames_completed": completed, "failed": failed, - "documents_processed": documents, + "documents_processed": caption_sync.updated_total, + "meili_documents_updated": caption_sync.updated_total, } ) except CaptionJobStopped: