Replace final caption rebuild with incremental Meili updates

This commit is contained in:
Sleeps
2026-10-07 14:25:25 +01:00
parent 4ed8b386e7
commit 6a24c53092
+33 -24
View File
@@ -17,7 +17,7 @@ from .config import settings
from .database import connect, init_db
from .frame_source import FrameSource
from .search_service import SearchService
from .sync import rebuild_meili
from .sync import CaptionMeiliSync
class CaptionJobStopped(Exception):
@@ -50,6 +50,9 @@ class CaptionJobManager:
"paused": False,
"pause_requested": False,
"stop_requested": False,
"meili_documents_updated": 0,
"meili_sync_pending": 0,
"meili_sync_error": None,
}
@property
@@ -177,6 +180,9 @@ class CaptionJobManager:
"paused": False,
"pause_requested": False,
"stop_requested": False,
"meili_documents_updated": 0,
"meili_sync_pending": 0,
"meili_sync_error": None,
}
self._thread = threading.Thread(
@@ -198,6 +204,12 @@ class CaptionJobManager:
limit=None,
library_ids=library_ids,
)
caption_sync = CaptionMeiliSync(
library_ids,
meili=self.search_service.meili,
batch_size=settings.caption_meili_batch_size,
progress=self._progress,
)
self._progress(
{
@@ -233,9 +245,11 @@ class CaptionJobManager:
"already_captioned": already_captioned,
"frames_selected": len(rows),
"frames_total": len(rows),
"preparing": None,
"preparing": "Catching up saved captions in search",
}
)
caption_sync.catch_up()
self._progress({"preparing": None})
if not rows:
self._progress(
@@ -248,9 +262,10 @@ class CaptionJobManager:
)
return
self._run(args, rows)
self._run(args, rows, caption_sync)
except CaptionJobStopped:
self._set_gpu_busy(False)
caption_sync.flush()
self._pause_requested.clear()
self._stop_requested.clear()
self._resume_event.set()
@@ -268,6 +283,7 @@ class CaptionJobManager:
)
except Exception as exc:
self._set_gpu_busy(False)
caption_sync.flush()
self._progress(
{
"running": False,
@@ -301,7 +317,12 @@ class CaptionJobManager:
with self._state_lock:
self._gpu_busy = value
def _run(self, args: SimpleNamespace, rows: list) -> None:
def _run(
self,
args: SimpleNamespace,
rows: list,
caption_sync: CaptionMeiliSync,
) -> None:
missing = 0
try:
self._progress({"stage": "routing"})
@@ -330,6 +351,8 @@ class CaptionJobManager:
finally:
self._set_gpu_busy(False)
caption_sync.catch_up()
if not routed:
self._progress(
{
@@ -401,6 +424,7 @@ class CaptionJobManager:
total_frames=frames_total,
frame_source=frame_source,
before_frame=lambda: self._wait_if_paused("captioning"),
after_saved=caption_sync.add,
)
if specialist_rows:
@@ -425,6 +449,7 @@ class CaptionJobManager:
total_frames=frames_total,
frame_source=frame_source,
before_frame=lambda: self._wait_if_paused("captioning"),
after_saved=caption_sync.add,
)
completed = general_done + specialist_done
@@ -432,25 +457,8 @@ class CaptionJobManager:
self._wait_if_paused("captioning")
self._progress(
{
"stage": "syncing",
"frames_completed": completed,
"failed": failed,
}
)
def sync_progress(update: dict) -> None:
self._progress(
{
**update,
"stage": "syncing",
"frames_completed": completed,
"failed": failed,
}
)
documents = rebuild_meili(progress=sync_progress)
caption_sync.flush()
caption_sync.catch_up()
self._progress(
{
@@ -459,7 +467,8 @@ class CaptionJobManager:
"finished_at": datetime.now(timezone.utc).isoformat(),
"frames_completed": completed,
"failed": failed,
"documents_processed": documents,
"documents_processed": caption_sync.updated_total,
"meili_documents_updated": caption_sync.updated_total,
}
)
except CaptionJobStopped: