mirror of
https://github.com/MrSleeps/jellyfin-scene-search.git
synced 2026-11-20 00:33:36 +00:00
Replace final caption rebuild with incremental Meili updates
This commit is contained in:
@@ -17,7 +17,7 @@ from .config import settings
|
|||||||
from .database import connect, init_db
|
from .database import connect, init_db
|
||||||
from .frame_source import FrameSource
|
from .frame_source import FrameSource
|
||||||
from .search_service import SearchService
|
from .search_service import SearchService
|
||||||
from .sync import rebuild_meili
|
from .sync import CaptionMeiliSync
|
||||||
|
|
||||||
|
|
||||||
class CaptionJobStopped(Exception):
|
class CaptionJobStopped(Exception):
|
||||||
@@ -50,6 +50,9 @@ class CaptionJobManager:
|
|||||||
"paused": False,
|
"paused": False,
|
||||||
"pause_requested": False,
|
"pause_requested": False,
|
||||||
"stop_requested": False,
|
"stop_requested": False,
|
||||||
|
"meili_documents_updated": 0,
|
||||||
|
"meili_sync_pending": 0,
|
||||||
|
"meili_sync_error": None,
|
||||||
}
|
}
|
||||||
|
|
||||||
@property
|
@property
|
||||||
@@ -177,6 +180,9 @@ class CaptionJobManager:
|
|||||||
"paused": False,
|
"paused": False,
|
||||||
"pause_requested": False,
|
"pause_requested": False,
|
||||||
"stop_requested": False,
|
"stop_requested": False,
|
||||||
|
"meili_documents_updated": 0,
|
||||||
|
"meili_sync_pending": 0,
|
||||||
|
"meili_sync_error": None,
|
||||||
}
|
}
|
||||||
|
|
||||||
self._thread = threading.Thread(
|
self._thread = threading.Thread(
|
||||||
@@ -198,6 +204,12 @@ class CaptionJobManager:
|
|||||||
limit=None,
|
limit=None,
|
||||||
library_ids=library_ids,
|
library_ids=library_ids,
|
||||||
)
|
)
|
||||||
|
caption_sync = CaptionMeiliSync(
|
||||||
|
library_ids,
|
||||||
|
meili=self.search_service.meili,
|
||||||
|
batch_size=settings.caption_meili_batch_size,
|
||||||
|
progress=self._progress,
|
||||||
|
)
|
||||||
|
|
||||||
self._progress(
|
self._progress(
|
||||||
{
|
{
|
||||||
@@ -233,9 +245,11 @@ class CaptionJobManager:
|
|||||||
"already_captioned": already_captioned,
|
"already_captioned": already_captioned,
|
||||||
"frames_selected": len(rows),
|
"frames_selected": len(rows),
|
||||||
"frames_total": len(rows),
|
"frames_total": len(rows),
|
||||||
"preparing": None,
|
"preparing": "Catching up saved captions in search",
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
caption_sync.catch_up()
|
||||||
|
self._progress({"preparing": None})
|
||||||
|
|
||||||
if not rows:
|
if not rows:
|
||||||
self._progress(
|
self._progress(
|
||||||
@@ -248,9 +262,10 @@ class CaptionJobManager:
|
|||||||
)
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
self._run(args, rows)
|
self._run(args, rows, caption_sync)
|
||||||
except CaptionJobStopped:
|
except CaptionJobStopped:
|
||||||
self._set_gpu_busy(False)
|
self._set_gpu_busy(False)
|
||||||
|
caption_sync.flush()
|
||||||
self._pause_requested.clear()
|
self._pause_requested.clear()
|
||||||
self._stop_requested.clear()
|
self._stop_requested.clear()
|
||||||
self._resume_event.set()
|
self._resume_event.set()
|
||||||
@@ -268,6 +283,7 @@ class CaptionJobManager:
|
|||||||
)
|
)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
self._set_gpu_busy(False)
|
self._set_gpu_busy(False)
|
||||||
|
caption_sync.flush()
|
||||||
self._progress(
|
self._progress(
|
||||||
{
|
{
|
||||||
"running": False,
|
"running": False,
|
||||||
@@ -301,7 +317,12 @@ class CaptionJobManager:
|
|||||||
with self._state_lock:
|
with self._state_lock:
|
||||||
self._gpu_busy = value
|
self._gpu_busy = value
|
||||||
|
|
||||||
def _run(self, args: SimpleNamespace, rows: list) -> None:
|
def _run(
|
||||||
|
self,
|
||||||
|
args: SimpleNamespace,
|
||||||
|
rows: list,
|
||||||
|
caption_sync: CaptionMeiliSync,
|
||||||
|
) -> None:
|
||||||
missing = 0
|
missing = 0
|
||||||
try:
|
try:
|
||||||
self._progress({"stage": "routing"})
|
self._progress({"stage": "routing"})
|
||||||
@@ -330,6 +351,8 @@ class CaptionJobManager:
|
|||||||
finally:
|
finally:
|
||||||
self._set_gpu_busy(False)
|
self._set_gpu_busy(False)
|
||||||
|
|
||||||
|
caption_sync.catch_up()
|
||||||
|
|
||||||
if not routed:
|
if not routed:
|
||||||
self._progress(
|
self._progress(
|
||||||
{
|
{
|
||||||
@@ -401,6 +424,7 @@ class CaptionJobManager:
|
|||||||
total_frames=frames_total,
|
total_frames=frames_total,
|
||||||
frame_source=frame_source,
|
frame_source=frame_source,
|
||||||
before_frame=lambda: self._wait_if_paused("captioning"),
|
before_frame=lambda: self._wait_if_paused("captioning"),
|
||||||
|
after_saved=caption_sync.add,
|
||||||
)
|
)
|
||||||
|
|
||||||
if specialist_rows:
|
if specialist_rows:
|
||||||
@@ -425,6 +449,7 @@ class CaptionJobManager:
|
|||||||
total_frames=frames_total,
|
total_frames=frames_total,
|
||||||
frame_source=frame_source,
|
frame_source=frame_source,
|
||||||
before_frame=lambda: self._wait_if_paused("captioning"),
|
before_frame=lambda: self._wait_if_paused("captioning"),
|
||||||
|
after_saved=caption_sync.add,
|
||||||
)
|
)
|
||||||
|
|
||||||
completed = general_done + specialist_done
|
completed = general_done + specialist_done
|
||||||
@@ -432,25 +457,8 @@ class CaptionJobManager:
|
|||||||
|
|
||||||
self._wait_if_paused("captioning")
|
self._wait_if_paused("captioning")
|
||||||
|
|
||||||
self._progress(
|
caption_sync.flush()
|
||||||
{
|
caption_sync.catch_up()
|
||||||
"stage": "syncing",
|
|
||||||
"frames_completed": completed,
|
|
||||||
"failed": failed,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
def sync_progress(update: dict) -> None:
|
|
||||||
self._progress(
|
|
||||||
{
|
|
||||||
**update,
|
|
||||||
"stage": "syncing",
|
|
||||||
"frames_completed": completed,
|
|
||||||
"failed": failed,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
documents = rebuild_meili(progress=sync_progress)
|
|
||||||
|
|
||||||
self._progress(
|
self._progress(
|
||||||
{
|
{
|
||||||
@@ -459,7 +467,8 @@ class CaptionJobManager:
|
|||||||
"finished_at": datetime.now(timezone.utc).isoformat(),
|
"finished_at": datetime.now(timezone.utc).isoformat(),
|
||||||
"frames_completed": completed,
|
"frames_completed": completed,
|
||||||
"failed": failed,
|
"failed": failed,
|
||||||
"documents_processed": documents,
|
"documents_processed": caption_sync.updated_total,
|
||||||
|
"meili_documents_updated": caption_sync.updated_total,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
except CaptionJobStopped:
|
except CaptionJobStopped:
|
||||||
|
|||||||
Reference in New Issue
Block a user