from __future__ import annotations import threading from copy import deepcopy from datetime import datetime, timezone from types import SimpleNamespace from typing import Any from .captioner import ensure_caption_prompt_file, ensure_specialist_prompt_file from .cli import ( _caption_queue, _route_rows, _select_caption_rows, _unload_ollama_model, ) from .config import settings from .database import connect, init_db from .frame_source import FrameSource from .search_service import SearchService from .sync import CaptionMeiliSync class CaptionJobStopped(Exception): """Raised at a safe boundary when a caption job is stopped.""" class CaptionJobManager: """Runs one resumable caption job at a time for indexed Jellyfin frames.""" def __init__(self, search_service: SearchService) -> None: self.search_service = search_service self._state_lock = threading.Lock() self._thread: threading.Thread | None = None self._gpu_busy = False self._pause_requested = threading.Event() self._stop_requested = threading.Event() self._resume_event = threading.Event() self._resume_event.set() self._state: dict[str, Any] = { "running": False, "stage": "idle", "started_at": None, "finished_at": None, "error": None, "frames_total": 0, "frames_completed": 0, "failed": 0, "retained": 0, "percent": 0.0, "paused": False, "pause_requested": False, "stop_requested": False, "meili_documents_updated": 0, "meili_sync_pending": 0, "meili_sync_error": None, } @property def running(self) -> bool: with self._state_lock: return bool(self._state.get("running")) @property def gpu_busy(self) -> bool: with self._state_lock: return self._gpu_busy @property def search_blocked(self) -> bool: with self._state_lock: stage = str(self._state.get("stage") or "") return self._gpu_busy or stage in {"sync", "syncing", "synced"} def status(self) -> dict[str, Any]: with self._state_lock: return deepcopy(self._state) def pause(self) -> dict[str, Any]: with self._state_lock: if not self._state.get("running"): raise RuntimeError("No captioning job is running.") if self._state.get("stage") in {"sync", "syncing", "synced"}: raise RuntimeError("Captioning has finished; the search index is rebuilding.") self._pause_requested.set() self._resume_event.clear() self._state["pause_requested"] = True return self.status() def resume(self) -> dict[str, Any]: with self._state_lock: if not self._state.get("running"): raise RuntimeError("No captioning job is running.") if not self._pause_requested.is_set() and not self._state.get("paused"): raise RuntimeError("Captioning is not paused.") self._pause_requested.clear() self._resume_event.set() self._state["pause_requested"] = False self._state["paused"] = False if self._state.get("stage") == "paused": self._state["stage"] = str( self._state.get("paused_from_stage") or "captioning" ) return self.status() def stop(self) -> dict[str, Any]: with self._state_lock: if not self._state.get("running"): raise RuntimeError("No captioning job is running.") if self._state.get("stage") in {"sync", "syncing", "synced"}: raise RuntimeError( "Captioning has finished; the search index is rebuilding." ) self._stop_requested.set() self._pause_requested.clear() self._resume_event.set() self._state["stop_requested"] = True self._state["pause_requested"] = False self._state["paused"] = False return self.status() def _wait_if_paused(self, resume_stage: str = "captioning") -> None: if self._stop_requested.is_set(): raise CaptionJobStopped() if self._pause_requested.is_set(): with self._state_lock: self._state["paused"] = True self._state["pause_requested"] = True self._state["paused_from_stage"] = resume_stage self._state["stage"] = "paused" while self._pause_requested.is_set() and not self._stop_requested.is_set(): self._resume_event.wait(timeout=0.5) if self._stop_requested.is_set(): raise CaptionJobStopped() with self._state_lock: self._state["paused"] = False self._state["pause_requested"] = False if self._state.get("running"): self._state["stage"] = resume_stage if self._stop_requested.is_set(): raise CaptionJobStopped() def start(self, library_ids: list[str], *, force: bool = False) -> dict[str, Any]: library_ids = [str(value).strip() for value in library_ids if str(value).strip()] if not library_ids: raise RuntimeError("Select at least one Jellyfin library.") with self._state_lock: if self._state.get("running"): raise RuntimeError("A captioning job is already running.") self._pause_requested.clear() self._stop_requested.clear() self._resume_event.set() started_at = datetime.now(timezone.utc).isoformat() with self._state_lock: self._state = { "running": True, "stage": "starting", "started_at": started_at, "finished_at": None, "error": None, "library_ids": library_ids, "force": force, "indexed_frames": 0, "already_captioned": 0, "frames_selected": 0, "frames_total": 0, "frames_completed": 0, "failed": 0, "retained": 0, "percent": 0.0, "paused": False, "pause_requested": False, "stop_requested": False, "meili_documents_updated": 0, "meili_sync_pending": 0, "meili_sync_error": None, } self._thread = threading.Thread( target=self._prepare_and_run, args=(library_ids, force), name="scene-search-caption", daemon=True, ) self._thread.start() return self.status() def _prepare_and_run(self, library_ids: list[str], force: bool) -> None: caption_sync: CaptionMeiliSync | None = None try: init_db(settings.db_path) args = SimpleNamespace( force=force, media_id=None, title=None, limit=None, library_ids=library_ids, ) caption_sync = CaptionMeiliSync( library_ids, meili=self.search_service.meili, batch_size=settings.caption_meili_batch_size, progress=self._progress, ) self._progress( { "stage": "starting", "preparing": "Finding frames that need captions", } ) rows = _select_caption_rows(args, only_needing_caption=True) if self._stop_requested.is_set(): raise CaptionJobStopped() with connect(settings.db_path) as con: placeholders = ",".join("?" for _ in library_ids) totals = con.execute( f""" SELECT COUNT(*) AS total, SUM(CASE WHEN f.caption IS NOT NULL THEN 1 ELSE 0 END) AS captioned FROM frames f JOIN media m ON m.id = f.media_id WHERE m.source_kind = 'jellyfin' AND m.jellyfin_library_id IN ({placeholders}) """, library_ids, ).fetchone() indexed_frames = int(totals["total"] or 0) already_captioned = int(totals["captioned"] or 0) self._progress( { "indexed_frames": indexed_frames, "already_captioned": already_captioned, "frames_selected": len(rows), "frames_total": len(rows), "preparing": "Catching up saved captions in search", } ) caption_sync.catch_up() self._progress({"preparing": None}) if not rows: self._progress( { "running": False, "stage": "complete", "finished_at": datetime.now(timezone.utc).isoformat(), "percent": 100.0, } ) return self._run(args, rows, caption_sync) except CaptionJobStopped: self._set_gpu_busy(False) if caption_sync is not None: caption_sync.flush() self._pause_requested.clear() self._stop_requested.clear() self._resume_event.set() self._progress( { "running": False, "stage": "stopped", "finished_at": datetime.now(timezone.utc).isoformat(), "paused": False, "pause_requested": False, "stop_requested": False, "preparing": None, "error": None, } ) except Exception as exc: self._set_gpu_busy(False) if caption_sync is not None: caption_sync.flush() self._progress( { "running": False, "stage": "failed", "finished_at": datetime.now(timezone.utc).isoformat(), "preparing": None, "error": str(exc), } ) def _progress(self, update: dict[str, Any]) -> None: with self._state_lock: self._state.update(update) stage = str(self._state.get("stage") or "") total = int(self._state.get("frames_total") or 0) completed = int(self._state.get("frames_completed") or 0) if stage == "complete": self._state["percent"] = 100.0 elif stage in {"sync", "syncing", "synced"}: self._state["percent"] = 99.0 elif total > 0: self._state["percent"] = round( min(max(completed / total, 0.0), 1.0) * 99.0, 1, ) else: self._state["percent"] = 0.0 def _set_gpu_busy(self, value: bool) -> None: with self._state_lock: self._gpu_busy = value def _run( self, args: SimpleNamespace, rows: list, caption_sync: CaptionMeiliSync, ) -> None: missing = 0 try: self._progress({"stage": "routing"}) # The normal router uses the local GPU. A global specialist # override has no classifier work, so do not block semantic search. if settings.caption_force_specialist: with FrameSource() as frame_source: routed, missing = _route_rows( rows, progress=self._progress, frame_source=frame_source, before_batch=lambda: self._wait_if_paused("routing"), ) else: self._set_gpu_busy(True) try: with self.search_service.model_lock: with FrameSource() as frame_source: routed, missing = _route_rows( rows, progress=self._progress, frame_source=frame_source, before_batch=lambda: self._wait_if_paused("routing"), ) finally: self._set_gpu_busy(False) caption_sync.catch_up() if not routed: self._progress( { "running": False, "stage": "complete", "finished_at": datetime.now(timezone.utc).isoformat(), "frames_total": 0, "frames_completed": 0, "failed": missing, } ) return general_prompt = ensure_caption_prompt_file(settings.caption_prompt_file) specialist_prompt = ensure_specialist_prompt_file( settings.caption_specialist_prompt_file ) general_rows = [ item for item in routed if item[1] == "general" and ( args.force or item[0]["caption"] is None or item[0]["caption_model"] != settings.caption_general_model ) ] specialist_rows = [ item for item in routed if item[1] == "specialist" and ( args.force or item[0]["caption"] is None or item[0]["caption_model"] != settings.caption_specialist_model ) ] retained = len(routed) - len(general_rows) - len(specialist_rows) frames_total = len(general_rows) + len(specialist_rows) self._progress( { "stage": "captioning", "frames_total": frames_total, "frames_completed": 0, "failed": missing, "retained": retained, "general_total": len(general_rows), "specialist_total": len(specialist_rows), } ) if general_rows: _unload_ollama_model(settings.caption_specialist_model) def general_progress(update: dict) -> None: adjusted = dict(update) adjusted["failed"] = missing + int(update.get("failed") or 0) self._progress(adjusted) with FrameSource() as frame_source: general_done, general_failed = _caption_queue( "General queue", general_rows, settings.caption_general_model, general_prompt, progress=general_progress, total_frames=frames_total, frame_source=frame_source, before_frame=lambda: self._wait_if_paused("captioning"), after_saved=caption_sync.add, ) if specialist_rows: _unload_ollama_model(settings.caption_general_model) def specialist_progress(update: dict) -> None: adjusted = dict(update) adjusted["failed"] = ( missing + general_failed + int(update.get("failed") or 0) ) self._progress(adjusted) specialist_done, specialist_failed = _caption_queue( "Specialist queue", specialist_rows, settings.caption_specialist_model, specialist_prompt, progress=specialist_progress, completed_offset=general_done, total_frames=frames_total, frame_source=frame_source, before_frame=lambda: self._wait_if_paused("captioning"), after_saved=caption_sync.add, ) completed = general_done + specialist_done failed = missing + general_failed + specialist_failed self._wait_if_paused("captioning") caption_sync.flush() caption_sync.catch_up() self._progress( { "running": False, "stage": "complete", "finished_at": datetime.now(timezone.utc).isoformat(), "frames_completed": completed, "failed": failed, "documents_processed": caption_sync.updated_total, "meili_documents_updated": caption_sync.updated_total, } ) except CaptionJobStopped: self._set_gpu_busy(False) caption_sync.flush() self._pause_requested.clear() self._stop_requested.clear() self._resume_event.set() self._progress( { "running": False, "stage": "stopped", "finished_at": datetime.now(timezone.utc).isoformat(), "paused": False, "pause_requested": False, "stop_requested": False, "error": None, } ) except Exception as exc: self._set_gpu_busy(False) caption_sync.flush() self._progress( { "running": False, "stage": "failed", "finished_at": datetime.now(timezone.utc).isoformat(), "error": str(exc), "failed": max(int(self._state.get("failed") or 0), missing), } )