from __future__ import annotations import time from collections.abc import Callable from typing import Any from .config import settings from .database import connect, init_db, unpack_vector from .jellyfin import tags_from_json from .meili import Meili SyncProgress = Callable[[dict], None] def _meili_client() -> Meili: return Meili( settings.meili_url, settings.meili_master_key, settings.meili_index, ) def sync_frame_metadata( frame_ids: list[str], *, meili: Meili | None = None, ) -> int: """Push caption/window/routing metadata for existing Meili documents only.""" unique_ids = list(dict.fromkeys(str(value) for value in frame_ids if value)) if not unique_ids: return 0 init_db(settings.db_path) placeholders = ",".join("?" for _ in unique_ids) with connect(settings.db_path) as con: rows = con.execute( f""" SELECT f.id, CASE WHEN cw.representative_frame_id = f.id THEN cw.caption ELSE f.caption END AS served_caption, CASE WHEN cw.representative_frame_id = f.id THEN cw.caption_model ELSE f.caption_model END AS served_caption_model, CASE WHEN cw.representative_frame_id = f.id THEN cw.captioned_at ELSE f.captioned_at END AS served_captioned_at, CASE WHEN cw.representative_frame_id = f.id THEN cw.caption_route ELSE f.caption_route END AS served_caption_route, CASE WHEN cw.representative_frame_id = f.id THEN cw.caption_route_reason ELSE f.caption_route_reason END AS served_caption_route_reason, CASE WHEN cw.representative_frame_id = f.id THEN cw.route_score ELSE f.route_score END AS served_route_score, CASE WHEN cw.representative_frame_id = f.id THEN cw.route_model ELSE f.route_model END AS served_route_model, CASE WHEN cw.representative_frame_id = f.id THEN cw.route_policy ELSE f.route_policy END AS served_route_policy, CASE WHEN cw.representative_frame_id = f.id THEN cw.routed_at ELSE f.routed_at END AS served_routed_at, CASE WHEN cw.representative_frame_id = f.id THEN cw.id ELSE NULL END AS caption_window_id, CASE WHEN cw.representative_frame_id = f.id THEN cw.start_ms ELSE NULL END AS caption_window_start_ms, CASE WHEN cw.representative_frame_id = f.id THEN cw.end_ms ELSE NULL END AS caption_window_end_ms FROM frames f LEFT JOIN caption_window_frames cwf ON cwf.frame_id = f.id LEFT JOIN caption_windows cw ON cw.id = cwf.window_id WHERE f.id IN ({placeholders}) """, unique_ids, ).fetchall() if not rows: return 0 documents = [ { "id": row["id"], "caption": row["served_caption"], "caption_model": row["served_caption_model"], "captioned_at": row["served_captioned_at"], "caption_route": row["served_caption_route"], "caption_route_reason": row["served_caption_route_reason"], "route_score": row["served_route_score"], "route_model": row["served_route_model"], "route_policy": row["served_route_policy"], "routed_at": row["served_routed_at"], "caption_window_id": row["caption_window_id"], "caption_window_start_ms": row["caption_window_start_ms"], "caption_window_end_ms": row["caption_window_end_ms"], } for row in rows ] client = meili or _meili_client() client.update_documents(documents, wait=True) synced_ids = [str(row["id"]) for row in rows] placeholders = ",".join("?" for _ in synced_ids) with connect(settings.db_path) as con: con.execute( f""" UPDATE frames SET meili_synced_at = CURRENT_TIMESTAMP WHERE id IN ({placeholders}) """, synced_ids, ) con.execute( f""" UPDATE caption_windows SET meili_synced_at = CURRENT_TIMESTAMP WHERE representative_frame_id IN ({placeholders}) AND caption IS NOT NULL """, synced_ids, ) return len(synced_ids) def unsynced_caption_frame_ids( library_ids: list[str], *, limit: int, ) -> list[str]: if not library_ids: return [] init_db(settings.db_path) placeholders = ",".join("?" for _ in library_ids) with connect(settings.db_path) as con: rows = con.execute( f""" SELECT f.id FROM frames f JOIN media m ON m.id = f.media_id LEFT JOIN caption_window_frames cwf ON cwf.frame_id = f.id LEFT JOIN caption_windows cw ON cw.id = cwf.window_id WHERE m.source_kind = 'jellyfin' AND m.jellyfin_library_id IN ({placeholders}) AND f.meili_synced_at IS NULL AND (f.caption IS NOT NULL OR cw.caption IS NOT NULL) ORDER BY COALESCE(cw.captioned_at, f.captioned_at), f.id LIMIT ? """, [*library_ids, limit], ).fetchall() return [str(row["id"]) for row in rows] def count_unsynced_caption_frames(library_ids: list[str]) -> int: if not library_ids: return 0 init_db(settings.db_path) placeholders = ",".join("?" for _ in library_ids) with connect(settings.db_path) as con: row = con.execute( f""" SELECT COUNT(*) AS total FROM frames f JOIN media m ON m.id = f.media_id LEFT JOIN caption_window_frames cwf ON cwf.frame_id = f.id LEFT JOIN caption_windows cw ON cw.id = cwf.window_id WHERE m.source_kind = 'jellyfin' AND m.jellyfin_library_id IN ({placeholders}) AND f.meili_synced_at IS NULL AND (f.caption IS NOT NULL OR cw.caption IS NOT NULL) """, library_ids, ).fetchone() return int(row["total"] or 0) class CaptionMeiliSync: """Incrementally sync caption metadata while SQLite remains canonical.""" def __init__( self, library_ids: list[str], *, meili: Meili, batch_size: int, progress: Callable[[dict[str, Any]], None] | None = None, ) -> None: self.library_ids = list(library_ids) self.meili = meili self.batch_size = max(1, int(batch_size)) self.progress = progress self.pending: list[str] = [] self.updated_total = 0 self.last_error: str | None = None def _report(self, *, pending: int | None = None) -> None: if self.progress is None: return update: dict[str, Any] = { "meili_documents_updated": self.updated_total, "meili_sync_error": self.last_error, } if pending is not None: update["meili_sync_pending"] = pending self.progress(update) def add(self, frame_id: str) -> None: self.pending.append(str(frame_id)) if len(self.pending) >= self.batch_size: self.flush() def flush(self) -> int: if not self.pending: return 0 frame_ids = self.pending self.pending = [] try: count = sync_frame_metadata(frame_ids, meili=self.meili) except Exception as exc: self.last_error = str(exc) self._report(pending=len(frame_ids)) print(f"WARN caption Meilisearch batch update failed: {exc}") return 0 self.updated_total += count self.last_error = None self._report() print( f"Updated {count} caption document(s) in Meilisearch " f"({self.updated_total} this run)." ) return count def catch_up(self) -> int: """Sync previously saved-but-unsynced captions in bounded batches.""" caught_up = 0 while True: frame_ids = unsynced_caption_frame_ids( self.library_ids, limit=self.batch_size, ) if not frame_ids: break try: count = sync_frame_metadata(frame_ids, meili=self.meili) except Exception as exc: self.last_error = str(exc) self._report() print(f"WARN caption Meilisearch catch-up failed: {exc}") break caught_up += count self.updated_total += count self.last_error = None self._report() if count == 0: break pending = ( count_unsynced_caption_frames(self.library_ids) if self.last_error else 0 ) self._report(pending=pending) return caught_up # Keep the full replacement representation identical for initial indexing # and targeted caption-document recovery. _FRAME_DOCUMENT_SELECT = """ SELECT f.id, f.media_id, f.timestamp_ms, f.thumb_path, f.embedding_f16, CASE WHEN cw.representative_frame_id = f.id THEN cw.caption ELSE f.caption END AS served_caption, CASE WHEN cw.representative_frame_id = f.id THEN cw.caption_model ELSE f.caption_model END AS served_caption_model, CASE WHEN cw.representative_frame_id = f.id THEN cw.captioned_at ELSE f.captioned_at END AS served_captioned_at, CASE WHEN cw.representative_frame_id = f.id THEN cw.caption_route ELSE f.caption_route END AS served_caption_route, CASE WHEN cw.representative_frame_id = f.id THEN cw.caption_route_reason ELSE f.caption_route_reason END AS served_caption_route_reason, CASE WHEN cw.representative_frame_id = f.id THEN cw.route_score ELSE f.route_score END AS served_route_score, CASE WHEN cw.representative_frame_id = f.id THEN cw.route_model ELSE f.route_model END AS served_route_model, CASE WHEN cw.representative_frame_id = f.id THEN cw.route_policy ELSE f.route_policy END AS served_route_policy, CASE WHEN cw.representative_frame_id = f.id THEN cw.routed_at ELSE f.routed_at END AS served_routed_at, CASE WHEN cw.representative_frame_id = f.id THEN cw.id ELSE NULL END AS caption_window_id, CASE WHEN cw.representative_frame_id = f.id THEN cw.start_ms ELSE NULL END AS caption_window_start_ms, CASE WHEN cw.representative_frame_id = f.id THEN cw.end_ms ELSE NULL END AS caption_window_end_ms, m.title, m.source_path, m.embedding_dim, m.model_id, m.source_kind, m.jellyfin_item_id, m.jellyfin_media_source_id, m.jellyfin_library_id, m.jellyfin_item_type, m.jellyfin_tags, m.trickplay_width FROM frames f JOIN media m ON m.id = f.media_id LEFT JOIN caption_window_frames cwf ON cwf.frame_id = f.id LEFT JOIN caption_windows cw ON cw.id = cwf.window_id """ def _complete_frame_document(row) -> dict: """Rebuild one complete serving document from canonical SQLite data.""" vec = unpack_vector(row["embedding_f16"], int(row["embedding_dim"])) return { "id": row["id"], "media_id": row["media_id"], "title": row["title"], "source_path": row["source_path"], "timestamp_ms": row["timestamp_ms"], "thumb_path": row["thumb_path"], "model_id": row["model_id"], "caption": row["served_caption"], "caption_model": row["served_caption_model"], "captioned_at": row["served_captioned_at"], "caption_route": row["served_caption_route"], "caption_route_reason": row["served_caption_route_reason"], "route_score": row["served_route_score"], "route_model": row["served_route_model"], "route_policy": row["served_route_policy"], "routed_at": row["served_routed_at"], "caption_window_id": row["caption_window_id"], "caption_window_start_ms": row["caption_window_start_ms"], "caption_window_end_ms": row["caption_window_end_ms"], "source_kind": row["source_kind"], "jellyfin_item_id": row["jellyfin_item_id"], "jellyfin_media_source_id": row["jellyfin_media_source_id"], "jellyfin_library_id": row["jellyfin_library_id"], "jellyfin_item_type": row["jellyfin_item_type"], "jellyfin_tags": tags_from_json(row["jellyfin_tags"]), "trickplay_width": row["trickplay_width"], "_vectors": {"siglip2": vec.tolist()}, } def count_caption_repair_documents(db_path=None) -> int: """Count only frames touched by captioning; never require a full reindex.""" db_path = db_path or settings.db_path init_db(db_path) with connect(db_path) as con: row = con.execute( """ SELECT COUNT(*) FROM ( SELECT frame_id FROM caption_window_frames UNION SELECT id FROM frames WHERE caption IS NOT NULL ) """ ).fetchone() return int(row[0]) _REPAIR_CHECKPOINT_SCHEMA = """ CREATE TABLE IF NOT EXISTS caption_index_repair_checkpoints ( index_uid TEXT PRIMARY KEY, last_frame_id TEXT, completed_documents INTEGER NOT NULL DEFAULT 0, pending_task_uid INTEGER, pending_last_frame_id TEXT, pending_count INTEGER NOT NULL DEFAULT 0, updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP ) """ def _repair_batch(db_path, after_frame_id: str | None, batch_size: int): """Fetch the next bounded batch without holding a SQLite cursor over IO.""" query = ( "WITH repair_ids AS (" " SELECT frame_id AS id FROM caption_window_frames" " UNION SELECT id FROM frames WHERE caption IS NOT NULL" ") " + _FRAME_DOCUMENT_SELECT.replace( "FROM frames f", "FROM repair_ids r JOIN frames f ON f.id = r.id", 1 ) + " WHERE (? IS NULL OR f.id > ?) ORDER BY f.id LIMIT ?" ) with connect(db_path) as con: return con.execute( query, (after_frame_id, after_frame_id, batch_size) ).fetchall() def _ensure_repair_checkpoint(db_path, index_uid: str) -> dict: with connect(db_path) as con: con.execute(_REPAIR_CHECKPOINT_SCHEMA) con.execute( """ INSERT OR IGNORE INTO caption_index_repair_checkpoints (index_uid) VALUES (?) """, (index_uid,), ) row = con.execute( "SELECT * FROM caption_index_repair_checkpoints WHERE index_uid = ?", (index_uid,), ).fetchone() return dict(row) def _record_pending_repair( db_path, index_uid: str, task_uid: int, last_frame_id: str, count: int ) -> None: with connect(db_path) as con: con.execute( """ UPDATE caption_index_repair_checkpoints SET pending_task_uid = ?, pending_last_frame_id = ?, pending_count = ?, updated_at = CURRENT_TIMESTAMP WHERE index_uid = ? AND pending_task_uid IS NULL """, (task_uid, last_frame_id, count, index_uid), ) if con.execute("SELECT changes()").fetchone()[0] != 1: raise RuntimeError("Repair checkpoint changed unexpectedly") def _finish_pending_repair(db_path, index_uid: str, task_uid: int) -> int: """Advance the durable cursor only once Meilisearch confirms success.""" with connect(db_path) as con: con.execute( """ UPDATE caption_index_repair_checkpoints SET last_frame_id = pending_last_frame_id, completed_documents = completed_documents + pending_count, pending_task_uid = NULL, pending_last_frame_id = NULL, pending_count = 0, updated_at = CURRENT_TIMESTAMP WHERE index_uid = ? AND pending_task_uid = ? """, (index_uid, task_uid), ) if con.execute("SELECT changes()").fetchone()[0] != 1: raise RuntimeError("Repair checkpoint changed unexpectedly") return int(con.execute( """ SELECT completed_documents FROM caption_index_repair_checkpoints WHERE index_uid = ? """, (index_uid,) ).fetchone()[0]) def repair_caption_documents( *, batch_size: int = 250, progress: SyncProgress | None = None, meili: Meili | None = None, db_path=None, resume_task_uid: int | None = None, ) -> int: """Resumable repair of only caption-affected Meilisearch documents. A durable SQLite checkpoint records each pending Meilisearch task *before* waiting on it. If interrupted, the next invocation waits for that same task, then continues at the last verified frame. Slow queues never trigger the 120-second caption-sync timeout. """ db_path = db_path or settings.db_path init_db(db_path) client = meili or _meili_client() index_uid = str(client.index) batch_size = max(1, int(batch_size)) restored_this_run = 0 checkpoint = _ensure_repair_checkpoint(db_path, index_uid) if resume_task_uid is not None: if checkpoint["pending_task_uid"] is not None: raise ValueError("An existing repair task is already checkpointed") if checkpoint["last_frame_id"] is not None: raise ValueError( "--resume-task is only for adopting the original first batch " "of an older, non-resumable repair" ) rows = _repair_batch(db_path, None, batch_size) if not rows: raise ValueError("No documents exist to associate with that task") task = client.get_task(resume_task_uid) if task.get("indexUid") != index_uid: raise ValueError( f"Task {resume_task_uid} belongs to index " f"{task.get('indexUid')!r}, not {index_uid!r}" ) if task.get("type") != "documentAdditionOrUpdate": raise ValueError( f"Task {resume_task_uid} is not a document update task" ) if task.get("status") in {"failed", "canceled"}: raise RuntimeError( f"Task {resume_task_uid} already {task.get('status')}; " "rerun without --resume-task to resubmit that batch" ) details = task.get("details") or {} expected = details.get("receivedDocuments") if expected is not None and int(expected) != len(rows): raise ValueError( f"Task {resume_task_uid} contains {expected} documents, " f"but the first repair batch contains {len(rows)}" ) _record_pending_repair( db_path, index_uid, resume_task_uid, str(rows[-1]["id"]), len(rows) ) print( f"Adopted existing Meilisearch task {resume_task_uid} " f"({len(rows)} documents); no duplicate batch submitted.", flush=True, ) while True: checkpoint = _ensure_repair_checkpoint(db_path, index_uid) task_uid = checkpoint["pending_task_uid"] if task_uid is None: rows = _repair_batch( db_path, checkpoint["last_frame_id"], batch_size ) if not rows: break docs = [_complete_frame_document(row) for row in rows] # Wait=False returns the asynchronous task id. Persist it before # waiting so a container restart cannot forget queued work. task_uid = client.add_documents(docs, wait=False) if task_uid is None: raise RuntimeError( "Meilisearch did not return a task ID; cannot resume safely" ) task_uid = int(task_uid) _record_pending_repair( db_path, index_uid, task_uid, str(rows[-1]["id"]), len(rows) ) print( f"Queued repair task {task_uid}: {len(rows)} documents " f"(already confirmed: {checkpoint['completed_documents']:,}).", flush=True, ) last_update = time.monotonic() - 16 def on_poll(task: dict) -> None: nonlocal last_update now = time.monotonic() if now - last_update >= 15: status = str(task.get("status") or "unknown") print( f"Waiting for Meilisearch task {task_uid}: {status} " f"(repair remains resumable with Ctrl+C).", flush=True, ) last_update = now # Unlike ordinary caption sync, a repair intentionally waits as long # as the index needs. Re-run the command after Ctrl+C or a restart: # the task UID is already stored in SQLite. try: client.wait_for_task( int(task_uid), timeout_seconds=None, poll_seconds=3.0, on_poll=on_poll, ) except RuntimeError: # Failed/canceled tasks are safe to resubmit. Preserve the # checkpoint on API/network errors to avoid duplicate jobs. try: task = client.get_task(int(task_uid)) except Exception: raise if task.get("status") in {"failed", "canceled"}: with connect(db_path) as con: con.execute( """ UPDATE caption_index_repair_checkpoints SET pending_task_uid = NULL, pending_last_frame_id = NULL, pending_count = 0, updated_at = CURRENT_TIMESTAMP WHERE index_uid = ? AND pending_task_uid = ? """, (index_uid, task_uid), ) raise count = int(checkpoint["pending_count"]) if checkpoint["pending_task_uid"] else len(rows) confirmed_total = _finish_pending_repair(db_path, index_uid, int(task_uid)) restored_this_run += count if progress is not None: progress({ "stage": "repair", "documents_processed": confirmed_total, "completed_this_run": restored_this_run, "task_uid": task_uid, }) print( f"Repair index {index_uid}: {checkpoint['completed_documents']:,} " "documents previously confirmed before this run.", flush=True, ) return restored_this_run def rebuild_meili(*, progress: SyncProgress | None = None) -> int: """Rebuild the disposable Meilisearch index from canonical SQLite.""" init_db(settings.db_path) meili = _meili_client() with connect(settings.db_path) as con: dims = [row[0] for row in con.execute( "SELECT DISTINCT embedding_dim FROM media" )] if not dims: meili.clear_documents() if progress is not None: progress({"stage": "sync", "documents_processed": 0}) return 0 if len(dims) != 1: raise RuntimeError( f"Multiple embedding dimensions found: {dims}. " "Use separate indexes per model." ) meili.configure_user_vectors(int(dims[0])) meili.clear_documents() cursor = con.execute(_FRAME_DOCUMENT_SELECT + " ORDER BY f.id") total = 0 while True: rows = cursor.fetchmany(100) if not rows: break docs = [] for row in rows: docs.append(_complete_frame_document(row)) meili.add_documents(docs) total += len(docs) if progress is not None: progress( { "stage": "sync", "documents_processed": total, } ) with connect(settings.db_path) as con: con.execute( """ UPDATE frames SET meili_synced_at = CURRENT_TIMESTAMP """ ) con.execute( """ UPDATE caption_windows SET meili_synced_at = CURRENT_TIMESTAMP WHERE caption IS NOT NULL """ ) if progress is not None: progress({"stage": "synced", "documents_processed": total}) return total