from __future__ import annotations from collections.abc import Callable from typing import Any from .config import settings from .database import connect, init_db, unpack_vector from .jellyfin import tags_from_json from .meili import Meili SyncProgress = Callable[[dict], None] def _meili_client() -> Meili: return Meili( settings.meili_url, settings.meili_master_key, settings.meili_index, ) def sync_frame_metadata( frame_ids: list[str], *, meili: Meili | None = None, ) -> int: """Push caption/routing metadata for existing Meili documents only.""" unique_ids = list(dict.fromkeys(str(value) for value in frame_ids if value)) if not unique_ids: return 0 init_db(settings.db_path) placeholders = ",".join("?" for _ in unique_ids) with connect(settings.db_path) as con: rows = con.execute( f""" SELECT id, caption, caption_model, captioned_at, caption_route, caption_route_reason, route_score, route_model, route_policy, routed_at FROM frames WHERE id IN ({placeholders}) """, unique_ids, ).fetchall() if not rows: return 0 documents = [ { "id": row["id"], "caption": row["caption"], "caption_model": row["caption_model"], "captioned_at": row["captioned_at"], "caption_route": row["caption_route"], "caption_route_reason": row["caption_route_reason"], "route_score": row["route_score"], "route_model": row["route_model"], "route_policy": row["route_policy"], "routed_at": row["routed_at"], } for row in rows ] client = meili or _meili_client() client.update_documents(documents, wait=True) synced_ids = [str(row["id"]) for row in rows] placeholders = ",".join("?" for _ in synced_ids) with connect(settings.db_path) as con: con.execute( f""" UPDATE frames SET meili_synced_at = CURRENT_TIMESTAMP WHERE id IN ({placeholders}) """, synced_ids, ) return len(synced_ids) def unsynced_caption_frame_ids( library_ids: list[str], *, limit: int, ) -> list[str]: if not library_ids: return [] init_db(settings.db_path) placeholders = ",".join("?" for _ in library_ids) with connect(settings.db_path) as con: rows = con.execute( f""" SELECT f.id FROM frames f JOIN media m ON m.id = f.media_id WHERE m.source_kind = 'jellyfin' AND m.jellyfin_library_id IN ({placeholders}) AND f.caption IS NOT NULL AND f.meili_synced_at IS NULL ORDER BY f.captioned_at, f.id LIMIT ? """, [*library_ids, limit], ).fetchall() return [str(row["id"]) for row in rows] def count_unsynced_caption_frames(library_ids: list[str]) -> int: if not library_ids: return 0 init_db(settings.db_path) placeholders = ",".join("?" for _ in library_ids) with connect(settings.db_path) as con: row = con.execute( f""" SELECT COUNT(*) AS total FROM frames f JOIN media m ON m.id = f.media_id WHERE m.source_kind = 'jellyfin' AND m.jellyfin_library_id IN ({placeholders}) AND f.caption IS NOT NULL AND f.meili_synced_at IS NULL """, library_ids, ).fetchone() return int(row["total"] or 0) class CaptionMeiliSync: """Incrementally sync caption metadata while SQLite remains canonical.""" def __init__( self, library_ids: list[str], *, meili: Meili, batch_size: int, progress: Callable[[dict[str, Any]], None] | None = None, ) -> None: self.library_ids = list(library_ids) self.meili = meili self.batch_size = max(1, int(batch_size)) self.progress = progress self.pending: list[str] = [] self.updated_total = 0 self.last_error: str | None = None def _report(self, *, pending: int | None = None) -> None: if self.progress is None: return update: dict[str, Any] = { "meili_documents_updated": self.updated_total, "meili_sync_error": self.last_error, } if pending is not None: update["meili_sync_pending"] = pending self.progress(update) def add(self, frame_id: str) -> None: self.pending.append(str(frame_id)) if len(self.pending) >= self.batch_size: self.flush() def flush(self) -> int: if not self.pending: return 0 frame_ids = self.pending self.pending = [] try: count = sync_frame_metadata(frame_ids, meili=self.meili) except Exception as exc: self.last_error = str(exc) self._report() print(f"WARN caption Meilisearch batch update failed: {exc}") return 0 self.updated_total += count self.last_error = None self._report() print( f"Updated {count} caption document(s) in Meilisearch " f"({self.updated_total} this run)." ) return count def catch_up(self) -> int: """Sync previously saved-but-unsynced captions in bounded batches.""" caught_up = 0 while True: frame_ids = unsynced_caption_frame_ids( self.library_ids, limit=self.batch_size, ) if not frame_ids: break try: count = sync_frame_metadata(frame_ids, meili=self.meili) except Exception as exc: self.last_error = str(exc) self._report() print(f"WARN caption Meilisearch catch-up failed: {exc}") break caught_up += count self.updated_total += count self.last_error = None self._report() if count == 0: break pending = ( count_unsynced_caption_frames(self.library_ids) if self.last_error else 0 ) self._report(pending=pending) return caught_up def rebuild_meili(*, progress: SyncProgress | None = None) -> int: """Rebuild the disposable Meilisearch index from canonical SQLite.""" init_db(settings.db_path) meili = _meili_client() with connect(settings.db_path) as con: dims = [row[0] for row in con.execute( "SELECT DISTINCT embedding_dim FROM media" )] if not dims: meili.clear_documents() if progress is not None: progress({"stage": "sync", "documents_processed": 0}) return 0 if len(dims) != 1: raise RuntimeError( f"Multiple embedding dimensions found: {dims}. " "Use separate indexes per model." ) meili.configure_user_vectors(int(dims[0])) meili.clear_documents() cursor = con.execute( """ SELECT f.id, f.media_id, f.timestamp_ms, f.thumb_path, f.embedding_f16, f.caption, f.caption_model, f.captioned_at, f.caption_route, f.caption_route_reason, f.route_score, f.route_model, f.route_policy, f.routed_at, m.title, m.source_path, m.embedding_dim, m.model_id, m.source_kind, m.jellyfin_item_id, m.jellyfin_media_source_id, m.jellyfin_library_id, m.jellyfin_item_type, m.jellyfin_tags, m.trickplay_width FROM frames f JOIN media m ON m.id = f.media_id ORDER BY f.id """ ) total = 0 while True: rows = cursor.fetchmany(100) if not rows: break docs = [] for row in rows: vec = unpack_vector( row["embedding_f16"], row["embedding_dim"], ) docs.append( { "id": row["id"], "media_id": row["media_id"], "title": row["title"], "source_path": row["source_path"], "timestamp_ms": row["timestamp_ms"], "thumb_path": row["thumb_path"], "model_id": row["model_id"], "caption": row["caption"], "caption_model": row["caption_model"], "captioned_at": row["captioned_at"], "caption_route": row["caption_route"], "caption_route_reason": row["caption_route_reason"], "route_score": row["route_score"], "route_model": row["route_model"], "route_policy": row["route_policy"], "routed_at": row["routed_at"], "source_kind": row["source_kind"], "jellyfin_item_id": row["jellyfin_item_id"], "jellyfin_media_source_id": row["jellyfin_media_source_id"], "jellyfin_library_id": row["jellyfin_library_id"], "jellyfin_item_type": row["jellyfin_item_type"], "jellyfin_tags": tags_from_json(row["jellyfin_tags"]), "trickplay_width": row["trickplay_width"], "_vectors": {"siglip2": vec.tolist()}, } ) meili.add_documents(docs) total += len(docs) if progress is not None: progress( { "stage": "sync", "documents_processed": total, } ) with connect(settings.db_path) as con: con.execute( """ UPDATE frames SET meili_synced_at = CURRENT_TIMESTAMP """ ) if progress is not None: progress({"stage": "synced", "documents_processed": total}) return total