diff --git a/scene_search/sync.py b/scene_search/sync.py index 86cbbe0..30f33e9 100644 --- a/scene_search/sync.py +++ b/scene_search/sync.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections.abc import Callable +from typing import Any from .config import settings from .database import connect, init_db, unpack_vector @@ -10,15 +11,218 @@ from .meili import Meili SyncProgress = Callable[[dict], None] -def rebuild_meili(*, progress: SyncProgress | None = None) -> int: - """Rebuild the disposable Meilisearch index from canonical SQLite.""" - init_db(settings.db_path) - meili = Meili( +def _meili_client() -> Meili: + return Meili( settings.meili_url, settings.meili_master_key, settings.meili_index, ) + +def sync_frame_metadata( + frame_ids: list[str], + *, + meili: Meili | None = None, +) -> int: + """Push caption/routing metadata for existing Meili documents only.""" + unique_ids = list(dict.fromkeys(str(value) for value in frame_ids if value)) + if not unique_ids: + return 0 + + init_db(settings.db_path) + placeholders = ",".join("?" for _ in unique_ids) + with connect(settings.db_path) as con: + rows = con.execute( + f""" + SELECT id, caption, caption_model, captioned_at, + caption_route, caption_route_reason, route_score, + route_model, route_policy, routed_at + FROM frames + WHERE id IN ({placeholders}) + """, + unique_ids, + ).fetchall() + + if not rows: + return 0 + + documents = [ + { + "id": row["id"], + "caption": row["caption"], + "caption_model": row["caption_model"], + "captioned_at": row["captioned_at"], + "caption_route": row["caption_route"], + "caption_route_reason": row["caption_route_reason"], + "route_score": row["route_score"], + "route_model": row["route_model"], + "route_policy": row["route_policy"], + "routed_at": row["routed_at"], + } + for row in rows + ] + + client = meili or _meili_client() + client.update_documents(documents) + + synced_ids = [str(row["id"]) for row in rows] + placeholders = ",".join("?" for _ in synced_ids) + with connect(settings.db_path) as con: + con.execute( + f""" + UPDATE frames + SET meili_synced_at = CURRENT_TIMESTAMP + WHERE id IN ({placeholders}) + """, + synced_ids, + ) + + return len(synced_ids) + + +def unsynced_caption_frame_ids( + library_ids: list[str], + *, + limit: int, +) -> list[str]: + if not library_ids: + return [] + + init_db(settings.db_path) + placeholders = ",".join("?" for _ in library_ids) + with connect(settings.db_path) as con: + rows = con.execute( + f""" + SELECT f.id + FROM frames f + JOIN media m ON m.id = f.media_id + WHERE m.source_kind = 'jellyfin' + AND m.jellyfin_library_id IN ({placeholders}) + AND f.caption IS NOT NULL + AND f.meili_synced_at IS NULL + ORDER BY f.captioned_at, f.id + LIMIT ? + """, + [*library_ids, limit], + ).fetchall() + return [str(row["id"]) for row in rows] + + +def count_unsynced_caption_frames(library_ids: list[str]) -> int: + if not library_ids: + return 0 + + init_db(settings.db_path) + placeholders = ",".join("?" for _ in library_ids) + with connect(settings.db_path) as con: + row = con.execute( + f""" + SELECT COUNT(*) AS total + FROM frames f + JOIN media m ON m.id = f.media_id + WHERE m.source_kind = 'jellyfin' + AND m.jellyfin_library_id IN ({placeholders}) + AND f.caption IS NOT NULL + AND f.meili_synced_at IS NULL + """, + library_ids, + ).fetchone() + return int(row["total"] or 0) + + +class CaptionMeiliSync: + """Incrementally sync caption metadata while SQLite remains canonical.""" + + def __init__( + self, + library_ids: list[str], + *, + meili: Meili, + batch_size: int, + progress: Callable[[dict[str, Any]], None] | None = None, + ) -> None: + self.library_ids = list(library_ids) + self.meili = meili + self.batch_size = max(1, int(batch_size)) + self.progress = progress + self.pending: list[str] = [] + self.updated_total = 0 + self.last_error: str | None = None + + def _report(self) -> None: + if self.progress is None: + return + self.progress( + { + "meili_documents_updated": self.updated_total, + "meili_sync_pending": count_unsynced_caption_frames( + self.library_ids + ), + "meili_sync_error": self.last_error, + } + ) + + def add(self, frame_id: str) -> None: + self.pending.append(str(frame_id)) + if len(self.pending) >= self.batch_size: + self.flush() + + def flush(self) -> int: + if not self.pending: + return 0 + + frame_ids = self.pending + self.pending = [] + try: + count = sync_frame_metadata(frame_ids, meili=self.meili) + except Exception as exc: + self.last_error = str(exc) + self._report() + print(f"WARN caption Meilisearch batch update failed: {exc}") + return 0 + + self.updated_total += count + self.last_error = None + self._report() + print( + f"Updated {count} caption document(s) in Meilisearch " + f"({self.updated_total} this run)." + ) + return count + + def catch_up(self) -> int: + """Sync previously saved-but-unsynced captions in bounded batches.""" + caught_up = 0 + while True: + frame_ids = unsynced_caption_frame_ids( + self.library_ids, + limit=self.batch_size, + ) + if not frame_ids: + break + try: + count = sync_frame_metadata(frame_ids, meili=self.meili) + except Exception as exc: + self.last_error = str(exc) + self._report() + print(f"WARN caption Meilisearch catch-up failed: {exc}") + break + + caught_up += count + self.updated_total += count + self.last_error = None + self._report() + if count == 0: + break + + return caught_up + + +def rebuild_meili(*, progress: SyncProgress | None = None) -> int: + """Rebuild the disposable Meilisearch index from canonical SQLite.""" + init_db(settings.db_path) + meili = _meili_client() + with connect(settings.db_path) as con: dims = [row[0] for row in con.execute( "SELECT DISTINCT embedding_dim FROM media" @@ -104,6 +308,14 @@ def rebuild_meili(*, progress: SyncProgress | None = None) -> int: } ) + with connect(settings.db_path) as con: + con.execute( + """ + UPDATE frames + SET meili_synced_at = CURRENT_TIMESTAMP + """ + ) + if progress is not None: progress({"stage": "synced", "documents_processed": total}) return total