From d76c9678ffa4917c196180bc80572ae002acd46b Mon Sep 17 00:00:00 2001 From: Sleeps Date: Thu, 8 Oct 2026 10:53:36 +0100 Subject: [PATCH] Add focused reconstruction of caption-damaged documents from SQLite --- scene_search/sync.py | 184 ++++++++++++++++++++++++++++--------------- 1 file changed, 122 insertions(+), 62 deletions(-) diff --git a/scene_search/sync.py b/scene_search/sync.py index 6c037b0..5ce2d52 100644 --- a/scene_search/sync.py +++ b/scene_search/sync.py @@ -264,31 +264,9 @@ class CaptionMeiliSync: return caught_up -def rebuild_meili(*, progress: SyncProgress | None = None) -> int: - """Rebuild the disposable Meilisearch index from canonical SQLite.""" - init_db(settings.db_path) - meili = _meili_client() - - with connect(settings.db_path) as con: - dims = [row[0] for row in con.execute( - "SELECT DISTINCT embedding_dim FROM media" - )] - if not dims: - meili.clear_documents() - if progress is not None: - progress({"stage": "sync", "documents_processed": 0}) - return 0 - if len(dims) != 1: - raise RuntimeError( - f"Multiple embedding dimensions found: {dims}. " - "Use separate indexes per model." - ) - - meili.configure_user_vectors(int(dims[0])) - meili.clear_documents() - - cursor = con.execute( - """ +# Keep the full replacement representation identical for initial indexing +# and targeted caption-document recovery. +_FRAME_DOCUMENT_SELECT = """ SELECT f.id, f.media_id, f.timestamp_ms, f.thumb_path, f.embedding_f16, CASE WHEN cw.representative_frame_id = f.id @@ -327,10 +305,126 @@ def rebuild_meili(*, progress: SyncProgress | None = None) -> int: JOIN media m ON m.id = f.media_id LEFT JOIN caption_window_frames cwf ON cwf.frame_id = f.id LEFT JOIN caption_windows cw ON cw.id = cwf.window_id - ORDER BY f.id """ - ) + +def _complete_frame_document(row) -> dict: + """Rebuild one complete serving document from canonical SQLite data.""" + vec = unpack_vector(row["embedding_f16"], int(row["embedding_dim"])) + return { +"id": row["id"], + "media_id": row["media_id"], + "title": row["title"], + "source_path": row["source_path"], + "timestamp_ms": row["timestamp_ms"], + "thumb_path": row["thumb_path"], + "model_id": row["model_id"], + "caption": row["served_caption"], + "caption_model": row["served_caption_model"], + "captioned_at": row["served_captioned_at"], + "caption_route": row["served_caption_route"], + "caption_route_reason": row["served_caption_route_reason"], + "route_score": row["served_route_score"], + "route_model": row["served_route_model"], + "route_policy": row["served_route_policy"], + "routed_at": row["served_routed_at"], + "caption_window_id": row["caption_window_id"], + "caption_window_start_ms": row["caption_window_start_ms"], + "caption_window_end_ms": row["caption_window_end_ms"], + "source_kind": row["source_kind"], + "jellyfin_item_id": row["jellyfin_item_id"], + "jellyfin_media_source_id": row["jellyfin_media_source_id"], + "jellyfin_library_id": row["jellyfin_library_id"], + "jellyfin_item_type": row["jellyfin_item_type"], + "jellyfin_tags": tags_from_json(row["jellyfin_tags"]), + "trickplay_width": row["trickplay_width"], + "_vectors": {"siglip2": vec.tolist()}, + } + + +def count_caption_repair_documents(db_path=None) -> int: + """Count only frames touched by captioning; never require a full reindex.""" + db_path = db_path or settings.db_path + init_db(db_path) + with connect(db_path) as con: + row = con.execute( + """ + SELECT COUNT(*) FROM ( + SELECT frame_id FROM caption_window_frames + UNION + SELECT id FROM frames WHERE caption IS NOT NULL + ) + """ + ).fetchone() + return int(row[0]) + + +def repair_caption_documents( + *, + batch_size: int = 250, + progress: SyncProgress | None = None, + meili: Meili | None = None, + db_path=None, +) -> int: + """Restore caption-touched documents overwritten by older POST updates. + + Uses existing SQLite captions, vectors and Jellyfin identity without + clearing Meilisearch, re-embedding or invoking Ollama. Each replacement + task is awaited so failed repairs are not silently reported as complete. + """ + db_path = db_path or settings.db_path + init_db(db_path) + client = meili or _meili_client() + batch_size = max(1, int(batch_size)) + total = 0 + query = ( + "WITH repair_ids AS (" + " SELECT frame_id AS id FROM caption_window_frames" + " UNION SELECT id FROM frames WHERE caption IS NOT NULL" + ") " + + _FRAME_DOCUMENT_SELECT.replace( + "FROM frames f", "FROM repair_ids r JOIN frames f ON f.id = r.id", 1 + ) + + " ORDER BY f.id" + ) + with connect(db_path) as con: + cursor = con.execute(query) + while True: + rows = cursor.fetchmany(batch_size) + if not rows: + break + docs = [_complete_frame_document(row) for row in rows] + client.add_documents(docs, wait=True) + total += len(docs) + if progress is not None: + progress({"stage": "repair", "documents_processed": total}) + return total + + +def rebuild_meili(*, progress: SyncProgress | None = None) -> int: + """Rebuild the disposable Meilisearch index from canonical SQLite.""" + init_db(settings.db_path) + meili = _meili_client() + + with connect(settings.db_path) as con: + dims = [row[0] for row in con.execute( + "SELECT DISTINCT embedding_dim FROM media" + )] + if not dims: + meili.clear_documents() + if progress is not None: + progress({"stage": "sync", "documents_processed": 0}) + return 0 + if len(dims) != 1: + raise RuntimeError( + f"Multiple embedding dimensions found: {dims}. " + "Use separate indexes per model." + ) + + meili.configure_user_vectors(int(dims[0])) + meili.clear_documents() + + cursor = con.execute(_FRAME_DOCUMENT_SELECT + " ORDER BY f.id") total = 0 while True: rows = cursor.fetchmany(100) @@ -339,41 +433,7 @@ def rebuild_meili(*, progress: SyncProgress | None = None) -> int: docs = [] for row in rows: - vec = unpack_vector( - row["embedding_f16"], - row["embedding_dim"], - ) - docs.append( - { - "id": row["id"], - "media_id": row["media_id"], - "title": row["title"], - "source_path": row["source_path"], - "timestamp_ms": row["timestamp_ms"], - "thumb_path": row["thumb_path"], - "model_id": row["model_id"], - "caption": row["served_caption"], - "caption_model": row["served_caption_model"], - "captioned_at": row["served_captioned_at"], - "caption_route": row["served_caption_route"], - "caption_route_reason": row["served_caption_route_reason"], - "route_score": row["served_route_score"], - "route_model": row["served_route_model"], - "route_policy": row["served_route_policy"], - "routed_at": row["served_routed_at"], - "caption_window_id": row["caption_window_id"], - "caption_window_start_ms": row["caption_window_start_ms"], - "caption_window_end_ms": row["caption_window_end_ms"], - "source_kind": row["source_kind"], - "jellyfin_item_id": row["jellyfin_item_id"], - "jellyfin_media_source_id": row["jellyfin_media_source_id"], - "jellyfin_library_id": row["jellyfin_library_id"], - "jellyfin_item_type": row["jellyfin_item_type"], - "jellyfin_tags": tags_from_json(row["jellyfin_tags"]), - "trickplay_width": row["trickplay_width"], - "_vectors": {"siglip2": vec.tolist()}, - } - ) + docs.append(_complete_frame_document(row)) meili.add_documents(docs) total += len(docs)