Extract reusable Meilisearch rebuild service

2026-11-14 22:11:27 +00:00 · 2026-10-06 17:55:10 +01:00
parent d19ae1b736
commit 21f092b072
1 changed files with 109 additions and 0 deletions
@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+
+from .config import settings
+from .database import connect, init_db, unpack_vector
+from .jellyfin import tags_from_json
+from .meili import Meili
+
+SyncProgress = Callable[[dict], None]
+
+
+def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
+    """Rebuild the disposable Meilisearch index from canonical SQLite."""
+    init_db(settings.db_path)
+    meili = Meili(
+        settings.meili_url,
+        settings.meili_master_key,
+        settings.meili_index,
+    )
+
+    with connect(settings.db_path) as con:
+        dims = [row[0] for row in con.execute(
+            "SELECT DISTINCT embedding_dim FROM media"
+        )]
+        if not dims:
+            meili.clear_documents()
+            if progress is not None:
+                progress({"stage": "sync", "documents_processed": 0})
+            return 0
+        if len(dims) != 1:
+            raise RuntimeError(
+                f"Multiple embedding dimensions found: {dims}. "
+                "Use separate indexes per model."
+            )
+
+        meili.configure_user_vectors(int(dims[0]))
+        meili.clear_documents()
+
+        cursor = con.execute(
+            """
+            SELECT f.id, f.media_id, f.timestamp_ms, f.thumb_path, f.embedding_f16,
+                   f.caption, f.caption_model, f.captioned_at,
+                   f.caption_route, f.caption_route_reason, f.route_score,
+                   f.route_model, f.route_policy, f.routed_at,
+                   m.title, m.source_path, m.embedding_dim, m.model_id,
+                   m.source_kind, m.jellyfin_item_id, m.jellyfin_media_source_id,
+                   m.jellyfin_library_id, m.jellyfin_item_type, m.jellyfin_tags,
+                   m.trickplay_width
+            FROM frames f
+            JOIN media m ON m.id = f.media_id
+            ORDER BY f.id
+            """
+        )
+
+        total = 0
+        while True:
+            rows = cursor.fetchmany(100)
+            if not rows:
+                break
+
+            docs = []
+            for row in rows:
+                vec = unpack_vector(
+                    row["embedding_f16"],
+                    row["embedding_dim"],
+                )
+                docs.append(
+                    {
+                        "id": row["id"],
+                        "media_id": row["media_id"],
+                        "title": row["title"],
+                        "source_path": row["source_path"],
+                        "timestamp_ms": row["timestamp_ms"],
+                        "thumb_path": row["thumb_path"],
+                        "model_id": row["model_id"],
+                        "caption": row["caption"],
+                        "caption_model": row["caption_model"],
+                        "captioned_at": row["captioned_at"],
+                        "caption_route": row["caption_route"],
+                        "caption_route_reason": row["caption_route_reason"],
+                        "route_score": row["route_score"],
+                        "route_model": row["route_model"],
+                        "route_policy": row["route_policy"],
+                        "routed_at": row["routed_at"],
+                        "source_kind": row["source_kind"],
+                        "jellyfin_item_id": row["jellyfin_item_id"],
+                        "jellyfin_media_source_id": row["jellyfin_media_source_id"],
+                        "jellyfin_library_id": row["jellyfin_library_id"],
+                        "jellyfin_item_type": row["jellyfin_item_type"],
+                        "jellyfin_tags": tags_from_json(row["jellyfin_tags"]),
+                        "trickplay_width": row["trickplay_width"],
+                        "_vectors": {"siglip2": vec.tolist()},
+                    }
+                )
+
+            meili.add_documents(docs)
+            total += len(docs)
+            if progress is not None:
+                progress(
+                    {
+                        "stage": "sync",
+                        "documents_processed": total,
+                    }
+                )
+
+    if progress is not None:
+        progress({"stage": "synced", "documents_processed": total})
+    return total