mirror of
https://github.com/MrSleeps/jellyfin-scene-search.git
synced 2026-11-20 00:33:36 +00:00
Add focused reconstruction of caption-damaged documents from SQLite
This commit is contained in:
+122
-62
@@ -264,31 +264,9 @@ class CaptionMeiliSync:
|
||||
return caught_up
|
||||
|
||||
|
||||
def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
|
||||
"""Rebuild the disposable Meilisearch index from canonical SQLite."""
|
||||
init_db(settings.db_path)
|
||||
meili = _meili_client()
|
||||
|
||||
with connect(settings.db_path) as con:
|
||||
dims = [row[0] for row in con.execute(
|
||||
"SELECT DISTINCT embedding_dim FROM media"
|
||||
)]
|
||||
if not dims:
|
||||
meili.clear_documents()
|
||||
if progress is not None:
|
||||
progress({"stage": "sync", "documents_processed": 0})
|
||||
return 0
|
||||
if len(dims) != 1:
|
||||
raise RuntimeError(
|
||||
f"Multiple embedding dimensions found: {dims}. "
|
||||
"Use separate indexes per model."
|
||||
)
|
||||
|
||||
meili.configure_user_vectors(int(dims[0]))
|
||||
meili.clear_documents()
|
||||
|
||||
cursor = con.execute(
|
||||
"""
|
||||
# Keep the full replacement representation identical for initial indexing
|
||||
# and targeted caption-document recovery.
|
||||
_FRAME_DOCUMENT_SELECT = """
|
||||
SELECT
|
||||
f.id, f.media_id, f.timestamp_ms, f.thumb_path, f.embedding_f16,
|
||||
CASE WHEN cw.representative_frame_id = f.id
|
||||
@@ -327,10 +305,126 @@ def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
|
||||
JOIN media m ON m.id = f.media_id
|
||||
LEFT JOIN caption_window_frames cwf ON cwf.frame_id = f.id
|
||||
LEFT JOIN caption_windows cw ON cw.id = cwf.window_id
|
||||
ORDER BY f.id
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def _complete_frame_document(row) -> dict:
|
||||
"""Rebuild one complete serving document from canonical SQLite data."""
|
||||
vec = unpack_vector(row["embedding_f16"], int(row["embedding_dim"]))
|
||||
return {
|
||||
"id": row["id"],
|
||||
"media_id": row["media_id"],
|
||||
"title": row["title"],
|
||||
"source_path": row["source_path"],
|
||||
"timestamp_ms": row["timestamp_ms"],
|
||||
"thumb_path": row["thumb_path"],
|
||||
"model_id": row["model_id"],
|
||||
"caption": row["served_caption"],
|
||||
"caption_model": row["served_caption_model"],
|
||||
"captioned_at": row["served_captioned_at"],
|
||||
"caption_route": row["served_caption_route"],
|
||||
"caption_route_reason": row["served_caption_route_reason"],
|
||||
"route_score": row["served_route_score"],
|
||||
"route_model": row["served_route_model"],
|
||||
"route_policy": row["served_route_policy"],
|
||||
"routed_at": row["served_routed_at"],
|
||||
"caption_window_id": row["caption_window_id"],
|
||||
"caption_window_start_ms": row["caption_window_start_ms"],
|
||||
"caption_window_end_ms": row["caption_window_end_ms"],
|
||||
"source_kind": row["source_kind"],
|
||||
"jellyfin_item_id": row["jellyfin_item_id"],
|
||||
"jellyfin_media_source_id": row["jellyfin_media_source_id"],
|
||||
"jellyfin_library_id": row["jellyfin_library_id"],
|
||||
"jellyfin_item_type": row["jellyfin_item_type"],
|
||||
"jellyfin_tags": tags_from_json(row["jellyfin_tags"]),
|
||||
"trickplay_width": row["trickplay_width"],
|
||||
"_vectors": {"siglip2": vec.tolist()},
|
||||
}
|
||||
|
||||
|
||||
def count_caption_repair_documents(db_path=None) -> int:
|
||||
"""Count only frames touched by captioning; never require a full reindex."""
|
||||
db_path = db_path or settings.db_path
|
||||
init_db(db_path)
|
||||
with connect(db_path) as con:
|
||||
row = con.execute(
|
||||
"""
|
||||
SELECT COUNT(*) FROM (
|
||||
SELECT frame_id FROM caption_window_frames
|
||||
UNION
|
||||
SELECT id FROM frames WHERE caption IS NOT NULL
|
||||
)
|
||||
"""
|
||||
).fetchone()
|
||||
return int(row[0])
|
||||
|
||||
|
||||
def repair_caption_documents(
|
||||
*,
|
||||
batch_size: int = 250,
|
||||
progress: SyncProgress | None = None,
|
||||
meili: Meili | None = None,
|
||||
db_path=None,
|
||||
) -> int:
|
||||
"""Restore caption-touched documents overwritten by older POST updates.
|
||||
|
||||
Uses existing SQLite captions, vectors and Jellyfin identity without
|
||||
clearing Meilisearch, re-embedding or invoking Ollama. Each replacement
|
||||
task is awaited so failed repairs are not silently reported as complete.
|
||||
"""
|
||||
db_path = db_path or settings.db_path
|
||||
init_db(db_path)
|
||||
client = meili or _meili_client()
|
||||
batch_size = max(1, int(batch_size))
|
||||
total = 0
|
||||
query = (
|
||||
"WITH repair_ids AS ("
|
||||
" SELECT frame_id AS id FROM caption_window_frames"
|
||||
" UNION SELECT id FROM frames WHERE caption IS NOT NULL"
|
||||
") "
|
||||
+ _FRAME_DOCUMENT_SELECT.replace(
|
||||
"FROM frames f", "FROM repair_ids r JOIN frames f ON f.id = r.id", 1
|
||||
)
|
||||
+ " ORDER BY f.id"
|
||||
)
|
||||
with connect(db_path) as con:
|
||||
cursor = con.execute(query)
|
||||
while True:
|
||||
rows = cursor.fetchmany(batch_size)
|
||||
if not rows:
|
||||
break
|
||||
docs = [_complete_frame_document(row) for row in rows]
|
||||
client.add_documents(docs, wait=True)
|
||||
total += len(docs)
|
||||
if progress is not None:
|
||||
progress({"stage": "repair", "documents_processed": total})
|
||||
return total
|
||||
|
||||
|
||||
def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
|
||||
"""Rebuild the disposable Meilisearch index from canonical SQLite."""
|
||||
init_db(settings.db_path)
|
||||
meili = _meili_client()
|
||||
|
||||
with connect(settings.db_path) as con:
|
||||
dims = [row[0] for row in con.execute(
|
||||
"SELECT DISTINCT embedding_dim FROM media"
|
||||
)]
|
||||
if not dims:
|
||||
meili.clear_documents()
|
||||
if progress is not None:
|
||||
progress({"stage": "sync", "documents_processed": 0})
|
||||
return 0
|
||||
if len(dims) != 1:
|
||||
raise RuntimeError(
|
||||
f"Multiple embedding dimensions found: {dims}. "
|
||||
"Use separate indexes per model."
|
||||
)
|
||||
|
||||
meili.configure_user_vectors(int(dims[0]))
|
||||
meili.clear_documents()
|
||||
|
||||
cursor = con.execute(_FRAME_DOCUMENT_SELECT + " ORDER BY f.id")
|
||||
total = 0
|
||||
while True:
|
||||
rows = cursor.fetchmany(100)
|
||||
@@ -339,41 +433,7 @@ def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
|
||||
|
||||
docs = []
|
||||
for row in rows:
|
||||
vec = unpack_vector(
|
||||
row["embedding_f16"],
|
||||
row["embedding_dim"],
|
||||
)
|
||||
docs.append(
|
||||
{
|
||||
"id": row["id"],
|
||||
"media_id": row["media_id"],
|
||||
"title": row["title"],
|
||||
"source_path": row["source_path"],
|
||||
"timestamp_ms": row["timestamp_ms"],
|
||||
"thumb_path": row["thumb_path"],
|
||||
"model_id": row["model_id"],
|
||||
"caption": row["served_caption"],
|
||||
"caption_model": row["served_caption_model"],
|
||||
"captioned_at": row["served_captioned_at"],
|
||||
"caption_route": row["served_caption_route"],
|
||||
"caption_route_reason": row["served_caption_route_reason"],
|
||||
"route_score": row["served_route_score"],
|
||||
"route_model": row["served_route_model"],
|
||||
"route_policy": row["served_route_policy"],
|
||||
"routed_at": row["served_routed_at"],
|
||||
"caption_window_id": row["caption_window_id"],
|
||||
"caption_window_start_ms": row["caption_window_start_ms"],
|
||||
"caption_window_end_ms": row["caption_window_end_ms"],
|
||||
"source_kind": row["source_kind"],
|
||||
"jellyfin_item_id": row["jellyfin_item_id"],
|
||||
"jellyfin_media_source_id": row["jellyfin_media_source_id"],
|
||||
"jellyfin_library_id": row["jellyfin_library_id"],
|
||||
"jellyfin_item_type": row["jellyfin_item_type"],
|
||||
"jellyfin_tags": tags_from_json(row["jellyfin_tags"]),
|
||||
"trickplay_width": row["trickplay_width"],
|
||||
"_vectors": {"siglip2": vec.tolist()},
|
||||
}
|
||||
)
|
||||
docs.append(_complete_frame_document(row))
|
||||
|
||||
meili.add_documents(docs)
|
||||
total += len(docs)
|
||||
|
||||
Reference in New Issue
Block a user