mirror of
https://github.com/MrSleeps/jellyfin-scene-search.git
synced 2026-11-20 00:33:36 +00:00
Add focused reconstruction of caption-damaged documents from SQLite
This commit is contained in:
+122
-62
@@ -264,31 +264,9 @@ class CaptionMeiliSync:
|
|||||||
return caught_up
|
return caught_up
|
||||||
|
|
||||||
|
|
||||||
def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
|
# Keep the full replacement representation identical for initial indexing
|
||||||
"""Rebuild the disposable Meilisearch index from canonical SQLite."""
|
# and targeted caption-document recovery.
|
||||||
init_db(settings.db_path)
|
_FRAME_DOCUMENT_SELECT = """
|
||||||
meili = _meili_client()
|
|
||||||
|
|
||||||
with connect(settings.db_path) as con:
|
|
||||||
dims = [row[0] for row in con.execute(
|
|
||||||
"SELECT DISTINCT embedding_dim FROM media"
|
|
||||||
)]
|
|
||||||
if not dims:
|
|
||||||
meili.clear_documents()
|
|
||||||
if progress is not None:
|
|
||||||
progress({"stage": "sync", "documents_processed": 0})
|
|
||||||
return 0
|
|
||||||
if len(dims) != 1:
|
|
||||||
raise RuntimeError(
|
|
||||||
f"Multiple embedding dimensions found: {dims}. "
|
|
||||||
"Use separate indexes per model."
|
|
||||||
)
|
|
||||||
|
|
||||||
meili.configure_user_vectors(int(dims[0]))
|
|
||||||
meili.clear_documents()
|
|
||||||
|
|
||||||
cursor = con.execute(
|
|
||||||
"""
|
|
||||||
SELECT
|
SELECT
|
||||||
f.id, f.media_id, f.timestamp_ms, f.thumb_path, f.embedding_f16,
|
f.id, f.media_id, f.timestamp_ms, f.thumb_path, f.embedding_f16,
|
||||||
CASE WHEN cw.representative_frame_id = f.id
|
CASE WHEN cw.representative_frame_id = f.id
|
||||||
@@ -327,10 +305,126 @@ def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
|
|||||||
JOIN media m ON m.id = f.media_id
|
JOIN media m ON m.id = f.media_id
|
||||||
LEFT JOIN caption_window_frames cwf ON cwf.frame_id = f.id
|
LEFT JOIN caption_window_frames cwf ON cwf.frame_id = f.id
|
||||||
LEFT JOIN caption_windows cw ON cw.id = cwf.window_id
|
LEFT JOIN caption_windows cw ON cw.id = cwf.window_id
|
||||||
ORDER BY f.id
|
|
||||||
"""
|
"""
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
|
def _complete_frame_document(row) -> dict:
|
||||||
|
"""Rebuild one complete serving document from canonical SQLite data."""
|
||||||
|
vec = unpack_vector(row["embedding_f16"], int(row["embedding_dim"]))
|
||||||
|
return {
|
||||||
|
"id": row["id"],
|
||||||
|
"media_id": row["media_id"],
|
||||||
|
"title": row["title"],
|
||||||
|
"source_path": row["source_path"],
|
||||||
|
"timestamp_ms": row["timestamp_ms"],
|
||||||
|
"thumb_path": row["thumb_path"],
|
||||||
|
"model_id": row["model_id"],
|
||||||
|
"caption": row["served_caption"],
|
||||||
|
"caption_model": row["served_caption_model"],
|
||||||
|
"captioned_at": row["served_captioned_at"],
|
||||||
|
"caption_route": row["served_caption_route"],
|
||||||
|
"caption_route_reason": row["served_caption_route_reason"],
|
||||||
|
"route_score": row["served_route_score"],
|
||||||
|
"route_model": row["served_route_model"],
|
||||||
|
"route_policy": row["served_route_policy"],
|
||||||
|
"routed_at": row["served_routed_at"],
|
||||||
|
"caption_window_id": row["caption_window_id"],
|
||||||
|
"caption_window_start_ms": row["caption_window_start_ms"],
|
||||||
|
"caption_window_end_ms": row["caption_window_end_ms"],
|
||||||
|
"source_kind": row["source_kind"],
|
||||||
|
"jellyfin_item_id": row["jellyfin_item_id"],
|
||||||
|
"jellyfin_media_source_id": row["jellyfin_media_source_id"],
|
||||||
|
"jellyfin_library_id": row["jellyfin_library_id"],
|
||||||
|
"jellyfin_item_type": row["jellyfin_item_type"],
|
||||||
|
"jellyfin_tags": tags_from_json(row["jellyfin_tags"]),
|
||||||
|
"trickplay_width": row["trickplay_width"],
|
||||||
|
"_vectors": {"siglip2": vec.tolist()},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def count_caption_repair_documents(db_path=None) -> int:
|
||||||
|
"""Count only frames touched by captioning; never require a full reindex."""
|
||||||
|
db_path = db_path or settings.db_path
|
||||||
|
init_db(db_path)
|
||||||
|
with connect(db_path) as con:
|
||||||
|
row = con.execute(
|
||||||
|
"""
|
||||||
|
SELECT COUNT(*) FROM (
|
||||||
|
SELECT frame_id FROM caption_window_frames
|
||||||
|
UNION
|
||||||
|
SELECT id FROM frames WHERE caption IS NOT NULL
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
).fetchone()
|
||||||
|
return int(row[0])
|
||||||
|
|
||||||
|
|
||||||
|
def repair_caption_documents(
|
||||||
|
*,
|
||||||
|
batch_size: int = 250,
|
||||||
|
progress: SyncProgress | None = None,
|
||||||
|
meili: Meili | None = None,
|
||||||
|
db_path=None,
|
||||||
|
) -> int:
|
||||||
|
"""Restore caption-touched documents overwritten by older POST updates.
|
||||||
|
|
||||||
|
Uses existing SQLite captions, vectors and Jellyfin identity without
|
||||||
|
clearing Meilisearch, re-embedding or invoking Ollama. Each replacement
|
||||||
|
task is awaited so failed repairs are not silently reported as complete.
|
||||||
|
"""
|
||||||
|
db_path = db_path or settings.db_path
|
||||||
|
init_db(db_path)
|
||||||
|
client = meili or _meili_client()
|
||||||
|
batch_size = max(1, int(batch_size))
|
||||||
|
total = 0
|
||||||
|
query = (
|
||||||
|
"WITH repair_ids AS ("
|
||||||
|
" SELECT frame_id AS id FROM caption_window_frames"
|
||||||
|
" UNION SELECT id FROM frames WHERE caption IS NOT NULL"
|
||||||
|
") "
|
||||||
|
+ _FRAME_DOCUMENT_SELECT.replace(
|
||||||
|
"FROM frames f", "FROM repair_ids r JOIN frames f ON f.id = r.id", 1
|
||||||
|
)
|
||||||
|
+ " ORDER BY f.id"
|
||||||
|
)
|
||||||
|
with connect(db_path) as con:
|
||||||
|
cursor = con.execute(query)
|
||||||
|
while True:
|
||||||
|
rows = cursor.fetchmany(batch_size)
|
||||||
|
if not rows:
|
||||||
|
break
|
||||||
|
docs = [_complete_frame_document(row) for row in rows]
|
||||||
|
client.add_documents(docs, wait=True)
|
||||||
|
total += len(docs)
|
||||||
|
if progress is not None:
|
||||||
|
progress({"stage": "repair", "documents_processed": total})
|
||||||
|
return total
|
||||||
|
|
||||||
|
|
||||||
|
def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
|
||||||
|
"""Rebuild the disposable Meilisearch index from canonical SQLite."""
|
||||||
|
init_db(settings.db_path)
|
||||||
|
meili = _meili_client()
|
||||||
|
|
||||||
|
with connect(settings.db_path) as con:
|
||||||
|
dims = [row[0] for row in con.execute(
|
||||||
|
"SELECT DISTINCT embedding_dim FROM media"
|
||||||
|
)]
|
||||||
|
if not dims:
|
||||||
|
meili.clear_documents()
|
||||||
|
if progress is not None:
|
||||||
|
progress({"stage": "sync", "documents_processed": 0})
|
||||||
|
return 0
|
||||||
|
if len(dims) != 1:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Multiple embedding dimensions found: {dims}. "
|
||||||
|
"Use separate indexes per model."
|
||||||
|
)
|
||||||
|
|
||||||
|
meili.configure_user_vectors(int(dims[0]))
|
||||||
|
meili.clear_documents()
|
||||||
|
|
||||||
|
cursor = con.execute(_FRAME_DOCUMENT_SELECT + " ORDER BY f.id")
|
||||||
total = 0
|
total = 0
|
||||||
while True:
|
while True:
|
||||||
rows = cursor.fetchmany(100)
|
rows = cursor.fetchmany(100)
|
||||||
@@ -339,41 +433,7 @@ def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
|
|||||||
|
|
||||||
docs = []
|
docs = []
|
||||||
for row in rows:
|
for row in rows:
|
||||||
vec = unpack_vector(
|
docs.append(_complete_frame_document(row))
|
||||||
row["embedding_f16"],
|
|
||||||
row["embedding_dim"],
|
|
||||||
)
|
|
||||||
docs.append(
|
|
||||||
{
|
|
||||||
"id": row["id"],
|
|
||||||
"media_id": row["media_id"],
|
|
||||||
"title": row["title"],
|
|
||||||
"source_path": row["source_path"],
|
|
||||||
"timestamp_ms": row["timestamp_ms"],
|
|
||||||
"thumb_path": row["thumb_path"],
|
|
||||||
"model_id": row["model_id"],
|
|
||||||
"caption": row["served_caption"],
|
|
||||||
"caption_model": row["served_caption_model"],
|
|
||||||
"captioned_at": row["served_captioned_at"],
|
|
||||||
"caption_route": row["served_caption_route"],
|
|
||||||
"caption_route_reason": row["served_caption_route_reason"],
|
|
||||||
"route_score": row["served_route_score"],
|
|
||||||
"route_model": row["served_route_model"],
|
|
||||||
"route_policy": row["served_route_policy"],
|
|
||||||
"routed_at": row["served_routed_at"],
|
|
||||||
"caption_window_id": row["caption_window_id"],
|
|
||||||
"caption_window_start_ms": row["caption_window_start_ms"],
|
|
||||||
"caption_window_end_ms": row["caption_window_end_ms"],
|
|
||||||
"source_kind": row["source_kind"],
|
|
||||||
"jellyfin_item_id": row["jellyfin_item_id"],
|
|
||||||
"jellyfin_media_source_id": row["jellyfin_media_source_id"],
|
|
||||||
"jellyfin_library_id": row["jellyfin_library_id"],
|
|
||||||
"jellyfin_item_type": row["jellyfin_item_type"],
|
|
||||||
"jellyfin_tags": tags_from_json(row["jellyfin_tags"]),
|
|
||||||
"trickplay_width": row["trickplay_width"],
|
|
||||||
"_vectors": {"siglip2": vec.tolist()},
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
meili.add_documents(docs)
|
meili.add_documents(docs)
|
||||||
total += len(docs)
|
total += len(docs)
|
||||||
|
|||||||
Reference in New Issue
Block a user