Add focused reconstruction of caption-damaged documents from SQLite

This commit is contained in:
Sleeps
2026-10-08 10:53:36 +01:00
parent b48b1b5ed0
commit d76c9678ff
+100 -40
View File
@@ -264,31 +264,9 @@ class CaptionMeiliSync:
return caught_up return caught_up
def rebuild_meili(*, progress: SyncProgress | None = None) -> int: # Keep the full replacement representation identical for initial indexing
"""Rebuild the disposable Meilisearch index from canonical SQLite.""" # and targeted caption-document recovery.
init_db(settings.db_path) _FRAME_DOCUMENT_SELECT = """
meili = _meili_client()
with connect(settings.db_path) as con:
dims = [row[0] for row in con.execute(
"SELECT DISTINCT embedding_dim FROM media"
)]
if not dims:
meili.clear_documents()
if progress is not None:
progress({"stage": "sync", "documents_processed": 0})
return 0
if len(dims) != 1:
raise RuntimeError(
f"Multiple embedding dimensions found: {dims}. "
"Use separate indexes per model."
)
meili.configure_user_vectors(int(dims[0]))
meili.clear_documents()
cursor = con.execute(
"""
SELECT SELECT
f.id, f.media_id, f.timestamp_ms, f.thumb_path, f.embedding_f16, f.id, f.media_id, f.timestamp_ms, f.thumb_path, f.embedding_f16,
CASE WHEN cw.representative_frame_id = f.id CASE WHEN cw.representative_frame_id = f.id
@@ -327,24 +305,13 @@ def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
JOIN media m ON m.id = f.media_id JOIN media m ON m.id = f.media_id
LEFT JOIN caption_window_frames cwf ON cwf.frame_id = f.id LEFT JOIN caption_window_frames cwf ON cwf.frame_id = f.id
LEFT JOIN caption_windows cw ON cw.id = cwf.window_id LEFT JOIN caption_windows cw ON cw.id = cwf.window_id
ORDER BY f.id
""" """
)
total = 0
while True:
rows = cursor.fetchmany(100)
if not rows:
break
docs = [] def _complete_frame_document(row) -> dict:
for row in rows: """Rebuild one complete serving document from canonical SQLite data."""
vec = unpack_vector( vec = unpack_vector(row["embedding_f16"], int(row["embedding_dim"]))
row["embedding_f16"], return {
row["embedding_dim"],
)
docs.append(
{
"id": row["id"], "id": row["id"],
"media_id": row["media_id"], "media_id": row["media_id"],
"title": row["title"], "title": row["title"],
@@ -373,7 +340,100 @@ def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
"trickplay_width": row["trickplay_width"], "trickplay_width": row["trickplay_width"],
"_vectors": {"siglip2": vec.tolist()}, "_vectors": {"siglip2": vec.tolist()},
} }
def count_caption_repair_documents(db_path=None) -> int:
"""Count only frames touched by captioning; never require a full reindex."""
db_path = db_path or settings.db_path
init_db(db_path)
with connect(db_path) as con:
row = con.execute(
"""
SELECT COUNT(*) FROM (
SELECT frame_id FROM caption_window_frames
UNION
SELECT id FROM frames WHERE caption IS NOT NULL
) )
"""
).fetchone()
return int(row[0])
def repair_caption_documents(
*,
batch_size: int = 250,
progress: SyncProgress | None = None,
meili: Meili | None = None,
db_path=None,
) -> int:
"""Restore caption-touched documents overwritten by older POST updates.
Uses existing SQLite captions, vectors and Jellyfin identity without
clearing Meilisearch, re-embedding or invoking Ollama. Each replacement
task is awaited so failed repairs are not silently reported as complete.
"""
db_path = db_path or settings.db_path
init_db(db_path)
client = meili or _meili_client()
batch_size = max(1, int(batch_size))
total = 0
query = (
"WITH repair_ids AS ("
" SELECT frame_id AS id FROM caption_window_frames"
" UNION SELECT id FROM frames WHERE caption IS NOT NULL"
") "
+ _FRAME_DOCUMENT_SELECT.replace(
"FROM frames f", "FROM repair_ids r JOIN frames f ON f.id = r.id", 1
)
+ " ORDER BY f.id"
)
with connect(db_path) as con:
cursor = con.execute(query)
while True:
rows = cursor.fetchmany(batch_size)
if not rows:
break
docs = [_complete_frame_document(row) for row in rows]
client.add_documents(docs, wait=True)
total += len(docs)
if progress is not None:
progress({"stage": "repair", "documents_processed": total})
return total
def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
"""Rebuild the disposable Meilisearch index from canonical SQLite."""
init_db(settings.db_path)
meili = _meili_client()
with connect(settings.db_path) as con:
dims = [row[0] for row in con.execute(
"SELECT DISTINCT embedding_dim FROM media"
)]
if not dims:
meili.clear_documents()
if progress is not None:
progress({"stage": "sync", "documents_processed": 0})
return 0
if len(dims) != 1:
raise RuntimeError(
f"Multiple embedding dimensions found: {dims}. "
"Use separate indexes per model."
)
meili.configure_user_vectors(int(dims[0]))
meili.clear_documents()
cursor = con.execute(_FRAME_DOCUMENT_SELECT + " ORDER BY f.id")
total = 0
while True:
rows = cursor.fetchmany(100)
if not rows:
break
docs = []
for row in rows:
docs.append(_complete_frame_document(row))
meili.add_documents(docs) meili.add_documents(docs)
total += len(docs) total += len(docs)