Add focused reconstruction of caption-damaged documents from SQLite

This commit is contained in:
Sleeps
2026-10-08 10:53:36 +01:00
parent b48b1b5ed0
commit d76c9678ff
+122 -62
View File
@@ -264,31 +264,9 @@ class CaptionMeiliSync:
return caught_up
def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
"""Rebuild the disposable Meilisearch index from canonical SQLite."""
init_db(settings.db_path)
meili = _meili_client()
with connect(settings.db_path) as con:
dims = [row[0] for row in con.execute(
"SELECT DISTINCT embedding_dim FROM media"
)]
if not dims:
meili.clear_documents()
if progress is not None:
progress({"stage": "sync", "documents_processed": 0})
return 0
if len(dims) != 1:
raise RuntimeError(
f"Multiple embedding dimensions found: {dims}. "
"Use separate indexes per model."
)
meili.configure_user_vectors(int(dims[0]))
meili.clear_documents()
cursor = con.execute(
"""
# Keep the full replacement representation identical for initial indexing
# and targeted caption-document recovery.
_FRAME_DOCUMENT_SELECT = """
SELECT
f.id, f.media_id, f.timestamp_ms, f.thumb_path, f.embedding_f16,
CASE WHEN cw.representative_frame_id = f.id
@@ -327,10 +305,126 @@ def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
JOIN media m ON m.id = f.media_id
LEFT JOIN caption_window_frames cwf ON cwf.frame_id = f.id
LEFT JOIN caption_windows cw ON cw.id = cwf.window_id
ORDER BY f.id
"""
)
def _complete_frame_document(row) -> dict:
"""Rebuild one complete serving document from canonical SQLite data."""
vec = unpack_vector(row["embedding_f16"], int(row["embedding_dim"]))
return {
"id": row["id"],
"media_id": row["media_id"],
"title": row["title"],
"source_path": row["source_path"],
"timestamp_ms": row["timestamp_ms"],
"thumb_path": row["thumb_path"],
"model_id": row["model_id"],
"caption": row["served_caption"],
"caption_model": row["served_caption_model"],
"captioned_at": row["served_captioned_at"],
"caption_route": row["served_caption_route"],
"caption_route_reason": row["served_caption_route_reason"],
"route_score": row["served_route_score"],
"route_model": row["served_route_model"],
"route_policy": row["served_route_policy"],
"routed_at": row["served_routed_at"],
"caption_window_id": row["caption_window_id"],
"caption_window_start_ms": row["caption_window_start_ms"],
"caption_window_end_ms": row["caption_window_end_ms"],
"source_kind": row["source_kind"],
"jellyfin_item_id": row["jellyfin_item_id"],
"jellyfin_media_source_id": row["jellyfin_media_source_id"],
"jellyfin_library_id": row["jellyfin_library_id"],
"jellyfin_item_type": row["jellyfin_item_type"],
"jellyfin_tags": tags_from_json(row["jellyfin_tags"]),
"trickplay_width": row["trickplay_width"],
"_vectors": {"siglip2": vec.tolist()},
}
def count_caption_repair_documents(db_path=None) -> int:
"""Count only frames touched by captioning; never require a full reindex."""
db_path = db_path or settings.db_path
init_db(db_path)
with connect(db_path) as con:
row = con.execute(
"""
SELECT COUNT(*) FROM (
SELECT frame_id FROM caption_window_frames
UNION
SELECT id FROM frames WHERE caption IS NOT NULL
)
"""
).fetchone()
return int(row[0])
def repair_caption_documents(
*,
batch_size: int = 250,
progress: SyncProgress | None = None,
meili: Meili | None = None,
db_path=None,
) -> int:
"""Restore caption-touched documents overwritten by older POST updates.
Uses existing SQLite captions, vectors and Jellyfin identity without
clearing Meilisearch, re-embedding or invoking Ollama. Each replacement
task is awaited so failed repairs are not silently reported as complete.
"""
db_path = db_path or settings.db_path
init_db(db_path)
client = meili or _meili_client()
batch_size = max(1, int(batch_size))
total = 0
query = (
"WITH repair_ids AS ("
" SELECT frame_id AS id FROM caption_window_frames"
" UNION SELECT id FROM frames WHERE caption IS NOT NULL"
") "
+ _FRAME_DOCUMENT_SELECT.replace(
"FROM frames f", "FROM repair_ids r JOIN frames f ON f.id = r.id", 1
)
+ " ORDER BY f.id"
)
with connect(db_path) as con:
cursor = con.execute(query)
while True:
rows = cursor.fetchmany(batch_size)
if not rows:
break
docs = [_complete_frame_document(row) for row in rows]
client.add_documents(docs, wait=True)
total += len(docs)
if progress is not None:
progress({"stage": "repair", "documents_processed": total})
return total
def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
"""Rebuild the disposable Meilisearch index from canonical SQLite."""
init_db(settings.db_path)
meili = _meili_client()
with connect(settings.db_path) as con:
dims = [row[0] for row in con.execute(
"SELECT DISTINCT embedding_dim FROM media"
)]
if not dims:
meili.clear_documents()
if progress is not None:
progress({"stage": "sync", "documents_processed": 0})
return 0
if len(dims) != 1:
raise RuntimeError(
f"Multiple embedding dimensions found: {dims}. "
"Use separate indexes per model."
)
meili.configure_user_vectors(int(dims[0]))
meili.clear_documents()
cursor = con.execute(_FRAME_DOCUMENT_SELECT + " ORDER BY f.id")
total = 0
while True:
rows = cursor.fetchmany(100)
@@ -339,41 +433,7 @@ def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
docs = []
for row in rows:
vec = unpack_vector(
row["embedding_f16"],
row["embedding_dim"],
)
docs.append(
{
"id": row["id"],
"media_id": row["media_id"],
"title": row["title"],
"source_path": row["source_path"],
"timestamp_ms": row["timestamp_ms"],
"thumb_path": row["thumb_path"],
"model_id": row["model_id"],
"caption": row["served_caption"],
"caption_model": row["served_caption_model"],
"captioned_at": row["served_captioned_at"],
"caption_route": row["served_caption_route"],
"caption_route_reason": row["served_caption_route_reason"],
"route_score": row["served_route_score"],
"route_model": row["served_route_model"],
"route_policy": row["served_route_policy"],
"routed_at": row["served_routed_at"],
"caption_window_id": row["caption_window_id"],
"caption_window_start_ms": row["caption_window_start_ms"],
"caption_window_end_ms": row["caption_window_end_ms"],
"source_kind": row["source_kind"],
"jellyfin_item_id": row["jellyfin_item_id"],
"jellyfin_media_source_id": row["jellyfin_media_source_id"],
"jellyfin_library_id": row["jellyfin_library_id"],
"jellyfin_item_type": row["jellyfin_item_type"],
"jellyfin_tags": tags_from_json(row["jellyfin_tags"]),
"trickplay_width": row["trickplay_width"],
"_vectors": {"siglip2": vec.tolist()},
}
)
docs.append(_complete_frame_document(row))
meili.add_documents(docs)
total += len(docs)