Files
jellyfin-scene-search/scene_search/sync.py
T

672 lines
25 KiB
Python

from __future__ import annotations
import time
from collections.abc import Callable
from typing import Any
from .config import settings
from .database import connect, init_db, unpack_vector
from .jellyfin import tags_from_json
from .meili import Meili
SyncProgress = Callable[[dict], None]
def _meili_client() -> Meili:
return Meili(
settings.meili_url,
settings.meili_master_key,
settings.meili_index,
)
def sync_frame_metadata(
frame_ids: list[str],
*,
meili: Meili | None = None,
) -> int:
"""Push caption/window/routing metadata for existing Meili documents only."""
unique_ids = list(dict.fromkeys(str(value) for value in frame_ids if value))
if not unique_ids:
return 0
init_db(settings.db_path)
placeholders = ",".join("?" for _ in unique_ids)
with connect(settings.db_path) as con:
rows = con.execute(
f"""
SELECT
f.id,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.caption ELSE f.caption END AS served_caption,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.caption_model ELSE f.caption_model END AS served_caption_model,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.captioned_at ELSE f.captioned_at END AS served_captioned_at,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.caption_route ELSE f.caption_route END AS served_caption_route,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.caption_route_reason ELSE f.caption_route_reason
END AS served_caption_route_reason,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.route_score ELSE f.route_score END AS served_route_score,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.route_model ELSE f.route_model END AS served_route_model,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.route_policy ELSE f.route_policy END AS served_route_policy,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.routed_at ELSE f.routed_at END AS served_routed_at,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.id ELSE NULL END AS caption_window_id,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.start_ms ELSE NULL END AS caption_window_start_ms,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.end_ms ELSE NULL END AS caption_window_end_ms
FROM frames f
LEFT JOIN caption_window_frames cwf ON cwf.frame_id = f.id
LEFT JOIN caption_windows cw ON cw.id = cwf.window_id
WHERE f.id IN ({placeholders})
""",
unique_ids,
).fetchall()
if not rows:
return 0
documents = [
{
"id": row["id"],
"caption": row["served_caption"],
"caption_model": row["served_caption_model"],
"captioned_at": row["served_captioned_at"],
"caption_route": row["served_caption_route"],
"caption_route_reason": row["served_caption_route_reason"],
"route_score": row["served_route_score"],
"route_model": row["served_route_model"],
"route_policy": row["served_route_policy"],
"routed_at": row["served_routed_at"],
"caption_window_id": row["caption_window_id"],
"caption_window_start_ms": row["caption_window_start_ms"],
"caption_window_end_ms": row["caption_window_end_ms"],
}
for row in rows
]
client = meili or _meili_client()
client.update_documents(documents, wait=True)
synced_ids = [str(row["id"]) for row in rows]
placeholders = ",".join("?" for _ in synced_ids)
with connect(settings.db_path) as con:
con.execute(
f"""
UPDATE frames
SET meili_synced_at = CURRENT_TIMESTAMP
WHERE id IN ({placeholders})
""",
synced_ids,
)
con.execute(
f"""
UPDATE caption_windows
SET meili_synced_at = CURRENT_TIMESTAMP
WHERE representative_frame_id IN ({placeholders})
AND caption IS NOT NULL
""",
synced_ids,
)
return len(synced_ids)
def unsynced_caption_frame_ids(
library_ids: list[str],
*,
limit: int,
) -> list[str]:
if not library_ids:
return []
init_db(settings.db_path)
placeholders = ",".join("?" for _ in library_ids)
with connect(settings.db_path) as con:
rows = con.execute(
f"""
SELECT f.id
FROM frames f
JOIN media m ON m.id = f.media_id
LEFT JOIN caption_window_frames cwf ON cwf.frame_id = f.id
LEFT JOIN caption_windows cw ON cw.id = cwf.window_id
WHERE m.source_kind = 'jellyfin'
AND m.jellyfin_library_id IN ({placeholders})
AND f.meili_synced_at IS NULL
AND (f.caption IS NOT NULL OR cw.caption IS NOT NULL)
ORDER BY COALESCE(cw.captioned_at, f.captioned_at), f.id
LIMIT ?
""",
[*library_ids, limit],
).fetchall()
return [str(row["id"]) for row in rows]
def count_unsynced_caption_frames(library_ids: list[str]) -> int:
if not library_ids:
return 0
init_db(settings.db_path)
placeholders = ",".join("?" for _ in library_ids)
with connect(settings.db_path) as con:
row = con.execute(
f"""
SELECT COUNT(*) AS total
FROM frames f
JOIN media m ON m.id = f.media_id
LEFT JOIN caption_window_frames cwf ON cwf.frame_id = f.id
LEFT JOIN caption_windows cw ON cw.id = cwf.window_id
WHERE m.source_kind = 'jellyfin'
AND m.jellyfin_library_id IN ({placeholders})
AND f.meili_synced_at IS NULL
AND (f.caption IS NOT NULL OR cw.caption IS NOT NULL)
""",
library_ids,
).fetchone()
return int(row["total"] or 0)
class CaptionMeiliSync:
"""Incrementally sync caption metadata while SQLite remains canonical."""
def __init__(
self,
library_ids: list[str],
*,
meili: Meili,
batch_size: int,
progress: Callable[[dict[str, Any]], None] | None = None,
) -> None:
self.library_ids = list(library_ids)
self.meili = meili
self.batch_size = max(1, int(batch_size))
self.progress = progress
self.pending: list[str] = []
self.updated_total = 0
self.last_error: str | None = None
def _report(self, *, pending: int | None = None) -> None:
if self.progress is None:
return
update: dict[str, Any] = {
"meili_documents_updated": self.updated_total,
"meili_sync_error": self.last_error,
}
if pending is not None:
update["meili_sync_pending"] = pending
self.progress(update)
def add(self, frame_id: str) -> None:
self.pending.append(str(frame_id))
if len(self.pending) >= self.batch_size:
self.flush()
def flush(self) -> int:
if not self.pending:
return 0
frame_ids = self.pending
self.pending = []
try:
count = sync_frame_metadata(frame_ids, meili=self.meili)
except Exception as exc:
self.last_error = str(exc)
self._report(pending=len(frame_ids))
print(f"WARN caption Meilisearch batch update failed: {exc}")
return 0
self.updated_total += count
self.last_error = None
self._report()
print(
f"Updated {count} caption document(s) in Meilisearch "
f"({self.updated_total} this run)."
)
return count
def catch_up(self) -> int:
"""Sync previously saved-but-unsynced captions in bounded batches."""
caught_up = 0
while True:
frame_ids = unsynced_caption_frame_ids(
self.library_ids,
limit=self.batch_size,
)
if not frame_ids:
break
try:
count = sync_frame_metadata(frame_ids, meili=self.meili)
except Exception as exc:
self.last_error = str(exc)
self._report()
print(f"WARN caption Meilisearch catch-up failed: {exc}")
break
caught_up += count
self.updated_total += count
self.last_error = None
self._report()
if count == 0:
break
pending = (
count_unsynced_caption_frames(self.library_ids)
if self.last_error
else 0
)
self._report(pending=pending)
return caught_up
# Keep the full replacement representation identical for initial indexing
# and targeted caption-document recovery.
_FRAME_DOCUMENT_SELECT = """
SELECT
f.id, f.media_id, f.timestamp_ms, f.thumb_path, f.embedding_f16,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.caption ELSE f.caption END AS served_caption,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.caption_model ELSE f.caption_model END
AS served_caption_model,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.captioned_at ELSE f.captioned_at END
AS served_captioned_at,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.caption_route ELSE f.caption_route END
AS served_caption_route,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.caption_route_reason ELSE f.caption_route_reason END
AS served_caption_route_reason,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.route_score ELSE f.route_score END AS served_route_score,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.route_model ELSE f.route_model END AS served_route_model,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.route_policy ELSE f.route_policy END AS served_route_policy,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.routed_at ELSE f.routed_at END AS served_routed_at,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.id ELSE NULL END AS caption_window_id,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.start_ms ELSE NULL END AS caption_window_start_ms,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.end_ms ELSE NULL END AS caption_window_end_ms,
m.title, m.source_path, m.embedding_dim, m.model_id,
m.source_kind, m.jellyfin_item_id, m.jellyfin_media_source_id,
m.jellyfin_library_id, m.jellyfin_item_type, m.jellyfin_tags,
m.trickplay_width
FROM frames f
JOIN media m ON m.id = f.media_id
LEFT JOIN caption_window_frames cwf ON cwf.frame_id = f.id
LEFT JOIN caption_windows cw ON cw.id = cwf.window_id
"""
def _complete_frame_document(row) -> dict:
"""Rebuild one complete serving document from canonical SQLite data."""
vec = unpack_vector(row["embedding_f16"], int(row["embedding_dim"]))
return {
"id": row["id"],
"media_id": row["media_id"],
"title": row["title"],
"source_path": row["source_path"],
"timestamp_ms": row["timestamp_ms"],
"thumb_path": row["thumb_path"],
"model_id": row["model_id"],
"caption": row["served_caption"],
"caption_model": row["served_caption_model"],
"captioned_at": row["served_captioned_at"],
"caption_route": row["served_caption_route"],
"caption_route_reason": row["served_caption_route_reason"],
"route_score": row["served_route_score"],
"route_model": row["served_route_model"],
"route_policy": row["served_route_policy"],
"routed_at": row["served_routed_at"],
"caption_window_id": row["caption_window_id"],
"caption_window_start_ms": row["caption_window_start_ms"],
"caption_window_end_ms": row["caption_window_end_ms"],
"source_kind": row["source_kind"],
"jellyfin_item_id": row["jellyfin_item_id"],
"jellyfin_media_source_id": row["jellyfin_media_source_id"],
"jellyfin_library_id": row["jellyfin_library_id"],
"jellyfin_item_type": row["jellyfin_item_type"],
"jellyfin_tags": tags_from_json(row["jellyfin_tags"]),
"trickplay_width": row["trickplay_width"],
"_vectors": {"siglip2": vec.tolist()},
}
def count_caption_repair_documents(db_path=None) -> int:
"""Count only frames touched by captioning; never require a full reindex."""
db_path = db_path or settings.db_path
init_db(db_path)
with connect(db_path) as con:
row = con.execute(
"""
SELECT COUNT(*) FROM (
SELECT frame_id FROM caption_window_frames
UNION
SELECT id FROM frames WHERE caption IS NOT NULL
)
"""
).fetchone()
return int(row[0])
_REPAIR_CHECKPOINT_SCHEMA = """
CREATE TABLE IF NOT EXISTS caption_index_repair_checkpoints (
index_uid TEXT PRIMARY KEY,
last_frame_id TEXT,
completed_documents INTEGER NOT NULL DEFAULT 0,
pending_task_uid INTEGER,
pending_last_frame_id TEXT,
pending_count INTEGER NOT NULL DEFAULT 0,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
)
"""
def _repair_batch(db_path, after_frame_id: str | None, batch_size: int):
"""Fetch the next bounded batch without holding a SQLite cursor over IO."""
query = (
"WITH repair_ids AS ("
" SELECT frame_id AS id FROM caption_window_frames"
" UNION SELECT id FROM frames WHERE caption IS NOT NULL"
") "
+ _FRAME_DOCUMENT_SELECT.replace(
"FROM frames f", "FROM repair_ids r JOIN frames f ON f.id = r.id", 1
)
+ " WHERE (? IS NULL OR f.id > ?) ORDER BY f.id LIMIT ?"
)
with connect(db_path) as con:
return con.execute(
query, (after_frame_id, after_frame_id, batch_size)
).fetchall()
def _ensure_repair_checkpoint(db_path, index_uid: str) -> dict:
with connect(db_path) as con:
con.execute(_REPAIR_CHECKPOINT_SCHEMA)
con.execute(
"""
INSERT OR IGNORE INTO caption_index_repair_checkpoints (index_uid)
VALUES (?)
""",
(index_uid,),
)
row = con.execute(
"SELECT * FROM caption_index_repair_checkpoints WHERE index_uid = ?",
(index_uid,),
).fetchone()
return dict(row)
def _record_pending_repair(
db_path, index_uid: str, task_uid: int, last_frame_id: str, count: int
) -> None:
with connect(db_path) as con:
con.execute(
"""
UPDATE caption_index_repair_checkpoints
SET pending_task_uid = ?, pending_last_frame_id = ?,
pending_count = ?, updated_at = CURRENT_TIMESTAMP
WHERE index_uid = ? AND pending_task_uid IS NULL
""",
(task_uid, last_frame_id, count, index_uid),
)
if con.execute("SELECT changes()").fetchone()[0] != 1:
raise RuntimeError("Repair checkpoint changed unexpectedly")
def _finish_pending_repair(db_path, index_uid: str, task_uid: int) -> int:
"""Advance the durable cursor only once Meilisearch confirms success."""
with connect(db_path) as con:
con.execute(
"""
UPDATE caption_index_repair_checkpoints
SET last_frame_id = pending_last_frame_id,
completed_documents = completed_documents + pending_count,
pending_task_uid = NULL, pending_last_frame_id = NULL,
pending_count = 0, updated_at = CURRENT_TIMESTAMP
WHERE index_uid = ? AND pending_task_uid = ?
""",
(index_uid, task_uid),
)
if con.execute("SELECT changes()").fetchone()[0] != 1:
raise RuntimeError("Repair checkpoint changed unexpectedly")
return int(con.execute(
"""
SELECT completed_documents FROM caption_index_repair_checkpoints
WHERE index_uid = ?
""", (index_uid,)
).fetchone()[0])
def repair_caption_documents(
*,
batch_size: int = 250,
progress: SyncProgress | None = None,
meili: Meili | None = None,
db_path=None,
resume_task_uid: int | None = None,
) -> int:
"""Resumable repair of only caption-affected Meilisearch documents.
A durable SQLite checkpoint records each pending Meilisearch task *before*
waiting on it. If interrupted, the next invocation waits for that same
task, then continues at the last verified frame. Slow queues never trigger
the 120-second caption-sync timeout.
"""
db_path = db_path or settings.db_path
init_db(db_path)
client = meili or _meili_client()
index_uid = str(client.index)
batch_size = max(1, int(batch_size))
restored_this_run = 0
checkpoint = _ensure_repair_checkpoint(db_path, index_uid)
if resume_task_uid is not None:
if checkpoint["pending_task_uid"] is not None:
raise ValueError("An existing repair task is already checkpointed")
if checkpoint["last_frame_id"] is not None:
raise ValueError(
"--resume-task is only for adopting the original first batch "
"of an older, non-resumable repair"
)
rows = _repair_batch(db_path, None, batch_size)
if not rows:
raise ValueError("No documents exist to associate with that task")
task = client.get_task(resume_task_uid)
if task.get("indexUid") != index_uid:
raise ValueError(
f"Task {resume_task_uid} belongs to index "
f"{task.get('indexUid')!r}, not {index_uid!r}"
)
if task.get("type") != "documentAdditionOrUpdate":
raise ValueError(
f"Task {resume_task_uid} is not a document update task"
)
if task.get("status") in {"failed", "canceled"}:
raise RuntimeError(
f"Task {resume_task_uid} already {task.get('status')}; "
"rerun without --resume-task to resubmit that batch"
)
details = task.get("details") or {}
expected = details.get("receivedDocuments")
if expected is not None and int(expected) != len(rows):
raise ValueError(
f"Task {resume_task_uid} contains {expected} documents, "
f"but the first repair batch contains {len(rows)}"
)
_record_pending_repair(
db_path, index_uid, resume_task_uid, str(rows[-1]["id"]), len(rows)
)
print(
f"Adopted existing Meilisearch task {resume_task_uid} "
f"({len(rows)} documents); no duplicate batch submitted.",
flush=True,
)
while True:
checkpoint = _ensure_repair_checkpoint(db_path, index_uid)
task_uid = checkpoint["pending_task_uid"]
if task_uid is None:
rows = _repair_batch(
db_path, checkpoint["last_frame_id"], batch_size
)
if not rows:
break
docs = [_complete_frame_document(row) for row in rows]
# Wait=False returns the asynchronous task id. Persist it before
# waiting so a container restart cannot forget queued work.
task_uid = client.add_documents(docs, wait=False)
if task_uid is None:
raise RuntimeError(
"Meilisearch did not return a task ID; cannot resume safely"
)
task_uid = int(task_uid)
_record_pending_repair(
db_path, index_uid, task_uid, str(rows[-1]["id"]), len(rows)
)
print(
f"Queued repair task {task_uid}: {len(rows)} documents "
f"(already confirmed: {checkpoint['completed_documents']:,}).",
flush=True,
)
last_update = time.monotonic() - 16
def on_poll(task: dict) -> None:
nonlocal last_update
now = time.monotonic()
if now - last_update >= 15:
status = str(task.get("status") or "unknown")
print(
f"Waiting for Meilisearch task {task_uid}: {status} "
f"(repair remains resumable with Ctrl+C).",
flush=True,
)
last_update = now
# Unlike ordinary caption sync, a repair intentionally waits as long
# as the index needs. Re-run the command after Ctrl+C or a restart:
# the task UID is already stored in SQLite.
try:
client.wait_for_task(
int(task_uid),
timeout_seconds=None,
poll_seconds=3.0,
on_poll=on_poll,
)
except RuntimeError:
# Failed/canceled tasks are safe to resubmit. Preserve the
# checkpoint on API/network errors to avoid duplicate jobs.
try:
task = client.get_task(int(task_uid))
except Exception:
raise
if task.get("status") in {"failed", "canceled"}:
with connect(db_path) as con:
con.execute(
"""
UPDATE caption_index_repair_checkpoints
SET pending_task_uid = NULL, pending_last_frame_id = NULL,
pending_count = 0, updated_at = CURRENT_TIMESTAMP
WHERE index_uid = ? AND pending_task_uid = ?
""",
(index_uid, task_uid),
)
raise
count = int(checkpoint["pending_count"]) if checkpoint["pending_task_uid"] else len(rows)
confirmed_total = _finish_pending_repair(db_path, index_uid, int(task_uid))
restored_this_run += count
if progress is not None:
progress({
"stage": "repair",
"documents_processed": confirmed_total,
"completed_this_run": restored_this_run,
"task_uid": task_uid,
})
print(
f"Repair index {index_uid}: {checkpoint['completed_documents']:,} "
"documents previously confirmed before this run.",
flush=True,
)
return restored_this_run
def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
"""Rebuild the disposable Meilisearch index from canonical SQLite."""
init_db(settings.db_path)
meili = _meili_client()
with connect(settings.db_path) as con:
dims = [row[0] for row in con.execute(
"SELECT DISTINCT embedding_dim FROM media"
)]
if not dims:
meili.clear_documents()
if progress is not None:
progress({"stage": "sync", "documents_processed": 0})
return 0
if len(dims) != 1:
raise RuntimeError(
f"Multiple embedding dimensions found: {dims}. "
"Use separate indexes per model."
)
meili.configure_user_vectors(int(dims[0]))
meili.clear_documents()
cursor = con.execute(_FRAME_DOCUMENT_SELECT + " ORDER BY f.id")
total = 0
while True:
rows = cursor.fetchmany(100)
if not rows:
break
docs = []
for row in rows:
docs.append(_complete_frame_document(row))
meili.add_documents(docs)
total += len(docs)
if progress is not None:
progress(
{
"stage": "sync",
"documents_processed": total,
}
)
with connect(settings.db_path) as con:
con.execute(
"""
UPDATE frames
SET meili_synced_at = CURRENT_TIMESTAMP
"""
)
con.execute(
"""
UPDATE caption_windows
SET meili_synced_at = CURRENT_TIMESTAMP
WHERE caption IS NOT NULL
"""
)
if progress is not None:
progress({"stage": "synced", "documents_processed": total})
return total