Files
jellyfin-scene-search/scene_search/sync.py
T

326 lines
10 KiB
Python

from __future__ import annotations
from collections.abc import Callable
from typing import Any
from .config import settings
from .database import connect, init_db, unpack_vector
from .jellyfin import tags_from_json
from .meili import Meili
SyncProgress = Callable[[dict], None]
def _meili_client() -> Meili:
return Meili(
settings.meili_url,
settings.meili_master_key,
settings.meili_index,
)
def sync_frame_metadata(
frame_ids: list[str],
*,
meili: Meili | None = None,
) -> int:
"""Push caption/routing metadata for existing Meili documents only."""
unique_ids = list(dict.fromkeys(str(value) for value in frame_ids if value))
if not unique_ids:
return 0
init_db(settings.db_path)
placeholders = ",".join("?" for _ in unique_ids)
with connect(settings.db_path) as con:
rows = con.execute(
f"""
SELECT id, caption, caption_model, captioned_at,
caption_route, caption_route_reason, route_score,
route_model, route_policy, routed_at
FROM frames
WHERE id IN ({placeholders})
""",
unique_ids,
).fetchall()
if not rows:
return 0
documents = [
{
"id": row["id"],
"caption": row["caption"],
"caption_model": row["caption_model"],
"captioned_at": row["captioned_at"],
"caption_route": row["caption_route"],
"caption_route_reason": row["caption_route_reason"],
"route_score": row["route_score"],
"route_model": row["route_model"],
"route_policy": row["route_policy"],
"routed_at": row["routed_at"],
}
for row in rows
]
client = meili or _meili_client()
client.update_documents(documents)
synced_ids = [str(row["id"]) for row in rows]
placeholders = ",".join("?" for _ in synced_ids)
with connect(settings.db_path) as con:
con.execute(
f"""
UPDATE frames
SET meili_synced_at = CURRENT_TIMESTAMP
WHERE id IN ({placeholders})
""",
synced_ids,
)
return len(synced_ids)
def unsynced_caption_frame_ids(
library_ids: list[str],
*,
limit: int,
) -> list[str]:
if not library_ids:
return []
init_db(settings.db_path)
placeholders = ",".join("?" for _ in library_ids)
with connect(settings.db_path) as con:
rows = con.execute(
f"""
SELECT f.id
FROM frames f
JOIN media m ON m.id = f.media_id
WHERE m.source_kind = 'jellyfin'
AND m.jellyfin_library_id IN ({placeholders})
AND f.caption IS NOT NULL
AND f.meili_synced_at IS NULL
ORDER BY f.captioned_at, f.id
LIMIT ?
""",
[*library_ids, limit],
).fetchall()
return [str(row["id"]) for row in rows]
def count_unsynced_caption_frames(library_ids: list[str]) -> int:
if not library_ids:
return 0
init_db(settings.db_path)
placeholders = ",".join("?" for _ in library_ids)
with connect(settings.db_path) as con:
row = con.execute(
f"""
SELECT COUNT(*) AS total
FROM frames f
JOIN media m ON m.id = f.media_id
WHERE m.source_kind = 'jellyfin'
AND m.jellyfin_library_id IN ({placeholders})
AND f.caption IS NOT NULL
AND f.meili_synced_at IS NULL
""",
library_ids,
).fetchone()
return int(row["total"] or 0)
class CaptionMeiliSync:
"""Incrementally sync caption metadata while SQLite remains canonical."""
def __init__(
self,
library_ids: list[str],
*,
meili: Meili,
batch_size: int,
progress: Callable[[dict[str, Any]], None] | None = None,
) -> None:
self.library_ids = list(library_ids)
self.meili = meili
self.batch_size = max(1, int(batch_size))
self.progress = progress
self.pending: list[str] = []
self.updated_total = 0
self.last_error: str | None = None
def _report(self, *, pending: int | None = None) -> None:
if self.progress is None:
return
update: dict[str, Any] = {
"meili_documents_updated": self.updated_total,
"meili_sync_error": self.last_error,
}
if pending is not None:
update["meili_sync_pending"] = pending
self.progress(update)
def add(self, frame_id: str) -> None:
self.pending.append(str(frame_id))
if len(self.pending) >= self.batch_size:
self.flush()
def flush(self) -> int:
if not self.pending:
return 0
frame_ids = self.pending
self.pending = []
try:
count = sync_frame_metadata(frame_ids, meili=self.meili)
except Exception as exc:
self.last_error = str(exc)
self._report()
print(f"WARN caption Meilisearch batch update failed: {exc}")
return 0
self.updated_total += count
self.last_error = None
self._report()
print(
f"Updated {count} caption document(s) in Meilisearch "
f"({self.updated_total} this run)."
)
return count
def catch_up(self) -> int:
"""Sync previously saved-but-unsynced captions in bounded batches."""
caught_up = 0
while True:
frame_ids = unsynced_caption_frame_ids(
self.library_ids,
limit=self.batch_size,
)
if not frame_ids:
break
try:
count = sync_frame_metadata(frame_ids, meili=self.meili)
except Exception as exc:
self.last_error = str(exc)
self._report()
print(f"WARN caption Meilisearch catch-up failed: {exc}")
break
caught_up += count
self.updated_total += count
self.last_error = None
self._report()
if count == 0:
break
pending = (
count_unsynced_caption_frames(self.library_ids)
if self.last_error
else 0
)
self._report(pending=pending)
return caught_up
def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
"""Rebuild the disposable Meilisearch index from canonical SQLite."""
init_db(settings.db_path)
meili = _meili_client()
with connect(settings.db_path) as con:
dims = [row[0] for row in con.execute(
"SELECT DISTINCT embedding_dim FROM media"
)]
if not dims:
meili.clear_documents()
if progress is not None:
progress({"stage": "sync", "documents_processed": 0})
return 0
if len(dims) != 1:
raise RuntimeError(
f"Multiple embedding dimensions found: {dims}. "
"Use separate indexes per model."
)
meili.configure_user_vectors(int(dims[0]))
meili.clear_documents()
cursor = con.execute(
"""
SELECT f.id, f.media_id, f.timestamp_ms, f.thumb_path, f.embedding_f16,
f.caption, f.caption_model, f.captioned_at,
f.caption_route, f.caption_route_reason, f.route_score,
f.route_model, f.route_policy, f.routed_at,
m.title, m.source_path, m.embedding_dim, m.model_id,
m.source_kind, m.jellyfin_item_id, m.jellyfin_media_source_id,
m.jellyfin_library_id, m.jellyfin_item_type, m.jellyfin_tags,
m.trickplay_width
FROM frames f
JOIN media m ON m.id = f.media_id
ORDER BY f.id
"""
)
total = 0
while True:
rows = cursor.fetchmany(100)
if not rows:
break
docs = []
for row in rows:
vec = unpack_vector(
row["embedding_f16"],
row["embedding_dim"],
)
docs.append(
{
"id": row["id"],
"media_id": row["media_id"],
"title": row["title"],
"source_path": row["source_path"],
"timestamp_ms": row["timestamp_ms"],
"thumb_path": row["thumb_path"],
"model_id": row["model_id"],
"caption": row["caption"],
"caption_model": row["caption_model"],
"captioned_at": row["captioned_at"],
"caption_route": row["caption_route"],
"caption_route_reason": row["caption_route_reason"],
"route_score": row["route_score"],
"route_model": row["route_model"],
"route_policy": row["route_policy"],
"routed_at": row["routed_at"],
"source_kind": row["source_kind"],
"jellyfin_item_id": row["jellyfin_item_id"],
"jellyfin_media_source_id": row["jellyfin_media_source_id"],
"jellyfin_library_id": row["jellyfin_library_id"],
"jellyfin_item_type": row["jellyfin_item_type"],
"jellyfin_tags": tags_from_json(row["jellyfin_tags"]),
"trickplay_width": row["trickplay_width"],
"_vectors": {"siglip2": vec.tolist()},
}
)
meili.add_documents(docs)
total += len(docs)
if progress is not None:
progress(
{
"stage": "sync",
"documents_processed": total,
}
)
with connect(settings.db_path) as con:
con.execute(
"""
UPDATE frames
SET meili_synced_at = CURRENT_TIMESTAMP
"""
)
if progress is not None:
progress({"stage": "synced", "documents_processed": total})
return total