mirror of
https://github.com/MrSleeps/jellyfin-scene-search.git
synced 2026-11-19 00:03:36 +00:00
326 lines
10 KiB
Python
326 lines
10 KiB
Python
from __future__ import annotations
|
|
|
|
from collections.abc import Callable
|
|
from typing import Any
|
|
|
|
from .config import settings
|
|
from .database import connect, init_db, unpack_vector
|
|
from .jellyfin import tags_from_json
|
|
from .meili import Meili
|
|
|
|
SyncProgress = Callable[[dict], None]
|
|
|
|
|
|
def _meili_client() -> Meili:
|
|
return Meili(
|
|
settings.meili_url,
|
|
settings.meili_master_key,
|
|
settings.meili_index,
|
|
)
|
|
|
|
|
|
def sync_frame_metadata(
|
|
frame_ids: list[str],
|
|
*,
|
|
meili: Meili | None = None,
|
|
) -> int:
|
|
"""Push caption/routing metadata for existing Meili documents only."""
|
|
unique_ids = list(dict.fromkeys(str(value) for value in frame_ids if value))
|
|
if not unique_ids:
|
|
return 0
|
|
|
|
init_db(settings.db_path)
|
|
placeholders = ",".join("?" for _ in unique_ids)
|
|
with connect(settings.db_path) as con:
|
|
rows = con.execute(
|
|
f"""
|
|
SELECT id, caption, caption_model, captioned_at,
|
|
caption_route, caption_route_reason, route_score,
|
|
route_model, route_policy, routed_at
|
|
FROM frames
|
|
WHERE id IN ({placeholders})
|
|
""",
|
|
unique_ids,
|
|
).fetchall()
|
|
|
|
if not rows:
|
|
return 0
|
|
|
|
documents = [
|
|
{
|
|
"id": row["id"],
|
|
"caption": row["caption"],
|
|
"caption_model": row["caption_model"],
|
|
"captioned_at": row["captioned_at"],
|
|
"caption_route": row["caption_route"],
|
|
"caption_route_reason": row["caption_route_reason"],
|
|
"route_score": row["route_score"],
|
|
"route_model": row["route_model"],
|
|
"route_policy": row["route_policy"],
|
|
"routed_at": row["routed_at"],
|
|
}
|
|
for row in rows
|
|
]
|
|
|
|
client = meili or _meili_client()
|
|
client.update_documents(documents, wait=True)
|
|
|
|
synced_ids = [str(row["id"]) for row in rows]
|
|
placeholders = ",".join("?" for _ in synced_ids)
|
|
with connect(settings.db_path) as con:
|
|
con.execute(
|
|
f"""
|
|
UPDATE frames
|
|
SET meili_synced_at = CURRENT_TIMESTAMP
|
|
WHERE id IN ({placeholders})
|
|
""",
|
|
synced_ids,
|
|
)
|
|
|
|
return len(synced_ids)
|
|
|
|
|
|
def unsynced_caption_frame_ids(
|
|
library_ids: list[str],
|
|
*,
|
|
limit: int,
|
|
) -> list[str]:
|
|
if not library_ids:
|
|
return []
|
|
|
|
init_db(settings.db_path)
|
|
placeholders = ",".join("?" for _ in library_ids)
|
|
with connect(settings.db_path) as con:
|
|
rows = con.execute(
|
|
f"""
|
|
SELECT f.id
|
|
FROM frames f
|
|
JOIN media m ON m.id = f.media_id
|
|
WHERE m.source_kind = 'jellyfin'
|
|
AND m.jellyfin_library_id IN ({placeholders})
|
|
AND f.caption IS NOT NULL
|
|
AND f.meili_synced_at IS NULL
|
|
ORDER BY f.captioned_at, f.id
|
|
LIMIT ?
|
|
""",
|
|
[*library_ids, limit],
|
|
).fetchall()
|
|
return [str(row["id"]) for row in rows]
|
|
|
|
|
|
def count_unsynced_caption_frames(library_ids: list[str]) -> int:
|
|
if not library_ids:
|
|
return 0
|
|
|
|
init_db(settings.db_path)
|
|
placeholders = ",".join("?" for _ in library_ids)
|
|
with connect(settings.db_path) as con:
|
|
row = con.execute(
|
|
f"""
|
|
SELECT COUNT(*) AS total
|
|
FROM frames f
|
|
JOIN media m ON m.id = f.media_id
|
|
WHERE m.source_kind = 'jellyfin'
|
|
AND m.jellyfin_library_id IN ({placeholders})
|
|
AND f.caption IS NOT NULL
|
|
AND f.meili_synced_at IS NULL
|
|
""",
|
|
library_ids,
|
|
).fetchone()
|
|
return int(row["total"] or 0)
|
|
|
|
|
|
class CaptionMeiliSync:
|
|
"""Incrementally sync caption metadata while SQLite remains canonical."""
|
|
|
|
def __init__(
|
|
self,
|
|
library_ids: list[str],
|
|
*,
|
|
meili: Meili,
|
|
batch_size: int,
|
|
progress: Callable[[dict[str, Any]], None] | None = None,
|
|
) -> None:
|
|
self.library_ids = list(library_ids)
|
|
self.meili = meili
|
|
self.batch_size = max(1, int(batch_size))
|
|
self.progress = progress
|
|
self.pending: list[str] = []
|
|
self.updated_total = 0
|
|
self.last_error: str | None = None
|
|
|
|
def _report(self, *, pending: int | None = None) -> None:
|
|
if self.progress is None:
|
|
return
|
|
update: dict[str, Any] = {
|
|
"meili_documents_updated": self.updated_total,
|
|
"meili_sync_error": self.last_error,
|
|
}
|
|
if pending is not None:
|
|
update["meili_sync_pending"] = pending
|
|
self.progress(update)
|
|
|
|
def add(self, frame_id: str) -> None:
|
|
self.pending.append(str(frame_id))
|
|
if len(self.pending) >= self.batch_size:
|
|
self.flush()
|
|
|
|
def flush(self) -> int:
|
|
if not self.pending:
|
|
return 0
|
|
|
|
frame_ids = self.pending
|
|
self.pending = []
|
|
try:
|
|
count = sync_frame_metadata(frame_ids, meili=self.meili)
|
|
except Exception as exc:
|
|
self.last_error = str(exc)
|
|
self._report()
|
|
print(f"WARN caption Meilisearch batch update failed: {exc}")
|
|
return 0
|
|
|
|
self.updated_total += count
|
|
self.last_error = None
|
|
self._report()
|
|
print(
|
|
f"Updated {count} caption document(s) in Meilisearch "
|
|
f"({self.updated_total} this run)."
|
|
)
|
|
return count
|
|
|
|
def catch_up(self) -> int:
|
|
"""Sync previously saved-but-unsynced captions in bounded batches."""
|
|
caught_up = 0
|
|
while True:
|
|
frame_ids = unsynced_caption_frame_ids(
|
|
self.library_ids,
|
|
limit=self.batch_size,
|
|
)
|
|
if not frame_ids:
|
|
break
|
|
try:
|
|
count = sync_frame_metadata(frame_ids, meili=self.meili)
|
|
except Exception as exc:
|
|
self.last_error = str(exc)
|
|
self._report()
|
|
print(f"WARN caption Meilisearch catch-up failed: {exc}")
|
|
break
|
|
|
|
caught_up += count
|
|
self.updated_total += count
|
|
self.last_error = None
|
|
self._report()
|
|
if count == 0:
|
|
break
|
|
|
|
pending = (
|
|
count_unsynced_caption_frames(self.library_ids)
|
|
if self.last_error
|
|
else 0
|
|
)
|
|
self._report(pending=pending)
|
|
return caught_up
|
|
|
|
|
|
def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
|
|
"""Rebuild the disposable Meilisearch index from canonical SQLite."""
|
|
init_db(settings.db_path)
|
|
meili = _meili_client()
|
|
|
|
with connect(settings.db_path) as con:
|
|
dims = [row[0] for row in con.execute(
|
|
"SELECT DISTINCT embedding_dim FROM media"
|
|
)]
|
|
if not dims:
|
|
meili.clear_documents()
|
|
if progress is not None:
|
|
progress({"stage": "sync", "documents_processed": 0})
|
|
return 0
|
|
if len(dims) != 1:
|
|
raise RuntimeError(
|
|
f"Multiple embedding dimensions found: {dims}. "
|
|
"Use separate indexes per model."
|
|
)
|
|
|
|
meili.configure_user_vectors(int(dims[0]))
|
|
meili.clear_documents()
|
|
|
|
cursor = con.execute(
|
|
"""
|
|
SELECT f.id, f.media_id, f.timestamp_ms, f.thumb_path, f.embedding_f16,
|
|
f.caption, f.caption_model, f.captioned_at,
|
|
f.caption_route, f.caption_route_reason, f.route_score,
|
|
f.route_model, f.route_policy, f.routed_at,
|
|
m.title, m.source_path, m.embedding_dim, m.model_id,
|
|
m.source_kind, m.jellyfin_item_id, m.jellyfin_media_source_id,
|
|
m.jellyfin_library_id, m.jellyfin_item_type, m.jellyfin_tags,
|
|
m.trickplay_width
|
|
FROM frames f
|
|
JOIN media m ON m.id = f.media_id
|
|
ORDER BY f.id
|
|
"""
|
|
)
|
|
|
|
total = 0
|
|
while True:
|
|
rows = cursor.fetchmany(100)
|
|
if not rows:
|
|
break
|
|
|
|
docs = []
|
|
for row in rows:
|
|
vec = unpack_vector(
|
|
row["embedding_f16"],
|
|
row["embedding_dim"],
|
|
)
|
|
docs.append(
|
|
{
|
|
"id": row["id"],
|
|
"media_id": row["media_id"],
|
|
"title": row["title"],
|
|
"source_path": row["source_path"],
|
|
"timestamp_ms": row["timestamp_ms"],
|
|
"thumb_path": row["thumb_path"],
|
|
"model_id": row["model_id"],
|
|
"caption": row["caption"],
|
|
"caption_model": row["caption_model"],
|
|
"captioned_at": row["captioned_at"],
|
|
"caption_route": row["caption_route"],
|
|
"caption_route_reason": row["caption_route_reason"],
|
|
"route_score": row["route_score"],
|
|
"route_model": row["route_model"],
|
|
"route_policy": row["route_policy"],
|
|
"routed_at": row["routed_at"],
|
|
"source_kind": row["source_kind"],
|
|
"jellyfin_item_id": row["jellyfin_item_id"],
|
|
"jellyfin_media_source_id": row["jellyfin_media_source_id"],
|
|
"jellyfin_library_id": row["jellyfin_library_id"],
|
|
"jellyfin_item_type": row["jellyfin_item_type"],
|
|
"jellyfin_tags": tags_from_json(row["jellyfin_tags"]),
|
|
"trickplay_width": row["trickplay_width"],
|
|
"_vectors": {"siglip2": vec.tolist()},
|
|
}
|
|
)
|
|
|
|
meili.add_documents(docs)
|
|
total += len(docs)
|
|
if progress is not None:
|
|
progress(
|
|
{
|
|
"stage": "sync",
|
|
"documents_processed": total,
|
|
}
|
|
)
|
|
|
|
with connect(settings.db_path) as con:
|
|
con.execute(
|
|
"""
|
|
UPDATE frames
|
|
SET meili_synced_at = CURRENT_TIMESTAMP
|
|
"""
|
|
)
|
|
|
|
if progress is not None:
|
|
progress({"stage": "synced", "documents_processed": total})
|
|
return total
|