mirror of
https://github.com/MrSleeps/jellyfin-scene-search.git
synced 2026-11-19 00:03:36 +00:00
Incrementally sync caption metadata to Meilisearch
This commit is contained in:
+216
-4
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from .config import settings
|
||||
from .database import connect, init_db, unpack_vector
|
||||
@@ -10,15 +11,218 @@ from .meili import Meili
|
||||
SyncProgress = Callable[[dict], None]
|
||||
|
||||
|
||||
def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
|
||||
"""Rebuild the disposable Meilisearch index from canonical SQLite."""
|
||||
init_db(settings.db_path)
|
||||
meili = Meili(
|
||||
def _meili_client() -> Meili:
|
||||
return Meili(
|
||||
settings.meili_url,
|
||||
settings.meili_master_key,
|
||||
settings.meili_index,
|
||||
)
|
||||
|
||||
|
||||
def sync_frame_metadata(
|
||||
frame_ids: list[str],
|
||||
*,
|
||||
meili: Meili | None = None,
|
||||
) -> int:
|
||||
"""Push caption/routing metadata for existing Meili documents only."""
|
||||
unique_ids = list(dict.fromkeys(str(value) for value in frame_ids if value))
|
||||
if not unique_ids:
|
||||
return 0
|
||||
|
||||
init_db(settings.db_path)
|
||||
placeholders = ",".join("?" for _ in unique_ids)
|
||||
with connect(settings.db_path) as con:
|
||||
rows = con.execute(
|
||||
f"""
|
||||
SELECT id, caption, caption_model, captioned_at,
|
||||
caption_route, caption_route_reason, route_score,
|
||||
route_model, route_policy, routed_at
|
||||
FROM frames
|
||||
WHERE id IN ({placeholders})
|
||||
""",
|
||||
unique_ids,
|
||||
).fetchall()
|
||||
|
||||
if not rows:
|
||||
return 0
|
||||
|
||||
documents = [
|
||||
{
|
||||
"id": row["id"],
|
||||
"caption": row["caption"],
|
||||
"caption_model": row["caption_model"],
|
||||
"captioned_at": row["captioned_at"],
|
||||
"caption_route": row["caption_route"],
|
||||
"caption_route_reason": row["caption_route_reason"],
|
||||
"route_score": row["route_score"],
|
||||
"route_model": row["route_model"],
|
||||
"route_policy": row["route_policy"],
|
||||
"routed_at": row["routed_at"],
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
client = meili or _meili_client()
|
||||
client.update_documents(documents)
|
||||
|
||||
synced_ids = [str(row["id"]) for row in rows]
|
||||
placeholders = ",".join("?" for _ in synced_ids)
|
||||
with connect(settings.db_path) as con:
|
||||
con.execute(
|
||||
f"""
|
||||
UPDATE frames
|
||||
SET meili_synced_at = CURRENT_TIMESTAMP
|
||||
WHERE id IN ({placeholders})
|
||||
""",
|
||||
synced_ids,
|
||||
)
|
||||
|
||||
return len(synced_ids)
|
||||
|
||||
|
||||
def unsynced_caption_frame_ids(
|
||||
library_ids: list[str],
|
||||
*,
|
||||
limit: int,
|
||||
) -> list[str]:
|
||||
if not library_ids:
|
||||
return []
|
||||
|
||||
init_db(settings.db_path)
|
||||
placeholders = ",".join("?" for _ in library_ids)
|
||||
with connect(settings.db_path) as con:
|
||||
rows = con.execute(
|
||||
f"""
|
||||
SELECT f.id
|
||||
FROM frames f
|
||||
JOIN media m ON m.id = f.media_id
|
||||
WHERE m.source_kind = 'jellyfin'
|
||||
AND m.jellyfin_library_id IN ({placeholders})
|
||||
AND f.caption IS NOT NULL
|
||||
AND f.meili_synced_at IS NULL
|
||||
ORDER BY f.captioned_at, f.id
|
||||
LIMIT ?
|
||||
""",
|
||||
[*library_ids, limit],
|
||||
).fetchall()
|
||||
return [str(row["id"]) for row in rows]
|
||||
|
||||
|
||||
def count_unsynced_caption_frames(library_ids: list[str]) -> int:
|
||||
if not library_ids:
|
||||
return 0
|
||||
|
||||
init_db(settings.db_path)
|
||||
placeholders = ",".join("?" for _ in library_ids)
|
||||
with connect(settings.db_path) as con:
|
||||
row = con.execute(
|
||||
f"""
|
||||
SELECT COUNT(*) AS total
|
||||
FROM frames f
|
||||
JOIN media m ON m.id = f.media_id
|
||||
WHERE m.source_kind = 'jellyfin'
|
||||
AND m.jellyfin_library_id IN ({placeholders})
|
||||
AND f.caption IS NOT NULL
|
||||
AND f.meili_synced_at IS NULL
|
||||
""",
|
||||
library_ids,
|
||||
).fetchone()
|
||||
return int(row["total"] or 0)
|
||||
|
||||
|
||||
class CaptionMeiliSync:
|
||||
"""Incrementally sync caption metadata while SQLite remains canonical."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
library_ids: list[str],
|
||||
*,
|
||||
meili: Meili,
|
||||
batch_size: int,
|
||||
progress: Callable[[dict[str, Any]], None] | None = None,
|
||||
) -> None:
|
||||
self.library_ids = list(library_ids)
|
||||
self.meili = meili
|
||||
self.batch_size = max(1, int(batch_size))
|
||||
self.progress = progress
|
||||
self.pending: list[str] = []
|
||||
self.updated_total = 0
|
||||
self.last_error: str | None = None
|
||||
|
||||
def _report(self) -> None:
|
||||
if self.progress is None:
|
||||
return
|
||||
self.progress(
|
||||
{
|
||||
"meili_documents_updated": self.updated_total,
|
||||
"meili_sync_pending": count_unsynced_caption_frames(
|
||||
self.library_ids
|
||||
),
|
||||
"meili_sync_error": self.last_error,
|
||||
}
|
||||
)
|
||||
|
||||
def add(self, frame_id: str) -> None:
|
||||
self.pending.append(str(frame_id))
|
||||
if len(self.pending) >= self.batch_size:
|
||||
self.flush()
|
||||
|
||||
def flush(self) -> int:
|
||||
if not self.pending:
|
||||
return 0
|
||||
|
||||
frame_ids = self.pending
|
||||
self.pending = []
|
||||
try:
|
||||
count = sync_frame_metadata(frame_ids, meili=self.meili)
|
||||
except Exception as exc:
|
||||
self.last_error = str(exc)
|
||||
self._report()
|
||||
print(f"WARN caption Meilisearch batch update failed: {exc}")
|
||||
return 0
|
||||
|
||||
self.updated_total += count
|
||||
self.last_error = None
|
||||
self._report()
|
||||
print(
|
||||
f"Updated {count} caption document(s) in Meilisearch "
|
||||
f"({self.updated_total} this run)."
|
||||
)
|
||||
return count
|
||||
|
||||
def catch_up(self) -> int:
|
||||
"""Sync previously saved-but-unsynced captions in bounded batches."""
|
||||
caught_up = 0
|
||||
while True:
|
||||
frame_ids = unsynced_caption_frame_ids(
|
||||
self.library_ids,
|
||||
limit=self.batch_size,
|
||||
)
|
||||
if not frame_ids:
|
||||
break
|
||||
try:
|
||||
count = sync_frame_metadata(frame_ids, meili=self.meili)
|
||||
except Exception as exc:
|
||||
self.last_error = str(exc)
|
||||
self._report()
|
||||
print(f"WARN caption Meilisearch catch-up failed: {exc}")
|
||||
break
|
||||
|
||||
caught_up += count
|
||||
self.updated_total += count
|
||||
self.last_error = None
|
||||
self._report()
|
||||
if count == 0:
|
||||
break
|
||||
|
||||
return caught_up
|
||||
|
||||
|
||||
def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
|
||||
"""Rebuild the disposable Meilisearch index from canonical SQLite."""
|
||||
init_db(settings.db_path)
|
||||
meili = _meili_client()
|
||||
|
||||
with connect(settings.db_path) as con:
|
||||
dims = [row[0] for row in con.execute(
|
||||
"SELECT DISTINCT embedding_dim FROM media"
|
||||
@@ -104,6 +308,14 @@ def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
|
||||
}
|
||||
)
|
||||
|
||||
with connect(settings.db_path) as con:
|
||||
con.execute(
|
||||
"""
|
||||
UPDATE frames
|
||||
SET meili_synced_at = CURRENT_TIMESTAMP
|
||||
"""
|
||||
)
|
||||
|
||||
if progress is not None:
|
||||
progress({"stage": "synced", "documents_processed": total})
|
||||
return total
|
||||
|
||||
Reference in New Issue
Block a user