mirror of
https://github.com/MrSleeps/jellyfin-scene-search.git
synced 2026-11-20 00:33:36 +00:00
Incrementally sync caption metadata to Meilisearch
This commit is contained in:
+216
-4
@@ -1,6 +1,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
from .config import settings
|
from .config import settings
|
||||||
from .database import connect, init_db, unpack_vector
|
from .database import connect, init_db, unpack_vector
|
||||||
@@ -10,15 +11,218 @@ from .meili import Meili
|
|||||||
SyncProgress = Callable[[dict], None]
|
SyncProgress = Callable[[dict], None]
|
||||||
|
|
||||||
|
|
||||||
def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
|
def _meili_client() -> Meili:
|
||||||
"""Rebuild the disposable Meilisearch index from canonical SQLite."""
|
return Meili(
|
||||||
init_db(settings.db_path)
|
|
||||||
meili = Meili(
|
|
||||||
settings.meili_url,
|
settings.meili_url,
|
||||||
settings.meili_master_key,
|
settings.meili_master_key,
|
||||||
settings.meili_index,
|
settings.meili_index,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def sync_frame_metadata(
|
||||||
|
frame_ids: list[str],
|
||||||
|
*,
|
||||||
|
meili: Meili | None = None,
|
||||||
|
) -> int:
|
||||||
|
"""Push caption/routing metadata for existing Meili documents only."""
|
||||||
|
unique_ids = list(dict.fromkeys(str(value) for value in frame_ids if value))
|
||||||
|
if not unique_ids:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
init_db(settings.db_path)
|
||||||
|
placeholders = ",".join("?" for _ in unique_ids)
|
||||||
|
with connect(settings.db_path) as con:
|
||||||
|
rows = con.execute(
|
||||||
|
f"""
|
||||||
|
SELECT id, caption, caption_model, captioned_at,
|
||||||
|
caption_route, caption_route_reason, route_score,
|
||||||
|
route_model, route_policy, routed_at
|
||||||
|
FROM frames
|
||||||
|
WHERE id IN ({placeholders})
|
||||||
|
""",
|
||||||
|
unique_ids,
|
||||||
|
).fetchall()
|
||||||
|
|
||||||
|
if not rows:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
documents = [
|
||||||
|
{
|
||||||
|
"id": row["id"],
|
||||||
|
"caption": row["caption"],
|
||||||
|
"caption_model": row["caption_model"],
|
||||||
|
"captioned_at": row["captioned_at"],
|
||||||
|
"caption_route": row["caption_route"],
|
||||||
|
"caption_route_reason": row["caption_route_reason"],
|
||||||
|
"route_score": row["route_score"],
|
||||||
|
"route_model": row["route_model"],
|
||||||
|
"route_policy": row["route_policy"],
|
||||||
|
"routed_at": row["routed_at"],
|
||||||
|
}
|
||||||
|
for row in rows
|
||||||
|
]
|
||||||
|
|
||||||
|
client = meili or _meili_client()
|
||||||
|
client.update_documents(documents)
|
||||||
|
|
||||||
|
synced_ids = [str(row["id"]) for row in rows]
|
||||||
|
placeholders = ",".join("?" for _ in synced_ids)
|
||||||
|
with connect(settings.db_path) as con:
|
||||||
|
con.execute(
|
||||||
|
f"""
|
||||||
|
UPDATE frames
|
||||||
|
SET meili_synced_at = CURRENT_TIMESTAMP
|
||||||
|
WHERE id IN ({placeholders})
|
||||||
|
""",
|
||||||
|
synced_ids,
|
||||||
|
)
|
||||||
|
|
||||||
|
return len(synced_ids)
|
||||||
|
|
||||||
|
|
||||||
|
def unsynced_caption_frame_ids(
|
||||||
|
library_ids: list[str],
|
||||||
|
*,
|
||||||
|
limit: int,
|
||||||
|
) -> list[str]:
|
||||||
|
if not library_ids:
|
||||||
|
return []
|
||||||
|
|
||||||
|
init_db(settings.db_path)
|
||||||
|
placeholders = ",".join("?" for _ in library_ids)
|
||||||
|
with connect(settings.db_path) as con:
|
||||||
|
rows = con.execute(
|
||||||
|
f"""
|
||||||
|
SELECT f.id
|
||||||
|
FROM frames f
|
||||||
|
JOIN media m ON m.id = f.media_id
|
||||||
|
WHERE m.source_kind = 'jellyfin'
|
||||||
|
AND m.jellyfin_library_id IN ({placeholders})
|
||||||
|
AND f.caption IS NOT NULL
|
||||||
|
AND f.meili_synced_at IS NULL
|
||||||
|
ORDER BY f.captioned_at, f.id
|
||||||
|
LIMIT ?
|
||||||
|
""",
|
||||||
|
[*library_ids, limit],
|
||||||
|
).fetchall()
|
||||||
|
return [str(row["id"]) for row in rows]
|
||||||
|
|
||||||
|
|
||||||
|
def count_unsynced_caption_frames(library_ids: list[str]) -> int:
|
||||||
|
if not library_ids:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
init_db(settings.db_path)
|
||||||
|
placeholders = ",".join("?" for _ in library_ids)
|
||||||
|
with connect(settings.db_path) as con:
|
||||||
|
row = con.execute(
|
||||||
|
f"""
|
||||||
|
SELECT COUNT(*) AS total
|
||||||
|
FROM frames f
|
||||||
|
JOIN media m ON m.id = f.media_id
|
||||||
|
WHERE m.source_kind = 'jellyfin'
|
||||||
|
AND m.jellyfin_library_id IN ({placeholders})
|
||||||
|
AND f.caption IS NOT NULL
|
||||||
|
AND f.meili_synced_at IS NULL
|
||||||
|
""",
|
||||||
|
library_ids,
|
||||||
|
).fetchone()
|
||||||
|
return int(row["total"] or 0)
|
||||||
|
|
||||||
|
|
||||||
|
class CaptionMeiliSync:
|
||||||
|
"""Incrementally sync caption metadata while SQLite remains canonical."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
library_ids: list[str],
|
||||||
|
*,
|
||||||
|
meili: Meili,
|
||||||
|
batch_size: int,
|
||||||
|
progress: Callable[[dict[str, Any]], None] | None = None,
|
||||||
|
) -> None:
|
||||||
|
self.library_ids = list(library_ids)
|
||||||
|
self.meili = meili
|
||||||
|
self.batch_size = max(1, int(batch_size))
|
||||||
|
self.progress = progress
|
||||||
|
self.pending: list[str] = []
|
||||||
|
self.updated_total = 0
|
||||||
|
self.last_error: str | None = None
|
||||||
|
|
||||||
|
def _report(self) -> None:
|
||||||
|
if self.progress is None:
|
||||||
|
return
|
||||||
|
self.progress(
|
||||||
|
{
|
||||||
|
"meili_documents_updated": self.updated_total,
|
||||||
|
"meili_sync_pending": count_unsynced_caption_frames(
|
||||||
|
self.library_ids
|
||||||
|
),
|
||||||
|
"meili_sync_error": self.last_error,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
def add(self, frame_id: str) -> None:
|
||||||
|
self.pending.append(str(frame_id))
|
||||||
|
if len(self.pending) >= self.batch_size:
|
||||||
|
self.flush()
|
||||||
|
|
||||||
|
def flush(self) -> int:
|
||||||
|
if not self.pending:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
frame_ids = self.pending
|
||||||
|
self.pending = []
|
||||||
|
try:
|
||||||
|
count = sync_frame_metadata(frame_ids, meili=self.meili)
|
||||||
|
except Exception as exc:
|
||||||
|
self.last_error = str(exc)
|
||||||
|
self._report()
|
||||||
|
print(f"WARN caption Meilisearch batch update failed: {exc}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
self.updated_total += count
|
||||||
|
self.last_error = None
|
||||||
|
self._report()
|
||||||
|
print(
|
||||||
|
f"Updated {count} caption document(s) in Meilisearch "
|
||||||
|
f"({self.updated_total} this run)."
|
||||||
|
)
|
||||||
|
return count
|
||||||
|
|
||||||
|
def catch_up(self) -> int:
|
||||||
|
"""Sync previously saved-but-unsynced captions in bounded batches."""
|
||||||
|
caught_up = 0
|
||||||
|
while True:
|
||||||
|
frame_ids = unsynced_caption_frame_ids(
|
||||||
|
self.library_ids,
|
||||||
|
limit=self.batch_size,
|
||||||
|
)
|
||||||
|
if not frame_ids:
|
||||||
|
break
|
||||||
|
try:
|
||||||
|
count = sync_frame_metadata(frame_ids, meili=self.meili)
|
||||||
|
except Exception as exc:
|
||||||
|
self.last_error = str(exc)
|
||||||
|
self._report()
|
||||||
|
print(f"WARN caption Meilisearch catch-up failed: {exc}")
|
||||||
|
break
|
||||||
|
|
||||||
|
caught_up += count
|
||||||
|
self.updated_total += count
|
||||||
|
self.last_error = None
|
||||||
|
self._report()
|
||||||
|
if count == 0:
|
||||||
|
break
|
||||||
|
|
||||||
|
return caught_up
|
||||||
|
|
||||||
|
|
||||||
|
def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
|
||||||
|
"""Rebuild the disposable Meilisearch index from canonical SQLite."""
|
||||||
|
init_db(settings.db_path)
|
||||||
|
meili = _meili_client()
|
||||||
|
|
||||||
with connect(settings.db_path) as con:
|
with connect(settings.db_path) as con:
|
||||||
dims = [row[0] for row in con.execute(
|
dims = [row[0] for row in con.execute(
|
||||||
"SELECT DISTINCT embedding_dim FROM media"
|
"SELECT DISTINCT embedding_dim FROM media"
|
||||||
@@ -104,6 +308,14 @@ def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
with connect(settings.db_path) as con:
|
||||||
|
con.execute(
|
||||||
|
"""
|
||||||
|
UPDATE frames
|
||||||
|
SET meili_synced_at = CURRENT_TIMESTAMP
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
if progress is not None:
|
if progress is not None:
|
||||||
progress({"stage": "synced", "documents_processed": total})
|
progress({"stage": "synced", "documents_processed": total})
|
||||||
return total
|
return total
|
||||||
|
|||||||
Reference in New Issue
Block a user