Incrementally sync caption metadata to Meilisearch

This commit is contained in:
Sleeps
2026-10-07 14:24:03 +01:00
parent 552db93fa6
commit fb4f827cd9
+216 -4
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
from collections.abc import Callable
from typing import Any
from .config import settings
from .database import connect, init_db, unpack_vector
@@ -10,15 +11,218 @@ from .meili import Meili
SyncProgress = Callable[[dict], None]
def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
"""Rebuild the disposable Meilisearch index from canonical SQLite."""
init_db(settings.db_path)
meili = Meili(
def _meili_client() -> Meili:
return Meili(
settings.meili_url,
settings.meili_master_key,
settings.meili_index,
)
def sync_frame_metadata(
frame_ids: list[str],
*,
meili: Meili | None = None,
) -> int:
"""Push caption/routing metadata for existing Meili documents only."""
unique_ids = list(dict.fromkeys(str(value) for value in frame_ids if value))
if not unique_ids:
return 0
init_db(settings.db_path)
placeholders = ",".join("?" for _ in unique_ids)
with connect(settings.db_path) as con:
rows = con.execute(
f"""
SELECT id, caption, caption_model, captioned_at,
caption_route, caption_route_reason, route_score,
route_model, route_policy, routed_at
FROM frames
WHERE id IN ({placeholders})
""",
unique_ids,
).fetchall()
if not rows:
return 0
documents = [
{
"id": row["id"],
"caption": row["caption"],
"caption_model": row["caption_model"],
"captioned_at": row["captioned_at"],
"caption_route": row["caption_route"],
"caption_route_reason": row["caption_route_reason"],
"route_score": row["route_score"],
"route_model": row["route_model"],
"route_policy": row["route_policy"],
"routed_at": row["routed_at"],
}
for row in rows
]
client = meili or _meili_client()
client.update_documents(documents)
synced_ids = [str(row["id"]) for row in rows]
placeholders = ",".join("?" for _ in synced_ids)
with connect(settings.db_path) as con:
con.execute(
f"""
UPDATE frames
SET meili_synced_at = CURRENT_TIMESTAMP
WHERE id IN ({placeholders})
""",
synced_ids,
)
return len(synced_ids)
def unsynced_caption_frame_ids(
library_ids: list[str],
*,
limit: int,
) -> list[str]:
if not library_ids:
return []
init_db(settings.db_path)
placeholders = ",".join("?" for _ in library_ids)
with connect(settings.db_path) as con:
rows = con.execute(
f"""
SELECT f.id
FROM frames f
JOIN media m ON m.id = f.media_id
WHERE m.source_kind = 'jellyfin'
AND m.jellyfin_library_id IN ({placeholders})
AND f.caption IS NOT NULL
AND f.meili_synced_at IS NULL
ORDER BY f.captioned_at, f.id
LIMIT ?
""",
[*library_ids, limit],
).fetchall()
return [str(row["id"]) for row in rows]
def count_unsynced_caption_frames(library_ids: list[str]) -> int:
if not library_ids:
return 0
init_db(settings.db_path)
placeholders = ",".join("?" for _ in library_ids)
with connect(settings.db_path) as con:
row = con.execute(
f"""
SELECT COUNT(*) AS total
FROM frames f
JOIN media m ON m.id = f.media_id
WHERE m.source_kind = 'jellyfin'
AND m.jellyfin_library_id IN ({placeholders})
AND f.caption IS NOT NULL
AND f.meili_synced_at IS NULL
""",
library_ids,
).fetchone()
return int(row["total"] or 0)
class CaptionMeiliSync:
"""Incrementally sync caption metadata while SQLite remains canonical."""
def __init__(
self,
library_ids: list[str],
*,
meili: Meili,
batch_size: int,
progress: Callable[[dict[str, Any]], None] | None = None,
) -> None:
self.library_ids = list(library_ids)
self.meili = meili
self.batch_size = max(1, int(batch_size))
self.progress = progress
self.pending: list[str] = []
self.updated_total = 0
self.last_error: str | None = None
def _report(self) -> None:
if self.progress is None:
return
self.progress(
{
"meili_documents_updated": self.updated_total,
"meili_sync_pending": count_unsynced_caption_frames(
self.library_ids
),
"meili_sync_error": self.last_error,
}
)
def add(self, frame_id: str) -> None:
self.pending.append(str(frame_id))
if len(self.pending) >= self.batch_size:
self.flush()
def flush(self) -> int:
if not self.pending:
return 0
frame_ids = self.pending
self.pending = []
try:
count = sync_frame_metadata(frame_ids, meili=self.meili)
except Exception as exc:
self.last_error = str(exc)
self._report()
print(f"WARN caption Meilisearch batch update failed: {exc}")
return 0
self.updated_total += count
self.last_error = None
self._report()
print(
f"Updated {count} caption document(s) in Meilisearch "
f"({self.updated_total} this run)."
)
return count
def catch_up(self) -> int:
"""Sync previously saved-but-unsynced captions in bounded batches."""
caught_up = 0
while True:
frame_ids = unsynced_caption_frame_ids(
self.library_ids,
limit=self.batch_size,
)
if not frame_ids:
break
try:
count = sync_frame_metadata(frame_ids, meili=self.meili)
except Exception as exc:
self.last_error = str(exc)
self._report()
print(f"WARN caption Meilisearch catch-up failed: {exc}")
break
caught_up += count
self.updated_total += count
self.last_error = None
self._report()
if count == 0:
break
return caught_up
def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
"""Rebuild the disposable Meilisearch index from canonical SQLite."""
init_db(settings.db_path)
meili = _meili_client()
with connect(settings.db_path) as con:
dims = [row[0] for row in con.execute(
"SELECT DISTINCT embedding_dim FROM media"
@@ -104,6 +308,14 @@ def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
}
)
with connect(settings.db_path) as con:
con.execute(
"""
UPDATE frames
SET meili_synced_at = CURRENT_TIMESTAMP
"""
)
if progress is not None:
progress({"stage": "synced", "documents_processed": total})
return total