mirror of
https://github.com/MrSleeps/jellyfin-scene-search.git
synced 2026-11-19 00:03:36 +00:00
672 lines
25 KiB
Python
672 lines
25 KiB
Python
from __future__ import annotations
|
|
|
|
import time
|
|
from collections.abc import Callable
|
|
from typing import Any
|
|
|
|
from .config import settings
|
|
from .database import connect, init_db, unpack_vector
|
|
from .jellyfin import tags_from_json
|
|
from .meili import Meili
|
|
|
|
SyncProgress = Callable[[dict], None]
|
|
|
|
|
|
def _meili_client() -> Meili:
|
|
return Meili(
|
|
settings.meili_url,
|
|
settings.meili_master_key,
|
|
settings.meili_index,
|
|
)
|
|
|
|
|
|
def sync_frame_metadata(
|
|
frame_ids: list[str],
|
|
*,
|
|
meili: Meili | None = None,
|
|
) -> int:
|
|
"""Push caption/window/routing metadata for existing Meili documents only."""
|
|
unique_ids = list(dict.fromkeys(str(value) for value in frame_ids if value))
|
|
if not unique_ids:
|
|
return 0
|
|
|
|
init_db(settings.db_path)
|
|
placeholders = ",".join("?" for _ in unique_ids)
|
|
with connect(settings.db_path) as con:
|
|
rows = con.execute(
|
|
f"""
|
|
SELECT
|
|
f.id,
|
|
CASE WHEN cw.representative_frame_id = f.id
|
|
THEN cw.caption ELSE f.caption END AS served_caption,
|
|
CASE WHEN cw.representative_frame_id = f.id
|
|
THEN cw.caption_model ELSE f.caption_model END AS served_caption_model,
|
|
CASE WHEN cw.representative_frame_id = f.id
|
|
THEN cw.captioned_at ELSE f.captioned_at END AS served_captioned_at,
|
|
CASE WHEN cw.representative_frame_id = f.id
|
|
THEN cw.caption_route ELSE f.caption_route END AS served_caption_route,
|
|
CASE WHEN cw.representative_frame_id = f.id
|
|
THEN cw.caption_route_reason ELSE f.caption_route_reason
|
|
END AS served_caption_route_reason,
|
|
CASE WHEN cw.representative_frame_id = f.id
|
|
THEN cw.route_score ELSE f.route_score END AS served_route_score,
|
|
CASE WHEN cw.representative_frame_id = f.id
|
|
THEN cw.route_model ELSE f.route_model END AS served_route_model,
|
|
CASE WHEN cw.representative_frame_id = f.id
|
|
THEN cw.route_policy ELSE f.route_policy END AS served_route_policy,
|
|
CASE WHEN cw.representative_frame_id = f.id
|
|
THEN cw.routed_at ELSE f.routed_at END AS served_routed_at,
|
|
CASE WHEN cw.representative_frame_id = f.id
|
|
THEN cw.id ELSE NULL END AS caption_window_id,
|
|
CASE WHEN cw.representative_frame_id = f.id
|
|
THEN cw.start_ms ELSE NULL END AS caption_window_start_ms,
|
|
CASE WHEN cw.representative_frame_id = f.id
|
|
THEN cw.end_ms ELSE NULL END AS caption_window_end_ms
|
|
FROM frames f
|
|
LEFT JOIN caption_window_frames cwf ON cwf.frame_id = f.id
|
|
LEFT JOIN caption_windows cw ON cw.id = cwf.window_id
|
|
WHERE f.id IN ({placeholders})
|
|
""",
|
|
unique_ids,
|
|
).fetchall()
|
|
|
|
if not rows:
|
|
return 0
|
|
|
|
documents = [
|
|
{
|
|
"id": row["id"],
|
|
"caption": row["served_caption"],
|
|
"caption_model": row["served_caption_model"],
|
|
"captioned_at": row["served_captioned_at"],
|
|
"caption_route": row["served_caption_route"],
|
|
"caption_route_reason": row["served_caption_route_reason"],
|
|
"route_score": row["served_route_score"],
|
|
"route_model": row["served_route_model"],
|
|
"route_policy": row["served_route_policy"],
|
|
"routed_at": row["served_routed_at"],
|
|
"caption_window_id": row["caption_window_id"],
|
|
"caption_window_start_ms": row["caption_window_start_ms"],
|
|
"caption_window_end_ms": row["caption_window_end_ms"],
|
|
}
|
|
for row in rows
|
|
]
|
|
|
|
client = meili or _meili_client()
|
|
client.update_documents(documents, wait=True)
|
|
|
|
synced_ids = [str(row["id"]) for row in rows]
|
|
placeholders = ",".join("?" for _ in synced_ids)
|
|
with connect(settings.db_path) as con:
|
|
con.execute(
|
|
f"""
|
|
UPDATE frames
|
|
SET meili_synced_at = CURRENT_TIMESTAMP
|
|
WHERE id IN ({placeholders})
|
|
""",
|
|
synced_ids,
|
|
)
|
|
con.execute(
|
|
f"""
|
|
UPDATE caption_windows
|
|
SET meili_synced_at = CURRENT_TIMESTAMP
|
|
WHERE representative_frame_id IN ({placeholders})
|
|
AND caption IS NOT NULL
|
|
""",
|
|
synced_ids,
|
|
)
|
|
|
|
return len(synced_ids)
|
|
|
|
|
|
def unsynced_caption_frame_ids(
|
|
library_ids: list[str],
|
|
*,
|
|
limit: int,
|
|
) -> list[str]:
|
|
if not library_ids:
|
|
return []
|
|
|
|
init_db(settings.db_path)
|
|
placeholders = ",".join("?" for _ in library_ids)
|
|
with connect(settings.db_path) as con:
|
|
rows = con.execute(
|
|
f"""
|
|
SELECT f.id
|
|
FROM frames f
|
|
JOIN media m ON m.id = f.media_id
|
|
LEFT JOIN caption_window_frames cwf ON cwf.frame_id = f.id
|
|
LEFT JOIN caption_windows cw ON cw.id = cwf.window_id
|
|
WHERE m.source_kind = 'jellyfin'
|
|
AND m.jellyfin_library_id IN ({placeholders})
|
|
AND f.meili_synced_at IS NULL
|
|
AND (f.caption IS NOT NULL OR cw.caption IS NOT NULL)
|
|
ORDER BY COALESCE(cw.captioned_at, f.captioned_at), f.id
|
|
LIMIT ?
|
|
""",
|
|
[*library_ids, limit],
|
|
).fetchall()
|
|
return [str(row["id"]) for row in rows]
|
|
|
|
|
|
def count_unsynced_caption_frames(library_ids: list[str]) -> int:
|
|
if not library_ids:
|
|
return 0
|
|
|
|
init_db(settings.db_path)
|
|
placeholders = ",".join("?" for _ in library_ids)
|
|
with connect(settings.db_path) as con:
|
|
row = con.execute(
|
|
f"""
|
|
SELECT COUNT(*) AS total
|
|
FROM frames f
|
|
JOIN media m ON m.id = f.media_id
|
|
LEFT JOIN caption_window_frames cwf ON cwf.frame_id = f.id
|
|
LEFT JOIN caption_windows cw ON cw.id = cwf.window_id
|
|
WHERE m.source_kind = 'jellyfin'
|
|
AND m.jellyfin_library_id IN ({placeholders})
|
|
AND f.meili_synced_at IS NULL
|
|
AND (f.caption IS NOT NULL OR cw.caption IS NOT NULL)
|
|
""",
|
|
library_ids,
|
|
).fetchone()
|
|
return int(row["total"] or 0)
|
|
|
|
|
|
class CaptionMeiliSync:
|
|
"""Incrementally sync caption metadata while SQLite remains canonical."""
|
|
|
|
def __init__(
|
|
self,
|
|
library_ids: list[str],
|
|
*,
|
|
meili: Meili,
|
|
batch_size: int,
|
|
progress: Callable[[dict[str, Any]], None] | None = None,
|
|
) -> None:
|
|
self.library_ids = list(library_ids)
|
|
self.meili = meili
|
|
self.batch_size = max(1, int(batch_size))
|
|
self.progress = progress
|
|
self.pending: list[str] = []
|
|
self.updated_total = 0
|
|
self.last_error: str | None = None
|
|
|
|
def _report(self, *, pending: int | None = None) -> None:
|
|
if self.progress is None:
|
|
return
|
|
update: dict[str, Any] = {
|
|
"meili_documents_updated": self.updated_total,
|
|
"meili_sync_error": self.last_error,
|
|
}
|
|
if pending is not None:
|
|
update["meili_sync_pending"] = pending
|
|
self.progress(update)
|
|
|
|
def add(self, frame_id: str) -> None:
|
|
self.pending.append(str(frame_id))
|
|
if len(self.pending) >= self.batch_size:
|
|
self.flush()
|
|
|
|
def flush(self) -> int:
|
|
if not self.pending:
|
|
return 0
|
|
|
|
frame_ids = self.pending
|
|
self.pending = []
|
|
try:
|
|
count = sync_frame_metadata(frame_ids, meili=self.meili)
|
|
except Exception as exc:
|
|
self.last_error = str(exc)
|
|
self._report(pending=len(frame_ids))
|
|
print(f"WARN caption Meilisearch batch update failed: {exc}")
|
|
return 0
|
|
|
|
self.updated_total += count
|
|
self.last_error = None
|
|
self._report()
|
|
print(
|
|
f"Updated {count} caption document(s) in Meilisearch "
|
|
f"({self.updated_total} this run)."
|
|
)
|
|
return count
|
|
|
|
def catch_up(self) -> int:
|
|
"""Sync previously saved-but-unsynced captions in bounded batches."""
|
|
caught_up = 0
|
|
while True:
|
|
frame_ids = unsynced_caption_frame_ids(
|
|
self.library_ids,
|
|
limit=self.batch_size,
|
|
)
|
|
if not frame_ids:
|
|
break
|
|
try:
|
|
count = sync_frame_metadata(frame_ids, meili=self.meili)
|
|
except Exception as exc:
|
|
self.last_error = str(exc)
|
|
self._report()
|
|
print(f"WARN caption Meilisearch catch-up failed: {exc}")
|
|
break
|
|
|
|
caught_up += count
|
|
self.updated_total += count
|
|
self.last_error = None
|
|
self._report()
|
|
if count == 0:
|
|
break
|
|
|
|
pending = (
|
|
count_unsynced_caption_frames(self.library_ids)
|
|
if self.last_error
|
|
else 0
|
|
)
|
|
self._report(pending=pending)
|
|
return caught_up
|
|
|
|
|
|
# Keep the full replacement representation identical for initial indexing
|
|
# and targeted caption-document recovery.
|
|
_FRAME_DOCUMENT_SELECT = """
|
|
SELECT
|
|
f.id, f.media_id, f.timestamp_ms, f.thumb_path, f.embedding_f16,
|
|
CASE WHEN cw.representative_frame_id = f.id
|
|
THEN cw.caption ELSE f.caption END AS served_caption,
|
|
CASE WHEN cw.representative_frame_id = f.id
|
|
THEN cw.caption_model ELSE f.caption_model END
|
|
AS served_caption_model,
|
|
CASE WHEN cw.representative_frame_id = f.id
|
|
THEN cw.captioned_at ELSE f.captioned_at END
|
|
AS served_captioned_at,
|
|
CASE WHEN cw.representative_frame_id = f.id
|
|
THEN cw.caption_route ELSE f.caption_route END
|
|
AS served_caption_route,
|
|
CASE WHEN cw.representative_frame_id = f.id
|
|
THEN cw.caption_route_reason ELSE f.caption_route_reason END
|
|
AS served_caption_route_reason,
|
|
CASE WHEN cw.representative_frame_id = f.id
|
|
THEN cw.route_score ELSE f.route_score END AS served_route_score,
|
|
CASE WHEN cw.representative_frame_id = f.id
|
|
THEN cw.route_model ELSE f.route_model END AS served_route_model,
|
|
CASE WHEN cw.representative_frame_id = f.id
|
|
THEN cw.route_policy ELSE f.route_policy END AS served_route_policy,
|
|
CASE WHEN cw.representative_frame_id = f.id
|
|
THEN cw.routed_at ELSE f.routed_at END AS served_routed_at,
|
|
CASE WHEN cw.representative_frame_id = f.id
|
|
THEN cw.id ELSE NULL END AS caption_window_id,
|
|
CASE WHEN cw.representative_frame_id = f.id
|
|
THEN cw.start_ms ELSE NULL END AS caption_window_start_ms,
|
|
CASE WHEN cw.representative_frame_id = f.id
|
|
THEN cw.end_ms ELSE NULL END AS caption_window_end_ms,
|
|
m.title, m.source_path, m.embedding_dim, m.model_id,
|
|
m.source_kind, m.jellyfin_item_id, m.jellyfin_media_source_id,
|
|
m.jellyfin_library_id, m.jellyfin_item_type, m.jellyfin_tags,
|
|
m.trickplay_width
|
|
FROM frames f
|
|
JOIN media m ON m.id = f.media_id
|
|
LEFT JOIN caption_window_frames cwf ON cwf.frame_id = f.id
|
|
LEFT JOIN caption_windows cw ON cw.id = cwf.window_id
|
|
"""
|
|
|
|
|
|
def _complete_frame_document(row) -> dict:
|
|
"""Rebuild one complete serving document from canonical SQLite data."""
|
|
vec = unpack_vector(row["embedding_f16"], int(row["embedding_dim"]))
|
|
return {
|
|
"id": row["id"],
|
|
"media_id": row["media_id"],
|
|
"title": row["title"],
|
|
"source_path": row["source_path"],
|
|
"timestamp_ms": row["timestamp_ms"],
|
|
"thumb_path": row["thumb_path"],
|
|
"model_id": row["model_id"],
|
|
"caption": row["served_caption"],
|
|
"caption_model": row["served_caption_model"],
|
|
"captioned_at": row["served_captioned_at"],
|
|
"caption_route": row["served_caption_route"],
|
|
"caption_route_reason": row["served_caption_route_reason"],
|
|
"route_score": row["served_route_score"],
|
|
"route_model": row["served_route_model"],
|
|
"route_policy": row["served_route_policy"],
|
|
"routed_at": row["served_routed_at"],
|
|
"caption_window_id": row["caption_window_id"],
|
|
"caption_window_start_ms": row["caption_window_start_ms"],
|
|
"caption_window_end_ms": row["caption_window_end_ms"],
|
|
"source_kind": row["source_kind"],
|
|
"jellyfin_item_id": row["jellyfin_item_id"],
|
|
"jellyfin_media_source_id": row["jellyfin_media_source_id"],
|
|
"jellyfin_library_id": row["jellyfin_library_id"],
|
|
"jellyfin_item_type": row["jellyfin_item_type"],
|
|
"jellyfin_tags": tags_from_json(row["jellyfin_tags"]),
|
|
"trickplay_width": row["trickplay_width"],
|
|
"_vectors": {"siglip2": vec.tolist()},
|
|
}
|
|
|
|
|
|
def count_caption_repair_documents(db_path=None) -> int:
|
|
"""Count only frames touched by captioning; never require a full reindex."""
|
|
db_path = db_path or settings.db_path
|
|
init_db(db_path)
|
|
with connect(db_path) as con:
|
|
row = con.execute(
|
|
"""
|
|
SELECT COUNT(*) FROM (
|
|
SELECT frame_id FROM caption_window_frames
|
|
UNION
|
|
SELECT id FROM frames WHERE caption IS NOT NULL
|
|
)
|
|
"""
|
|
).fetchone()
|
|
return int(row[0])
|
|
|
|
|
|
_REPAIR_CHECKPOINT_SCHEMA = """
|
|
CREATE TABLE IF NOT EXISTS caption_index_repair_checkpoints (
|
|
index_uid TEXT PRIMARY KEY,
|
|
last_frame_id TEXT,
|
|
completed_documents INTEGER NOT NULL DEFAULT 0,
|
|
pending_task_uid INTEGER,
|
|
pending_last_frame_id TEXT,
|
|
pending_count INTEGER NOT NULL DEFAULT 0,
|
|
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
"""
|
|
|
|
|
|
def _repair_batch(db_path, after_frame_id: str | None, batch_size: int):
|
|
"""Fetch the next bounded batch without holding a SQLite cursor over IO."""
|
|
query = (
|
|
"WITH repair_ids AS ("
|
|
" SELECT frame_id AS id FROM caption_window_frames"
|
|
" UNION SELECT id FROM frames WHERE caption IS NOT NULL"
|
|
") "
|
|
+ _FRAME_DOCUMENT_SELECT.replace(
|
|
"FROM frames f", "FROM repair_ids r JOIN frames f ON f.id = r.id", 1
|
|
)
|
|
+ " WHERE (? IS NULL OR f.id > ?) ORDER BY f.id LIMIT ?"
|
|
)
|
|
with connect(db_path) as con:
|
|
return con.execute(
|
|
query, (after_frame_id, after_frame_id, batch_size)
|
|
).fetchall()
|
|
|
|
|
|
def _ensure_repair_checkpoint(db_path, index_uid: str) -> dict:
|
|
with connect(db_path) as con:
|
|
con.execute(_REPAIR_CHECKPOINT_SCHEMA)
|
|
con.execute(
|
|
"""
|
|
INSERT OR IGNORE INTO caption_index_repair_checkpoints (index_uid)
|
|
VALUES (?)
|
|
""",
|
|
(index_uid,),
|
|
)
|
|
row = con.execute(
|
|
"SELECT * FROM caption_index_repair_checkpoints WHERE index_uid = ?",
|
|
(index_uid,),
|
|
).fetchone()
|
|
return dict(row)
|
|
|
|
|
|
def _record_pending_repair(
|
|
db_path, index_uid: str, task_uid: int, last_frame_id: str, count: int
|
|
) -> None:
|
|
with connect(db_path) as con:
|
|
con.execute(
|
|
"""
|
|
UPDATE caption_index_repair_checkpoints
|
|
SET pending_task_uid = ?, pending_last_frame_id = ?,
|
|
pending_count = ?, updated_at = CURRENT_TIMESTAMP
|
|
WHERE index_uid = ? AND pending_task_uid IS NULL
|
|
""",
|
|
(task_uid, last_frame_id, count, index_uid),
|
|
)
|
|
if con.execute("SELECT changes()").fetchone()[0] != 1:
|
|
raise RuntimeError("Repair checkpoint changed unexpectedly")
|
|
|
|
|
|
def _finish_pending_repair(db_path, index_uid: str, task_uid: int) -> int:
|
|
"""Advance the durable cursor only once Meilisearch confirms success."""
|
|
with connect(db_path) as con:
|
|
con.execute(
|
|
"""
|
|
UPDATE caption_index_repair_checkpoints
|
|
SET last_frame_id = pending_last_frame_id,
|
|
completed_documents = completed_documents + pending_count,
|
|
pending_task_uid = NULL, pending_last_frame_id = NULL,
|
|
pending_count = 0, updated_at = CURRENT_TIMESTAMP
|
|
WHERE index_uid = ? AND pending_task_uid = ?
|
|
""",
|
|
(index_uid, task_uid),
|
|
)
|
|
if con.execute("SELECT changes()").fetchone()[0] != 1:
|
|
raise RuntimeError("Repair checkpoint changed unexpectedly")
|
|
return int(con.execute(
|
|
"""
|
|
SELECT completed_documents FROM caption_index_repair_checkpoints
|
|
WHERE index_uid = ?
|
|
""", (index_uid,)
|
|
).fetchone()[0])
|
|
|
|
|
|
def repair_caption_documents(
|
|
*,
|
|
batch_size: int = 250,
|
|
progress: SyncProgress | None = None,
|
|
meili: Meili | None = None,
|
|
db_path=None,
|
|
resume_task_uid: int | None = None,
|
|
) -> int:
|
|
"""Resumable repair of only caption-affected Meilisearch documents.
|
|
|
|
A durable SQLite checkpoint records each pending Meilisearch task *before*
|
|
waiting on it. If interrupted, the next invocation waits for that same
|
|
task, then continues at the last verified frame. Slow queues never trigger
|
|
the 120-second caption-sync timeout.
|
|
"""
|
|
db_path = db_path or settings.db_path
|
|
init_db(db_path)
|
|
client = meili or _meili_client()
|
|
index_uid = str(client.index)
|
|
batch_size = max(1, int(batch_size))
|
|
restored_this_run = 0
|
|
checkpoint = _ensure_repair_checkpoint(db_path, index_uid)
|
|
|
|
if resume_task_uid is not None:
|
|
if checkpoint["pending_task_uid"] is not None:
|
|
raise ValueError("An existing repair task is already checkpointed")
|
|
if checkpoint["last_frame_id"] is not None:
|
|
raise ValueError(
|
|
"--resume-task is only for adopting the original first batch "
|
|
"of an older, non-resumable repair"
|
|
)
|
|
|
|
rows = _repair_batch(db_path, None, batch_size)
|
|
if not rows:
|
|
raise ValueError("No documents exist to associate with that task")
|
|
task = client.get_task(resume_task_uid)
|
|
if task.get("indexUid") != index_uid:
|
|
raise ValueError(
|
|
f"Task {resume_task_uid} belongs to index "
|
|
f"{task.get('indexUid')!r}, not {index_uid!r}"
|
|
)
|
|
if task.get("type") != "documentAdditionOrUpdate":
|
|
raise ValueError(
|
|
f"Task {resume_task_uid} is not a document update task"
|
|
)
|
|
if task.get("status") in {"failed", "canceled"}:
|
|
raise RuntimeError(
|
|
f"Task {resume_task_uid} already {task.get('status')}; "
|
|
"rerun without --resume-task to resubmit that batch"
|
|
)
|
|
details = task.get("details") or {}
|
|
expected = details.get("receivedDocuments")
|
|
if expected is not None and int(expected) != len(rows):
|
|
raise ValueError(
|
|
f"Task {resume_task_uid} contains {expected} documents, "
|
|
f"but the first repair batch contains {len(rows)}"
|
|
)
|
|
_record_pending_repair(
|
|
db_path, index_uid, resume_task_uid, str(rows[-1]["id"]), len(rows)
|
|
)
|
|
print(
|
|
f"Adopted existing Meilisearch task {resume_task_uid} "
|
|
f"({len(rows)} documents); no duplicate batch submitted.",
|
|
flush=True,
|
|
)
|
|
|
|
while True:
|
|
checkpoint = _ensure_repair_checkpoint(db_path, index_uid)
|
|
task_uid = checkpoint["pending_task_uid"]
|
|
|
|
if task_uid is None:
|
|
rows = _repair_batch(
|
|
db_path, checkpoint["last_frame_id"], batch_size
|
|
)
|
|
if not rows:
|
|
break
|
|
|
|
docs = [_complete_frame_document(row) for row in rows]
|
|
# Wait=False returns the asynchronous task id. Persist it before
|
|
# waiting so a container restart cannot forget queued work.
|
|
task_uid = client.add_documents(docs, wait=False)
|
|
if task_uid is None:
|
|
raise RuntimeError(
|
|
"Meilisearch did not return a task ID; cannot resume safely"
|
|
)
|
|
task_uid = int(task_uid)
|
|
_record_pending_repair(
|
|
db_path, index_uid, task_uid, str(rows[-1]["id"]), len(rows)
|
|
)
|
|
print(
|
|
f"Queued repair task {task_uid}: {len(rows)} documents "
|
|
f"(already confirmed: {checkpoint['completed_documents']:,}).",
|
|
flush=True,
|
|
)
|
|
|
|
last_update = time.monotonic() - 16
|
|
|
|
def on_poll(task: dict) -> None:
|
|
nonlocal last_update
|
|
now = time.monotonic()
|
|
if now - last_update >= 15:
|
|
status = str(task.get("status") or "unknown")
|
|
print(
|
|
f"Waiting for Meilisearch task {task_uid}: {status} "
|
|
f"(repair remains resumable with Ctrl+C).",
|
|
flush=True,
|
|
)
|
|
last_update = now
|
|
|
|
# Unlike ordinary caption sync, a repair intentionally waits as long
|
|
# as the index needs. Re-run the command after Ctrl+C or a restart:
|
|
# the task UID is already stored in SQLite.
|
|
try:
|
|
client.wait_for_task(
|
|
int(task_uid),
|
|
timeout_seconds=None,
|
|
poll_seconds=3.0,
|
|
on_poll=on_poll,
|
|
)
|
|
except RuntimeError:
|
|
# Failed/canceled tasks are safe to resubmit. Preserve the
|
|
# checkpoint on API/network errors to avoid duplicate jobs.
|
|
try:
|
|
task = client.get_task(int(task_uid))
|
|
except Exception:
|
|
raise
|
|
if task.get("status") in {"failed", "canceled"}:
|
|
with connect(db_path) as con:
|
|
con.execute(
|
|
"""
|
|
UPDATE caption_index_repair_checkpoints
|
|
SET pending_task_uid = NULL, pending_last_frame_id = NULL,
|
|
pending_count = 0, updated_at = CURRENT_TIMESTAMP
|
|
WHERE index_uid = ? AND pending_task_uid = ?
|
|
""",
|
|
(index_uid, task_uid),
|
|
)
|
|
raise
|
|
|
|
count = int(checkpoint["pending_count"]) if checkpoint["pending_task_uid"] else len(rows)
|
|
confirmed_total = _finish_pending_repair(db_path, index_uid, int(task_uid))
|
|
restored_this_run += count
|
|
if progress is not None:
|
|
progress({
|
|
"stage": "repair",
|
|
"documents_processed": confirmed_total,
|
|
"completed_this_run": restored_this_run,
|
|
"task_uid": task_uid,
|
|
})
|
|
|
|
print(
|
|
f"Repair index {index_uid}: {checkpoint['completed_documents']:,} "
|
|
"documents previously confirmed before this run.",
|
|
flush=True,
|
|
)
|
|
return restored_this_run
|
|
|
|
|
|
def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
|
|
"""Rebuild the disposable Meilisearch index from canonical SQLite."""
|
|
init_db(settings.db_path)
|
|
meili = _meili_client()
|
|
|
|
with connect(settings.db_path) as con:
|
|
dims = [row[0] for row in con.execute(
|
|
"SELECT DISTINCT embedding_dim FROM media"
|
|
)]
|
|
if not dims:
|
|
meili.clear_documents()
|
|
if progress is not None:
|
|
progress({"stage": "sync", "documents_processed": 0})
|
|
return 0
|
|
if len(dims) != 1:
|
|
raise RuntimeError(
|
|
f"Multiple embedding dimensions found: {dims}. "
|
|
"Use separate indexes per model."
|
|
)
|
|
|
|
meili.configure_user_vectors(int(dims[0]))
|
|
meili.clear_documents()
|
|
|
|
cursor = con.execute(_FRAME_DOCUMENT_SELECT + " ORDER BY f.id")
|
|
total = 0
|
|
while True:
|
|
rows = cursor.fetchmany(100)
|
|
if not rows:
|
|
break
|
|
|
|
docs = []
|
|
for row in rows:
|
|
docs.append(_complete_frame_document(row))
|
|
|
|
meili.add_documents(docs)
|
|
total += len(docs)
|
|
if progress is not None:
|
|
progress(
|
|
{
|
|
"stage": "sync",
|
|
"documents_processed": total,
|
|
}
|
|
)
|
|
|
|
with connect(settings.db_path) as con:
|
|
con.execute(
|
|
"""
|
|
UPDATE frames
|
|
SET meili_synced_at = CURRENT_TIMESTAMP
|
|
"""
|
|
)
|
|
con.execute(
|
|
"""
|
|
UPDATE caption_windows
|
|
SET meili_synced_at = CURRENT_TIMESTAMP
|
|
WHERE caption IS NOT NULL
|
|
"""
|
|
)
|
|
|
|
if progress is not None:
|
|
progress({"stage": "synced", "documents_processed": total})
|
|
return total
|