diff --git a/scene_search/sync.py b/scene_search/sync.py index 26a5bb8..4f924fa 100644 --- a/scene_search/sync.py +++ b/scene_search/sync.py @@ -1,5 +1,6 @@ from __future__ import annotations +import time from collections.abc import Callable from typing import Any @@ -359,24 +360,21 @@ def count_caption_repair_documents(db_path=None) -> int: return int(row[0]) -def repair_caption_documents( - *, - batch_size: int = 250, - progress: SyncProgress | None = None, - meili: Meili | None = None, - db_path=None, -) -> int: - """Restore caption-touched documents overwritten by older POST updates. +_REPAIR_CHECKPOINT_SCHEMA = """ + CREATE TABLE IF NOT EXISTS caption_index_repair_checkpoints ( + index_uid TEXT PRIMARY KEY, + last_frame_id TEXT, + completed_documents INTEGER NOT NULL DEFAULT 0, + pending_task_uid INTEGER, + pending_last_frame_id TEXT, + pending_count INTEGER NOT NULL DEFAULT 0, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ) +""" - Uses existing SQLite captions, vectors and Jellyfin identity without - clearing Meilisearch, re-embedding or invoking Ollama. Each replacement - task is awaited so failed repairs are not silently reported as complete. - """ - db_path = db_path or settings.db_path - init_db(db_path) - client = meili or _meili_client() - batch_size = max(1, int(batch_size)) - total = 0 + +def _repair_batch(db_path, after_frame_id: str | None, batch_size: int): + """Fetch the next bounded batch without holding a SQLite cursor over IO.""" query = ( "WITH repair_ids AS (" " SELECT frame_id AS id FROM caption_window_frames" @@ -385,20 +383,228 @@ def repair_caption_documents( + _FRAME_DOCUMENT_SELECT.replace( "FROM frames f", "FROM repair_ids r JOIN frames f ON f.id = r.id", 1 ) - + " ORDER BY f.id" + + " WHERE (? IS NULL OR f.id > ?) ORDER BY f.id LIMIT ?" ) with connect(db_path) as con: - cursor = con.execute(query) - while True: - rows = cursor.fetchmany(batch_size) + return con.execute( + query, (after_frame_id, after_frame_id, batch_size) + ).fetchall() + + +def _ensure_repair_checkpoint(db_path, index_uid: str) -> dict: + with connect(db_path) as con: + con.execute(_REPAIR_CHECKPOINT_SCHEMA) + con.execute( + """ + INSERT OR IGNORE INTO caption_index_repair_checkpoints (index_uid) + VALUES (?) + """, + (index_uid,), + ) + row = con.execute( + "SELECT * FROM caption_index_repair_checkpoints WHERE index_uid = ?", + (index_uid,), + ).fetchone() + return dict(row) + + +def _record_pending_repair( + db_path, index_uid: str, task_uid: int, last_frame_id: str, count: int +) -> None: + with connect(db_path) as con: + con.execute( + """ + UPDATE caption_index_repair_checkpoints + SET pending_task_uid = ?, pending_last_frame_id = ?, + pending_count = ?, updated_at = CURRENT_TIMESTAMP + WHERE index_uid = ? AND pending_task_uid IS NULL + """, + (task_uid, last_frame_id, count, index_uid), + ) + if con.execute("SELECT changes()").fetchone()[0] != 1: + raise RuntimeError("Repair checkpoint changed unexpectedly") + + +def _finish_pending_repair(db_path, index_uid: str, task_uid: int) -> int: + """Advance the durable cursor only once Meilisearch confirms success.""" + with connect(db_path) as con: + con.execute( + """ + UPDATE caption_index_repair_checkpoints + SET last_frame_id = pending_last_frame_id, + completed_documents = completed_documents + pending_count, + pending_task_uid = NULL, pending_last_frame_id = NULL, + pending_count = 0, updated_at = CURRENT_TIMESTAMP + WHERE index_uid = ? AND pending_task_uid = ? + """, + (index_uid, task_uid), + ) + if con.execute("SELECT changes()").fetchone()[0] != 1: + raise RuntimeError("Repair checkpoint changed unexpectedly") + return int(con.execute( + """ + SELECT completed_documents FROM caption_index_repair_checkpoints + WHERE index_uid = ? + """, (index_uid,) + ).fetchone()[0]) + + +def repair_caption_documents( + *, + batch_size: int = 250, + progress: SyncProgress | None = None, + meili: Meili | None = None, + db_path=None, + resume_task_uid: int | None = None, +) -> int: + """Resumable repair of only caption-affected Meilisearch documents. + + A durable SQLite checkpoint records each pending Meilisearch task *before* + waiting on it. If interrupted, the next invocation waits for that same + task, then continues at the last verified frame. Slow queues never trigger + the 120-second caption-sync timeout. + """ + db_path = db_path or settings.db_path + init_db(db_path) + client = meili or _meili_client() + index_uid = str(client.index) + batch_size = max(1, int(batch_size)) + restored_this_run = 0 + checkpoint = _ensure_repair_checkpoint(db_path, index_uid) + + if resume_task_uid is not None: + if checkpoint["pending_task_uid"] is not None: + raise ValueError("An existing repair task is already checkpointed") + if checkpoint["last_frame_id"] is not None: + raise ValueError( + "--resume-task is only for adopting the original first batch " + "of an older, non-resumable repair" + ) + + rows = _repair_batch(db_path, None, batch_size) + if not rows: + raise ValueError("No documents exist to associate with that task") + task = client.get_task(resume_task_uid) + if task.get("indexUid") != index_uid: + raise ValueError( + f"Task {resume_task_uid} belongs to index " + f"{task.get('indexUid')!r}, not {index_uid!r}" + ) + if task.get("type") != "documentAdditionOrUpdate": + raise ValueError( + f"Task {resume_task_uid} is not a document update task" + ) + if task.get("status") in {"failed", "canceled"}: + raise RuntimeError( + f"Task {resume_task_uid} already {task.get('status')}; " + "rerun without --resume-task to resubmit that batch" + ) + details = task.get("details") or {} + expected = details.get("receivedDocuments") + if expected is not None and int(expected) != len(rows): + raise ValueError( + f"Task {resume_task_uid} contains {expected} documents, " + f"but the first repair batch contains {len(rows)}" + ) + _record_pending_repair( + db_path, index_uid, resume_task_uid, str(rows[-1]["id"]), len(rows) + ) + print( + f"Adopted existing Meilisearch task {resume_task_uid} " + f"({len(rows)} documents); no duplicate batch submitted.", + flush=True, + ) + + while True: + checkpoint = _ensure_repair_checkpoint(db_path, index_uid) + task_uid = checkpoint["pending_task_uid"] + + if task_uid is None: + rows = _repair_batch( + db_path, checkpoint["last_frame_id"], batch_size + ) if not rows: break + docs = [_complete_frame_document(row) for row in rows] - client.add_documents(docs, wait=True) - total += len(docs) - if progress is not None: - progress({"stage": "repair", "documents_processed": total}) - return total + # Wait=False returns the asynchronous task id. Persist it before + # waiting so a container restart cannot forget queued work. + task_uid = client.add_documents(docs, wait=False) + if task_uid is None: + raise RuntimeError( + "Meilisearch did not return a task ID; cannot resume safely" + ) + task_uid = int(task_uid) + _record_pending_repair( + db_path, index_uid, task_uid, str(rows[-1]["id"]), len(rows) + ) + print( + f"Queued repair task {task_uid}: {len(rows)} documents " + f"(already confirmed: {checkpoint['completed_documents']:,}).", + flush=True, + ) + + last_update = time.monotonic() - 16 + + def on_poll(task: dict) -> None: + nonlocal last_update + now = time.monotonic() + if now - last_update >= 15: + status = str(task.get("status") or "unknown") + print( + f"Waiting for Meilisearch task {task_uid}: {status} " + f"(repair remains resumable with Ctrl+C).", + flush=True, + ) + last_update = now + + # Unlike ordinary caption sync, a repair intentionally waits as long + # as the index needs. Re-run the command after Ctrl+C or a restart: + # the task UID is already stored in SQLite. + try: + client.wait_for_task( + int(task_uid), + timeout_seconds=None, + poll_seconds=3.0, + on_poll=on_poll, + ) + except RuntimeError: + # Failed/canceled tasks are safe to resubmit. Preserve the + # checkpoint on API/network errors to avoid duplicate jobs. + try: + task = client.get_task(int(task_uid)) + except Exception: + raise + if task.get("status") in {"failed", "canceled"}: + with connect(db_path) as con: + con.execute( + """ + UPDATE caption_index_repair_checkpoints + SET pending_task_uid = NULL, pending_last_frame_id = NULL, + pending_count = 0, updated_at = CURRENT_TIMESTAMP + WHERE index_uid = ? AND pending_task_uid = ? + """, + (index_uid, task_uid), + ) + raise + + count = int(checkpoint["pending_count"]) if checkpoint["pending_task_uid"] else len(rows) + confirmed_total = _finish_pending_repair(db_path, index_uid, int(task_uid)) + restored_this_run += count + if progress is not None: + progress({ + "stage": "repair", + "documents_processed": confirmed_total, + "completed_this_run": restored_this_run, + "task_uid": task_uid, + }) + + print( + f"Repair index {index_uid}: {checkpoint['completed_documents']:,} " + "documents previously confirmed before this run.", + flush=True, + ) + return restored_this_run def rebuild_meili(*, progress: SyncProgress | None = None) -> int: