From de28dee493355ff4a17aa15180b9d918cf9fff12 Mon Sep 17 00:00:00 2001 From: Sleeps Date: Thu, 8 Oct 2026 11:07:52 +0100 Subject: [PATCH 1/5] Expose task IDs and support unbounded, observable Meilisearch waits --- scene_search/meili.py | 50 ++++++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/scene_search/meili.py b/scene_search/meili.py index 63f40b0..c615e4e 100644 --- a/scene_search/meili.py +++ b/scene_search/meili.py @@ -3,6 +3,7 @@ from __future__ import annotations import json import time from collections.abc import Iterable +from typing import Callable import requests @@ -73,7 +74,7 @@ class Meili: documents: list[dict], *, wait: bool = False, - ) -> None: + ) -> int | None: """Replace complete documents (used for indexing and targeted repairs).""" if not documents: return @@ -82,18 +83,38 @@ class Meili: params={"primaryKey": "id"}, json=documents, )) - if wait: - task_uid = response.json().get("taskUid") - if task_uid is None: + task_uid = response.json().get("taskUid") + if task_uid is None: + if wait: raise RuntimeError("Meilisearch replacement did not return a taskUid") - self.wait_for_task(int(task_uid)) + return None + task_uid = int(task_uid) + if wait: + self.wait_for_task(task_uid) + return task_uid - def wait_for_task(self, task_uid: int, *, timeout_seconds: float = 120) -> dict: - deadline = time.monotonic() + timeout_seconds + def get_task(self, task_uid: int) -> dict: + """Read the durable status of an asynchronous Meilisearch task.""" + return self._check( + self.session.get(f"{self.url}/tasks/{task_uid}", timeout=15) + ).json() + + def wait_for_task( + self, + task_uid: int, + *, + timeout_seconds: float | None = 120, + poll_seconds: float = 2.0, + on_poll: Callable[[dict], None] | None = None, + ) -> dict: + """Wait for a task. None timeout permits an intentionally long repair.""" + deadline = ( + time.monotonic() + timeout_seconds + if timeout_seconds is not None + else None + ) while True: - task = self._check( - self.session.get(f"{self.url}/tasks/{task_uid}") - ).json() + task = self.get_task(task_uid) status = str(task.get("status") or "") if status == "succeeded": return task @@ -104,11 +125,14 @@ class Meili: f"Meilisearch task {task_uid} {status}: " f"{message or task}" ) - if time.monotonic() >= deadline: + if on_poll is not None: + on_poll(task) + if deadline is not None and time.monotonic() >= deadline: raise RuntimeError( - f"Timed out waiting for Meilisearch task {task_uid}" + f"Timed out waiting for Meilisearch task {task_uid} " + f"(last status: {status or 'unknown'})" ) - time.sleep(0.1) + time.sleep(max(0.1, poll_seconds)) def update_documents( self, From d609f43f5f304833d3809064f18955a06bffe83e Mon Sep 17 00:00:00 2001 From: Sleeps Date: Thu, 8 Oct 2026 11:08:34 +0100 Subject: [PATCH 2/5] Checkpoint repair task IDs in SQLite and resume slow asynchronous Meilisearch batches --- scene_search/sync.py | 258 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 232 insertions(+), 26 deletions(-) diff --git a/scene_search/sync.py b/scene_search/sync.py index 26a5bb8..4f924fa 100644 --- a/scene_search/sync.py +++ b/scene_search/sync.py @@ -1,5 +1,6 @@ from __future__ import annotations +import time from collections.abc import Callable from typing import Any @@ -359,24 +360,21 @@ def count_caption_repair_documents(db_path=None) -> int: return int(row[0]) -def repair_caption_documents( - *, - batch_size: int = 250, - progress: SyncProgress | None = None, - meili: Meili | None = None, - db_path=None, -) -> int: - """Restore caption-touched documents overwritten by older POST updates. +_REPAIR_CHECKPOINT_SCHEMA = """ + CREATE TABLE IF NOT EXISTS caption_index_repair_checkpoints ( + index_uid TEXT PRIMARY KEY, + last_frame_id TEXT, + completed_documents INTEGER NOT NULL DEFAULT 0, + pending_task_uid INTEGER, + pending_last_frame_id TEXT, + pending_count INTEGER NOT NULL DEFAULT 0, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ) +""" - Uses existing SQLite captions, vectors and Jellyfin identity without - clearing Meilisearch, re-embedding or invoking Ollama. Each replacement - task is awaited so failed repairs are not silently reported as complete. - """ - db_path = db_path or settings.db_path - init_db(db_path) - client = meili or _meili_client() - batch_size = max(1, int(batch_size)) - total = 0 + +def _repair_batch(db_path, after_frame_id: str | None, batch_size: int): + """Fetch the next bounded batch without holding a SQLite cursor over IO.""" query = ( "WITH repair_ids AS (" " SELECT frame_id AS id FROM caption_window_frames" @@ -385,20 +383,228 @@ def repair_caption_documents( + _FRAME_DOCUMENT_SELECT.replace( "FROM frames f", "FROM repair_ids r JOIN frames f ON f.id = r.id", 1 ) - + " ORDER BY f.id" + + " WHERE (? IS NULL OR f.id > ?) ORDER BY f.id LIMIT ?" ) with connect(db_path) as con: - cursor = con.execute(query) - while True: - rows = cursor.fetchmany(batch_size) + return con.execute( + query, (after_frame_id, after_frame_id, batch_size) + ).fetchall() + + +def _ensure_repair_checkpoint(db_path, index_uid: str) -> dict: + with connect(db_path) as con: + con.execute(_REPAIR_CHECKPOINT_SCHEMA) + con.execute( + """ + INSERT OR IGNORE INTO caption_index_repair_checkpoints (index_uid) + VALUES (?) + """, + (index_uid,), + ) + row = con.execute( + "SELECT * FROM caption_index_repair_checkpoints WHERE index_uid = ?", + (index_uid,), + ).fetchone() + return dict(row) + + +def _record_pending_repair( + db_path, index_uid: str, task_uid: int, last_frame_id: str, count: int +) -> None: + with connect(db_path) as con: + con.execute( + """ + UPDATE caption_index_repair_checkpoints + SET pending_task_uid = ?, pending_last_frame_id = ?, + pending_count = ?, updated_at = CURRENT_TIMESTAMP + WHERE index_uid = ? AND pending_task_uid IS NULL + """, + (task_uid, last_frame_id, count, index_uid), + ) + if con.execute("SELECT changes()").fetchone()[0] != 1: + raise RuntimeError("Repair checkpoint changed unexpectedly") + + +def _finish_pending_repair(db_path, index_uid: str, task_uid: int) -> int: + """Advance the durable cursor only once Meilisearch confirms success.""" + with connect(db_path) as con: + con.execute( + """ + UPDATE caption_index_repair_checkpoints + SET last_frame_id = pending_last_frame_id, + completed_documents = completed_documents + pending_count, + pending_task_uid = NULL, pending_last_frame_id = NULL, + pending_count = 0, updated_at = CURRENT_TIMESTAMP + WHERE index_uid = ? AND pending_task_uid = ? + """, + (index_uid, task_uid), + ) + if con.execute("SELECT changes()").fetchone()[0] != 1: + raise RuntimeError("Repair checkpoint changed unexpectedly") + return int(con.execute( + """ + SELECT completed_documents FROM caption_index_repair_checkpoints + WHERE index_uid = ? + """, (index_uid,) + ).fetchone()[0]) + + +def repair_caption_documents( + *, + batch_size: int = 250, + progress: SyncProgress | None = None, + meili: Meili | None = None, + db_path=None, + resume_task_uid: int | None = None, +) -> int: + """Resumable repair of only caption-affected Meilisearch documents. + + A durable SQLite checkpoint records each pending Meilisearch task *before* + waiting on it. If interrupted, the next invocation waits for that same + task, then continues at the last verified frame. Slow queues never trigger + the 120-second caption-sync timeout. + """ + db_path = db_path or settings.db_path + init_db(db_path) + client = meili or _meili_client() + index_uid = str(client.index) + batch_size = max(1, int(batch_size)) + restored_this_run = 0 + checkpoint = _ensure_repair_checkpoint(db_path, index_uid) + + if resume_task_uid is not None: + if checkpoint["pending_task_uid"] is not None: + raise ValueError("An existing repair task is already checkpointed") + if checkpoint["last_frame_id"] is not None: + raise ValueError( + "--resume-task is only for adopting the original first batch " + "of an older, non-resumable repair" + ) + + rows = _repair_batch(db_path, None, batch_size) + if not rows: + raise ValueError("No documents exist to associate with that task") + task = client.get_task(resume_task_uid) + if task.get("indexUid") != index_uid: + raise ValueError( + f"Task {resume_task_uid} belongs to index " + f"{task.get('indexUid')!r}, not {index_uid!r}" + ) + if task.get("type") != "documentAdditionOrUpdate": + raise ValueError( + f"Task {resume_task_uid} is not a document update task" + ) + if task.get("status") in {"failed", "canceled"}: + raise RuntimeError( + f"Task {resume_task_uid} already {task.get('status')}; " + "rerun without --resume-task to resubmit that batch" + ) + details = task.get("details") or {} + expected = details.get("receivedDocuments") + if expected is not None and int(expected) != len(rows): + raise ValueError( + f"Task {resume_task_uid} contains {expected} documents, " + f"but the first repair batch contains {len(rows)}" + ) + _record_pending_repair( + db_path, index_uid, resume_task_uid, str(rows[-1]["id"]), len(rows) + ) + print( + f"Adopted existing Meilisearch task {resume_task_uid} " + f"({len(rows)} documents); no duplicate batch submitted.", + flush=True, + ) + + while True: + checkpoint = _ensure_repair_checkpoint(db_path, index_uid) + task_uid = checkpoint["pending_task_uid"] + + if task_uid is None: + rows = _repair_batch( + db_path, checkpoint["last_frame_id"], batch_size + ) if not rows: break + docs = [_complete_frame_document(row) for row in rows] - client.add_documents(docs, wait=True) - total += len(docs) - if progress is not None: - progress({"stage": "repair", "documents_processed": total}) - return total + # Wait=False returns the asynchronous task id. Persist it before + # waiting so a container restart cannot forget queued work. + task_uid = client.add_documents(docs, wait=False) + if task_uid is None: + raise RuntimeError( + "Meilisearch did not return a task ID; cannot resume safely" + ) + task_uid = int(task_uid) + _record_pending_repair( + db_path, index_uid, task_uid, str(rows[-1]["id"]), len(rows) + ) + print( + f"Queued repair task {task_uid}: {len(rows)} documents " + f"(already confirmed: {checkpoint['completed_documents']:,}).", + flush=True, + ) + + last_update = time.monotonic() - 16 + + def on_poll(task: dict) -> None: + nonlocal last_update + now = time.monotonic() + if now - last_update >= 15: + status = str(task.get("status") or "unknown") + print( + f"Waiting for Meilisearch task {task_uid}: {status} " + f"(repair remains resumable with Ctrl+C).", + flush=True, + ) + last_update = now + + # Unlike ordinary caption sync, a repair intentionally waits as long + # as the index needs. Re-run the command after Ctrl+C or a restart: + # the task UID is already stored in SQLite. + try: + client.wait_for_task( + int(task_uid), + timeout_seconds=None, + poll_seconds=3.0, + on_poll=on_poll, + ) + except RuntimeError: + # Failed/canceled tasks are safe to resubmit. Preserve the + # checkpoint on API/network errors to avoid duplicate jobs. + try: + task = client.get_task(int(task_uid)) + except Exception: + raise + if task.get("status") in {"failed", "canceled"}: + with connect(db_path) as con: + con.execute( + """ + UPDATE caption_index_repair_checkpoints + SET pending_task_uid = NULL, pending_last_frame_id = NULL, + pending_count = 0, updated_at = CURRENT_TIMESTAMP + WHERE index_uid = ? AND pending_task_uid = ? + """, + (index_uid, task_uid), + ) + raise + + count = int(checkpoint["pending_count"]) if checkpoint["pending_task_uid"] else len(rows) + confirmed_total = _finish_pending_repair(db_path, index_uid, int(task_uid)) + restored_this_run += count + if progress is not None: + progress({ + "stage": "repair", + "documents_processed": confirmed_total, + "completed_this_run": restored_this_run, + "task_uid": task_uid, + }) + + print( + f"Repair index {index_uid}: {checkpoint['completed_documents']:,} " + "documents previously confirmed before this run.", + flush=True, + ) + return restored_this_run def rebuild_meili(*, progress: SyncProgress | None = None) -> int: From 9fadd68ed1f6a18dc86f2380e91d9bf06d7ad546 Mon Sep 17 00:00:00 2001 From: Sleeps Date: Thu, 8 Oct 2026 11:08:50 +0100 Subject: [PATCH 3/5] Add CLI option to adopt original timed-out Meilisearch task --- scene_search/cli.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scene_search/cli.py b/scene_search/cli.py index 43c606d..5d12b59 100644 --- a/scene_search/cli.py +++ b/scene_search/cli.py @@ -199,9 +199,11 @@ def cmd_repair_caption_index(args: argparse.Namespace) -> None: repaired = repair_caption_documents( batch_size=args.batch_size, progress=progress, + resume_task_uid=args.resume_task, ) print( - f"Repair complete: {repaired:,} full documents restored from SQLite. " + f"Repair invocation complete: {repaired:,} full documents confirmed " + "during this invocation (progress is saved in SQLite). " "No visual embeddings or captions were regenerated." ) @@ -1044,6 +1046,13 @@ def make_parser() -> argparse.ArgumentParser: "--batch-size", type=int, default=250, help="Documents per Meilisearch repair task (default: 250)", ) + repair.add_argument( + "--resume-task", type=int, + help=( + "Adopt a known existing first-batch Meilisearch task from the old " + "repair implementation, rather than enqueueing it again" + ), + ) repair.set_defaults(func=cmd_repair_caption_index) jf_test = sub.add_parser("jellyfin-test", help="Verify Jellyfin API connectivity") From 8299c3f6265508404ea737b3bbf1a5fdc4f870c6 Mon Sep 17 00:00:00 2001 From: Sleeps Date: Thu, 8 Oct 2026 11:09:16 +0100 Subject: [PATCH 4/5] Test durable resume, old task adoption and no duplicate writes after errors --- tests/test_incremental_caption_sync.py | 123 ++++++++++++++++++++++++- 1 file changed, 122 insertions(+), 1 deletion(-) diff --git a/tests/test_incremental_caption_sync.py b/tests/test_incremental_caption_sync.py index 385476e..1a06d19 100644 --- a/tests/test_incremental_caption_sync.py +++ b/tests/test_incremental_caption_sync.py @@ -266,6 +266,8 @@ class IncrementalCaptionSyncTests(unittest.TestCase): self.assertEqual(count_caption_repair_documents(db_path), 3) meili = MagicMock() + meili.index = "scenes" + meili.add_documents.side_effect = [101, 102] progress = [] restored = repair_caption_documents( db_path=db_path, meili=meili, @@ -275,8 +277,10 @@ class IncrementalCaptionSyncTests(unittest.TestCase): self.assertEqual(restored, 3) self.assertEqual(meili.add_documents.call_count, 2) self.assertTrue(all( - call.kwargs["wait"] for call in meili.add_documents.call_args_list + call.kwargs["wait"] is False + for call in meili.add_documents.call_args_list )) + self.assertEqual(meili.wait_for_task.call_count, 2) docs = [ document for call in meili.add_documents.call_args_list @@ -296,6 +300,123 @@ class IncrementalCaptionSyncTests(unittest.TestCase): self.assertIn("siglip2", doc["_vectors"]) self.assertEqual(len(doc["_vectors"]["siglip2"]), 2) self.assertEqual(progress[-1]["documents_processed"], 3) + with connect(db_path) as con: + checkpoint = con.execute( + "SELECT * FROM caption_index_repair_checkpoints " + "WHERE index_uid = 'scenes'" + ).fetchone() + self.assertEqual(checkpoint["completed_documents"], 3) + self.assertIsNone(checkpoint["pending_task_uid"]) + + + def test_repair_resumes_pending_task_without_submitting_again(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + db_path = Path(tmp) / "scenes.sqlite3" + self._seed(db_path, count=2) + meili = MagicMock() + meili.index = "scenes" + meili.add_documents.return_value = 710 + meili.wait_for_task.side_effect = KeyboardInterrupt() + + with self.assertRaises(KeyboardInterrupt): + repair_caption_documents( + db_path=db_path, meili=meili, batch_size=2 + ) + + with connect(db_path) as con: + checkpoint = con.execute( + "SELECT * FROM caption_index_repair_checkpoints" + ).fetchone() + self.assertEqual(checkpoint["pending_task_uid"], 710) + self.assertEqual(checkpoint["completed_documents"], 0) + + meili.wait_for_task.side_effect = None + restored = repair_caption_documents( + db_path=db_path, meili=meili, batch_size=2 + ) + + self.assertEqual(restored, 2) + self.assertEqual(meili.add_documents.call_count, 1) + self.assertEqual(meili.wait_for_task.call_count, 2) + with connect(db_path) as con: + checkpoint = con.execute( + "SELECT * FROM caption_index_repair_checkpoints" + ).fetchone() + self.assertIsNone(checkpoint["pending_task_uid"]) + self.assertEqual(checkpoint["completed_documents"], 2) + + # Subsequent runs skip everything already confirmed. + self.assertEqual( + repair_caption_documents( + db_path=db_path, meili=meili, batch_size=2 + ), 0 + ) + self.assertEqual(meili.add_documents.call_count, 1) + + def test_adopt_old_first_batch_avoids_duplicating_task(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + db_path = Path(tmp) / "scenes.sqlite3" + self._seed(db_path, count=2) + meili = MagicMock() + meili.index = "scenes" + meili.get_task.return_value = { + "uid": 41758, + "indexUid": "scenes", + "type": "documentAdditionOrUpdate", + "status": "succeeded", + "details": {"receivedDocuments": 2}, + } + + restored = repair_caption_documents( + db_path=db_path, + meili=meili, + batch_size=2, + resume_task_uid=41758, + ) + + self.assertEqual(restored, 2) + meili.add_documents.assert_not_called() + meili.wait_for_task.assert_called_once() + with connect(db_path) as con: + row = con.execute( + "SELECT * FROM caption_index_repair_checkpoints" + ).fetchone() + self.assertEqual(row["completed_documents"], 2) + + def test_pending_task_error_is_not_silently_duplicated(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + db_path = Path(tmp) / "scenes.sqlite3" + self._seed(db_path, count=1) + meili = MagicMock() + meili.index = "scenes" + meili.add_documents.return_value = 512 + meili.wait_for_task.side_effect = RuntimeError( + "API temporarily unavailable" + ) + meili.get_task.return_value = { + "status": "processing", + } + with self.assertRaisesRegex( + RuntimeError, "API temporarily unavailable" + ): + repair_caption_documents( + db_path=db_path, meili=meili, batch_size=1 + ) + with connect(db_path) as con: + row = con.execute( + "SELECT * FROM caption_index_repair_checkpoints" + ).fetchone() + self.assertEqual(row["pending_task_uid"], 512) + self.assertEqual(row["completed_documents"], 0) + + meili.wait_for_task.side_effect = None + self.assertEqual( + repair_caption_documents( + db_path=db_path, meili=meili, batch_size=1 + ), 1 + ) + self.assertEqual(meili.add_documents.call_count, 1) + if __name__ == "__main__": From 812a3583f592cf1af1bf8e5cb7be2d44bdafe1a7 Mon Sep 17 00:00:00 2001 From: Sleeps Date: Thu, 8 Oct 2026 11:09:53 +0100 Subject: [PATCH 5/5] Document resumable repair and adoption of task 41758 --- README.md | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 8f8db8a..76efdeb 100644 --- a/README.md +++ b/README.md @@ -38,10 +38,35 @@ The command repopulates each affected Meilisearch document with the original frame ID, Jellyfin video/library IDs, title, timestamp, caption-window metadata, tags and stored SigLIP embedding from SQLite, waiting for each Meilisearch task to succeed. It doesn't call Ollama, re-embed frames, clear -Meilisearch or force a whole-library rebuild. If interrupted, simply rerun: -the repair is safe and idempotent. Keep indexing and captioning stopped while -it runs. After completion, test a distinctive saved caption on the standalone -web search and in the Jellyfin plugin diagnostics. +Meilisearch or force a whole-library rebuild. + +**Large-index repairs are resumable:** each submitted task ID and the last +confirmed frame ID are saved in the canonical SQLite database. The repair +waits as long as necessary for a busy Meilisearch index rather than aborting +after 120 seconds. Status lines are printed during slow tasks. If you press +Ctrl+C, restart Docker, or lose the terminal, rerun the same `--apply` +command: it will check the existing queued task before moving on and will +not resubmit completed batches. + +**Recovering the old task-timeout bug:** if you previously ran the older +repair implementation and it stopped with `Timed out waiting for Meilisearch +task 41758` **before displaying any completed batches**, use: + +```bash +docker compose exec -T scene-search python -m scene_search.cli repair-caption-index --apply --resume-task 41758 +``` + +Replace `41758` with the actual task ID from that timeout. The command +verifies the task belongs to the current index, is a document update, and has +the expected number of documents; it waits for the original task rather than +creating a duplicate. If that older task already failed or was canceled, +rerun normally without `--resume-task`. If any earlier batches already +completed under the old script, they might be processed again (harmless, +but may take extra time); only one previously queued task can be adopted. + +Keep captioning and library indexing stopped throughout the repair. After +completion, test a distinctive saved caption on the standalone web search +and in Jellyfin plugin diagnostics. Note: very old orphaned documents no longer associated with any saved caption or caption window are outside this targeted selection; a full `sync-meili`