Checkpoint repair task IDs in SQLite and resume slow asynchronous Meilisearch batches

This commit is contained in:
Sleeps
2026-10-08 11:08:34 +01:00
parent de28dee493
commit d609f43f5f
+230 -24
View File
@@ -1,5 +1,6 @@
from __future__ import annotations from __future__ import annotations
import time
from collections.abc import Callable from collections.abc import Callable
from typing import Any from typing import Any
@@ -359,24 +360,21 @@ def count_caption_repair_documents(db_path=None) -> int:
return int(row[0]) return int(row[0])
def repair_caption_documents( _REPAIR_CHECKPOINT_SCHEMA = """
*, CREATE TABLE IF NOT EXISTS caption_index_repair_checkpoints (
batch_size: int = 250, index_uid TEXT PRIMARY KEY,
progress: SyncProgress | None = None, last_frame_id TEXT,
meili: Meili | None = None, completed_documents INTEGER NOT NULL DEFAULT 0,
db_path=None, pending_task_uid INTEGER,
) -> int: pending_last_frame_id TEXT,
"""Restore caption-touched documents overwritten by older POST updates. pending_count INTEGER NOT NULL DEFAULT 0,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
Uses existing SQLite captions, vectors and Jellyfin identity without )
clearing Meilisearch, re-embedding or invoking Ollama. Each replacement
task is awaited so failed repairs are not silently reported as complete.
""" """
db_path = db_path or settings.db_path
init_db(db_path)
client = meili or _meili_client() def _repair_batch(db_path, after_frame_id: str | None, batch_size: int):
batch_size = max(1, int(batch_size)) """Fetch the next bounded batch without holding a SQLite cursor over IO."""
total = 0
query = ( query = (
"WITH repair_ids AS (" "WITH repair_ids AS ("
" SELECT frame_id AS id FROM caption_window_frames" " SELECT frame_id AS id FROM caption_window_frames"
@@ -385,20 +383,228 @@ def repair_caption_documents(
+ _FRAME_DOCUMENT_SELECT.replace( + _FRAME_DOCUMENT_SELECT.replace(
"FROM frames f", "FROM repair_ids r JOIN frames f ON f.id = r.id", 1 "FROM frames f", "FROM repair_ids r JOIN frames f ON f.id = r.id", 1
) )
+ " ORDER BY f.id" + " WHERE (? IS NULL OR f.id > ?) ORDER BY f.id LIMIT ?"
) )
with connect(db_path) as con: with connect(db_path) as con:
cursor = con.execute(query) return con.execute(
query, (after_frame_id, after_frame_id, batch_size)
).fetchall()
def _ensure_repair_checkpoint(db_path, index_uid: str) -> dict:
with connect(db_path) as con:
con.execute(_REPAIR_CHECKPOINT_SCHEMA)
con.execute(
"""
INSERT OR IGNORE INTO caption_index_repair_checkpoints (index_uid)
VALUES (?)
""",
(index_uid,),
)
row = con.execute(
"SELECT * FROM caption_index_repair_checkpoints WHERE index_uid = ?",
(index_uid,),
).fetchone()
return dict(row)
def _record_pending_repair(
db_path, index_uid: str, task_uid: int, last_frame_id: str, count: int
) -> None:
with connect(db_path) as con:
con.execute(
"""
UPDATE caption_index_repair_checkpoints
SET pending_task_uid = ?, pending_last_frame_id = ?,
pending_count = ?, updated_at = CURRENT_TIMESTAMP
WHERE index_uid = ? AND pending_task_uid IS NULL
""",
(task_uid, last_frame_id, count, index_uid),
)
if con.execute("SELECT changes()").fetchone()[0] != 1:
raise RuntimeError("Repair checkpoint changed unexpectedly")
def _finish_pending_repair(db_path, index_uid: str, task_uid: int) -> int:
"""Advance the durable cursor only once Meilisearch confirms success."""
with connect(db_path) as con:
con.execute(
"""
UPDATE caption_index_repair_checkpoints
SET last_frame_id = pending_last_frame_id,
completed_documents = completed_documents + pending_count,
pending_task_uid = NULL, pending_last_frame_id = NULL,
pending_count = 0, updated_at = CURRENT_TIMESTAMP
WHERE index_uid = ? AND pending_task_uid = ?
""",
(index_uid, task_uid),
)
if con.execute("SELECT changes()").fetchone()[0] != 1:
raise RuntimeError("Repair checkpoint changed unexpectedly")
return int(con.execute(
"""
SELECT completed_documents FROM caption_index_repair_checkpoints
WHERE index_uid = ?
""", (index_uid,)
).fetchone()[0])
def repair_caption_documents(
*,
batch_size: int = 250,
progress: SyncProgress | None = None,
meili: Meili | None = None,
db_path=None,
resume_task_uid: int | None = None,
) -> int:
"""Resumable repair of only caption-affected Meilisearch documents.
A durable SQLite checkpoint records each pending Meilisearch task *before*
waiting on it. If interrupted, the next invocation waits for that same
task, then continues at the last verified frame. Slow queues never trigger
the 120-second caption-sync timeout.
"""
db_path = db_path or settings.db_path
init_db(db_path)
client = meili or _meili_client()
index_uid = str(client.index)
batch_size = max(1, int(batch_size))
restored_this_run = 0
checkpoint = _ensure_repair_checkpoint(db_path, index_uid)
if resume_task_uid is not None:
if checkpoint["pending_task_uid"] is not None:
raise ValueError("An existing repair task is already checkpointed")
if checkpoint["last_frame_id"] is not None:
raise ValueError(
"--resume-task is only for adopting the original first batch "
"of an older, non-resumable repair"
)
rows = _repair_batch(db_path, None, batch_size)
if not rows:
raise ValueError("No documents exist to associate with that task")
task = client.get_task(resume_task_uid)
if task.get("indexUid") != index_uid:
raise ValueError(
f"Task {resume_task_uid} belongs to index "
f"{task.get('indexUid')!r}, not {index_uid!r}"
)
if task.get("type") != "documentAdditionOrUpdate":
raise ValueError(
f"Task {resume_task_uid} is not a document update task"
)
if task.get("status") in {"failed", "canceled"}:
raise RuntimeError(
f"Task {resume_task_uid} already {task.get('status')}; "
"rerun without --resume-task to resubmit that batch"
)
details = task.get("details") or {}
expected = details.get("receivedDocuments")
if expected is not None and int(expected) != len(rows):
raise ValueError(
f"Task {resume_task_uid} contains {expected} documents, "
f"but the first repair batch contains {len(rows)}"
)
_record_pending_repair(
db_path, index_uid, resume_task_uid, str(rows[-1]["id"]), len(rows)
)
print(
f"Adopted existing Meilisearch task {resume_task_uid} "
f"({len(rows)} documents); no duplicate batch submitted.",
flush=True,
)
while True: while True:
rows = cursor.fetchmany(batch_size) checkpoint = _ensure_repair_checkpoint(db_path, index_uid)
task_uid = checkpoint["pending_task_uid"]
if task_uid is None:
rows = _repair_batch(
db_path, checkpoint["last_frame_id"], batch_size
)
if not rows: if not rows:
break break
docs = [_complete_frame_document(row) for row in rows] docs = [_complete_frame_document(row) for row in rows]
client.add_documents(docs, wait=True) # Wait=False returns the asynchronous task id. Persist it before
total += len(docs) # waiting so a container restart cannot forget queued work.
task_uid = client.add_documents(docs, wait=False)
if task_uid is None:
raise RuntimeError(
"Meilisearch did not return a task ID; cannot resume safely"
)
task_uid = int(task_uid)
_record_pending_repair(
db_path, index_uid, task_uid, str(rows[-1]["id"]), len(rows)
)
print(
f"Queued repair task {task_uid}: {len(rows)} documents "
f"(already confirmed: {checkpoint['completed_documents']:,}).",
flush=True,
)
last_update = time.monotonic() - 16
def on_poll(task: dict) -> None:
nonlocal last_update
now = time.monotonic()
if now - last_update >= 15:
status = str(task.get("status") or "unknown")
print(
f"Waiting for Meilisearch task {task_uid}: {status} "
f"(repair remains resumable with Ctrl+C).",
flush=True,
)
last_update = now
# Unlike ordinary caption sync, a repair intentionally waits as long
# as the index needs. Re-run the command after Ctrl+C or a restart:
# the task UID is already stored in SQLite.
try:
client.wait_for_task(
int(task_uid),
timeout_seconds=None,
poll_seconds=3.0,
on_poll=on_poll,
)
except RuntimeError:
# Failed/canceled tasks are safe to resubmit. Preserve the
# checkpoint on API/network errors to avoid duplicate jobs.
try:
task = client.get_task(int(task_uid))
except Exception:
raise
if task.get("status") in {"failed", "canceled"}:
with connect(db_path) as con:
con.execute(
"""
UPDATE caption_index_repair_checkpoints
SET pending_task_uid = NULL, pending_last_frame_id = NULL,
pending_count = 0, updated_at = CURRENT_TIMESTAMP
WHERE index_uid = ? AND pending_task_uid = ?
""",
(index_uid, task_uid),
)
raise
count = int(checkpoint["pending_count"]) if checkpoint["pending_task_uid"] else len(rows)
confirmed_total = _finish_pending_repair(db_path, index_uid, int(task_uid))
restored_this_run += count
if progress is not None: if progress is not None:
progress({"stage": "repair", "documents_processed": total}) progress({
return total "stage": "repair",
"documents_processed": confirmed_total,
"completed_this_run": restored_this_run,
"task_uid": task_uid,
})
print(
f"Repair index {index_uid}: {checkpoint['completed_documents']:,} "
"documents previously confirmed before this run.",
flush=True,
)
return restored_this_run
def rebuild_meili(*, progress: SyncProgress | None = None) -> int: def rebuild_meili(*, progress: SyncProgress | None = None) -> int: