mirror of
https://github.com/MrSleeps/jellyfin-scene-search.git
synced 2026-11-20 00:33:36 +00:00
Checkpoint repair task IDs in SQLite and resume slow asynchronous Meilisearch batches
This commit is contained in:
+232
-26
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
@@ -359,24 +360,21 @@ def count_caption_repair_documents(db_path=None) -> int:
|
||||
return int(row[0])
|
||||
|
||||
|
||||
def repair_caption_documents(
|
||||
*,
|
||||
batch_size: int = 250,
|
||||
progress: SyncProgress | None = None,
|
||||
meili: Meili | None = None,
|
||||
db_path=None,
|
||||
) -> int:
|
||||
"""Restore caption-touched documents overwritten by older POST updates.
|
||||
_REPAIR_CHECKPOINT_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS caption_index_repair_checkpoints (
|
||||
index_uid TEXT PRIMARY KEY,
|
||||
last_frame_id TEXT,
|
||||
completed_documents INTEGER NOT NULL DEFAULT 0,
|
||||
pending_task_uid INTEGER,
|
||||
pending_last_frame_id TEXT,
|
||||
pending_count INTEGER NOT NULL DEFAULT 0,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
"""
|
||||
|
||||
Uses existing SQLite captions, vectors and Jellyfin identity without
|
||||
clearing Meilisearch, re-embedding or invoking Ollama. Each replacement
|
||||
task is awaited so failed repairs are not silently reported as complete.
|
||||
"""
|
||||
db_path = db_path or settings.db_path
|
||||
init_db(db_path)
|
||||
client = meili or _meili_client()
|
||||
batch_size = max(1, int(batch_size))
|
||||
total = 0
|
||||
|
||||
def _repair_batch(db_path, after_frame_id: str | None, batch_size: int):
|
||||
"""Fetch the next bounded batch without holding a SQLite cursor over IO."""
|
||||
query = (
|
||||
"WITH repair_ids AS ("
|
||||
" SELECT frame_id AS id FROM caption_window_frames"
|
||||
@@ -385,20 +383,228 @@ def repair_caption_documents(
|
||||
+ _FRAME_DOCUMENT_SELECT.replace(
|
||||
"FROM frames f", "FROM repair_ids r JOIN frames f ON f.id = r.id", 1
|
||||
)
|
||||
+ " ORDER BY f.id"
|
||||
+ " WHERE (? IS NULL OR f.id > ?) ORDER BY f.id LIMIT ?"
|
||||
)
|
||||
with connect(db_path) as con:
|
||||
cursor = con.execute(query)
|
||||
while True:
|
||||
rows = cursor.fetchmany(batch_size)
|
||||
return con.execute(
|
||||
query, (after_frame_id, after_frame_id, batch_size)
|
||||
).fetchall()
|
||||
|
||||
|
||||
def _ensure_repair_checkpoint(db_path, index_uid: str) -> dict:
|
||||
with connect(db_path) as con:
|
||||
con.execute(_REPAIR_CHECKPOINT_SCHEMA)
|
||||
con.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO caption_index_repair_checkpoints (index_uid)
|
||||
VALUES (?)
|
||||
""",
|
||||
(index_uid,),
|
||||
)
|
||||
row = con.execute(
|
||||
"SELECT * FROM caption_index_repair_checkpoints WHERE index_uid = ?",
|
||||
(index_uid,),
|
||||
).fetchone()
|
||||
return dict(row)
|
||||
|
||||
|
||||
def _record_pending_repair(
|
||||
db_path, index_uid: str, task_uid: int, last_frame_id: str, count: int
|
||||
) -> None:
|
||||
with connect(db_path) as con:
|
||||
con.execute(
|
||||
"""
|
||||
UPDATE caption_index_repair_checkpoints
|
||||
SET pending_task_uid = ?, pending_last_frame_id = ?,
|
||||
pending_count = ?, updated_at = CURRENT_TIMESTAMP
|
||||
WHERE index_uid = ? AND pending_task_uid IS NULL
|
||||
""",
|
||||
(task_uid, last_frame_id, count, index_uid),
|
||||
)
|
||||
if con.execute("SELECT changes()").fetchone()[0] != 1:
|
||||
raise RuntimeError("Repair checkpoint changed unexpectedly")
|
||||
|
||||
|
||||
def _finish_pending_repair(db_path, index_uid: str, task_uid: int) -> int:
|
||||
"""Advance the durable cursor only once Meilisearch confirms success."""
|
||||
with connect(db_path) as con:
|
||||
con.execute(
|
||||
"""
|
||||
UPDATE caption_index_repair_checkpoints
|
||||
SET last_frame_id = pending_last_frame_id,
|
||||
completed_documents = completed_documents + pending_count,
|
||||
pending_task_uid = NULL, pending_last_frame_id = NULL,
|
||||
pending_count = 0, updated_at = CURRENT_TIMESTAMP
|
||||
WHERE index_uid = ? AND pending_task_uid = ?
|
||||
""",
|
||||
(index_uid, task_uid),
|
||||
)
|
||||
if con.execute("SELECT changes()").fetchone()[0] != 1:
|
||||
raise RuntimeError("Repair checkpoint changed unexpectedly")
|
||||
return int(con.execute(
|
||||
"""
|
||||
SELECT completed_documents FROM caption_index_repair_checkpoints
|
||||
WHERE index_uid = ?
|
||||
""", (index_uid,)
|
||||
).fetchone()[0])
|
||||
|
||||
|
||||
def repair_caption_documents(
|
||||
*,
|
||||
batch_size: int = 250,
|
||||
progress: SyncProgress | None = None,
|
||||
meili: Meili | None = None,
|
||||
db_path=None,
|
||||
resume_task_uid: int | None = None,
|
||||
) -> int:
|
||||
"""Resumable repair of only caption-affected Meilisearch documents.
|
||||
|
||||
A durable SQLite checkpoint records each pending Meilisearch task *before*
|
||||
waiting on it. If interrupted, the next invocation waits for that same
|
||||
task, then continues at the last verified frame. Slow queues never trigger
|
||||
the 120-second caption-sync timeout.
|
||||
"""
|
||||
db_path = db_path or settings.db_path
|
||||
init_db(db_path)
|
||||
client = meili or _meili_client()
|
||||
index_uid = str(client.index)
|
||||
batch_size = max(1, int(batch_size))
|
||||
restored_this_run = 0
|
||||
checkpoint = _ensure_repair_checkpoint(db_path, index_uid)
|
||||
|
||||
if resume_task_uid is not None:
|
||||
if checkpoint["pending_task_uid"] is not None:
|
||||
raise ValueError("An existing repair task is already checkpointed")
|
||||
if checkpoint["last_frame_id"] is not None:
|
||||
raise ValueError(
|
||||
"--resume-task is only for adopting the original first batch "
|
||||
"of an older, non-resumable repair"
|
||||
)
|
||||
|
||||
rows = _repair_batch(db_path, None, batch_size)
|
||||
if not rows:
|
||||
raise ValueError("No documents exist to associate with that task")
|
||||
task = client.get_task(resume_task_uid)
|
||||
if task.get("indexUid") != index_uid:
|
||||
raise ValueError(
|
||||
f"Task {resume_task_uid} belongs to index "
|
||||
f"{task.get('indexUid')!r}, not {index_uid!r}"
|
||||
)
|
||||
if task.get("type") != "documentAdditionOrUpdate":
|
||||
raise ValueError(
|
||||
f"Task {resume_task_uid} is not a document update task"
|
||||
)
|
||||
if task.get("status") in {"failed", "canceled"}:
|
||||
raise RuntimeError(
|
||||
f"Task {resume_task_uid} already {task.get('status')}; "
|
||||
"rerun without --resume-task to resubmit that batch"
|
||||
)
|
||||
details = task.get("details") or {}
|
||||
expected = details.get("receivedDocuments")
|
||||
if expected is not None and int(expected) != len(rows):
|
||||
raise ValueError(
|
||||
f"Task {resume_task_uid} contains {expected} documents, "
|
||||
f"but the first repair batch contains {len(rows)}"
|
||||
)
|
||||
_record_pending_repair(
|
||||
db_path, index_uid, resume_task_uid, str(rows[-1]["id"]), len(rows)
|
||||
)
|
||||
print(
|
||||
f"Adopted existing Meilisearch task {resume_task_uid} "
|
||||
f"({len(rows)} documents); no duplicate batch submitted.",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
while True:
|
||||
checkpoint = _ensure_repair_checkpoint(db_path, index_uid)
|
||||
task_uid = checkpoint["pending_task_uid"]
|
||||
|
||||
if task_uid is None:
|
||||
rows = _repair_batch(
|
||||
db_path, checkpoint["last_frame_id"], batch_size
|
||||
)
|
||||
if not rows:
|
||||
break
|
||||
|
||||
docs = [_complete_frame_document(row) for row in rows]
|
||||
client.add_documents(docs, wait=True)
|
||||
total += len(docs)
|
||||
if progress is not None:
|
||||
progress({"stage": "repair", "documents_processed": total})
|
||||
return total
|
||||
# Wait=False returns the asynchronous task id. Persist it before
|
||||
# waiting so a container restart cannot forget queued work.
|
||||
task_uid = client.add_documents(docs, wait=False)
|
||||
if task_uid is None:
|
||||
raise RuntimeError(
|
||||
"Meilisearch did not return a task ID; cannot resume safely"
|
||||
)
|
||||
task_uid = int(task_uid)
|
||||
_record_pending_repair(
|
||||
db_path, index_uid, task_uid, str(rows[-1]["id"]), len(rows)
|
||||
)
|
||||
print(
|
||||
f"Queued repair task {task_uid}: {len(rows)} documents "
|
||||
f"(already confirmed: {checkpoint['completed_documents']:,}).",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
last_update = time.monotonic() - 16
|
||||
|
||||
def on_poll(task: dict) -> None:
|
||||
nonlocal last_update
|
||||
now = time.monotonic()
|
||||
if now - last_update >= 15:
|
||||
status = str(task.get("status") or "unknown")
|
||||
print(
|
||||
f"Waiting for Meilisearch task {task_uid}: {status} "
|
||||
f"(repair remains resumable with Ctrl+C).",
|
||||
flush=True,
|
||||
)
|
||||
last_update = now
|
||||
|
||||
# Unlike ordinary caption sync, a repair intentionally waits as long
|
||||
# as the index needs. Re-run the command after Ctrl+C or a restart:
|
||||
# the task UID is already stored in SQLite.
|
||||
try:
|
||||
client.wait_for_task(
|
||||
int(task_uid),
|
||||
timeout_seconds=None,
|
||||
poll_seconds=3.0,
|
||||
on_poll=on_poll,
|
||||
)
|
||||
except RuntimeError:
|
||||
# Failed/canceled tasks are safe to resubmit. Preserve the
|
||||
# checkpoint on API/network errors to avoid duplicate jobs.
|
||||
try:
|
||||
task = client.get_task(int(task_uid))
|
||||
except Exception:
|
||||
raise
|
||||
if task.get("status") in {"failed", "canceled"}:
|
||||
with connect(db_path) as con:
|
||||
con.execute(
|
||||
"""
|
||||
UPDATE caption_index_repair_checkpoints
|
||||
SET pending_task_uid = NULL, pending_last_frame_id = NULL,
|
||||
pending_count = 0, updated_at = CURRENT_TIMESTAMP
|
||||
WHERE index_uid = ? AND pending_task_uid = ?
|
||||
""",
|
||||
(index_uid, task_uid),
|
||||
)
|
||||
raise
|
||||
|
||||
count = int(checkpoint["pending_count"]) if checkpoint["pending_task_uid"] else len(rows)
|
||||
confirmed_total = _finish_pending_repair(db_path, index_uid, int(task_uid))
|
||||
restored_this_run += count
|
||||
if progress is not None:
|
||||
progress({
|
||||
"stage": "repair",
|
||||
"documents_processed": confirmed_total,
|
||||
"completed_this_run": restored_this_run,
|
||||
"task_uid": task_uid,
|
||||
})
|
||||
|
||||
print(
|
||||
f"Repair index {index_uid}: {checkpoint['completed_documents']:,} "
|
||||
"documents previously confirmed before this run.",
|
||||
flush=True,
|
||||
)
|
||||
return restored_this_run
|
||||
|
||||
|
||||
def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
|
||||
|
||||
Reference in New Issue
Block a user