mirror of
https://github.com/MrSleeps/jellyfin-scene-search.git
synced 2026-11-20 00:33:36 +00:00
Checkpoint repair task IDs in SQLite and resume slow asynchronous Meilisearch batches
This commit is contained in:
+232
-26
@@ -1,5 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import time
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
@@ -359,24 +360,21 @@ def count_caption_repair_documents(db_path=None) -> int:
|
|||||||
return int(row[0])
|
return int(row[0])
|
||||||
|
|
||||||
|
|
||||||
def repair_caption_documents(
|
_REPAIR_CHECKPOINT_SCHEMA = """
|
||||||
*,
|
CREATE TABLE IF NOT EXISTS caption_index_repair_checkpoints (
|
||||||
batch_size: int = 250,
|
index_uid TEXT PRIMARY KEY,
|
||||||
progress: SyncProgress | None = None,
|
last_frame_id TEXT,
|
||||||
meili: Meili | None = None,
|
completed_documents INTEGER NOT NULL DEFAULT 0,
|
||||||
db_path=None,
|
pending_task_uid INTEGER,
|
||||||
) -> int:
|
pending_last_frame_id TEXT,
|
||||||
"""Restore caption-touched documents overwritten by older POST updates.
|
pending_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
|
||||||
Uses existing SQLite captions, vectors and Jellyfin identity without
|
|
||||||
clearing Meilisearch, re-embedding or invoking Ollama. Each replacement
|
def _repair_batch(db_path, after_frame_id: str | None, batch_size: int):
|
||||||
task is awaited so failed repairs are not silently reported as complete.
|
"""Fetch the next bounded batch without holding a SQLite cursor over IO."""
|
||||||
"""
|
|
||||||
db_path = db_path or settings.db_path
|
|
||||||
init_db(db_path)
|
|
||||||
client = meili or _meili_client()
|
|
||||||
batch_size = max(1, int(batch_size))
|
|
||||||
total = 0
|
|
||||||
query = (
|
query = (
|
||||||
"WITH repair_ids AS ("
|
"WITH repair_ids AS ("
|
||||||
" SELECT frame_id AS id FROM caption_window_frames"
|
" SELECT frame_id AS id FROM caption_window_frames"
|
||||||
@@ -385,20 +383,228 @@ def repair_caption_documents(
|
|||||||
+ _FRAME_DOCUMENT_SELECT.replace(
|
+ _FRAME_DOCUMENT_SELECT.replace(
|
||||||
"FROM frames f", "FROM repair_ids r JOIN frames f ON f.id = r.id", 1
|
"FROM frames f", "FROM repair_ids r JOIN frames f ON f.id = r.id", 1
|
||||||
)
|
)
|
||||||
+ " ORDER BY f.id"
|
+ " WHERE (? IS NULL OR f.id > ?) ORDER BY f.id LIMIT ?"
|
||||||
)
|
)
|
||||||
with connect(db_path) as con:
|
with connect(db_path) as con:
|
||||||
cursor = con.execute(query)
|
return con.execute(
|
||||||
while True:
|
query, (after_frame_id, after_frame_id, batch_size)
|
||||||
rows = cursor.fetchmany(batch_size)
|
).fetchall()
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_repair_checkpoint(db_path, index_uid: str) -> dict:
|
||||||
|
with connect(db_path) as con:
|
||||||
|
con.execute(_REPAIR_CHECKPOINT_SCHEMA)
|
||||||
|
con.execute(
|
||||||
|
"""
|
||||||
|
INSERT OR IGNORE INTO caption_index_repair_checkpoints (index_uid)
|
||||||
|
VALUES (?)
|
||||||
|
""",
|
||||||
|
(index_uid,),
|
||||||
|
)
|
||||||
|
row = con.execute(
|
||||||
|
"SELECT * FROM caption_index_repair_checkpoints WHERE index_uid = ?",
|
||||||
|
(index_uid,),
|
||||||
|
).fetchone()
|
||||||
|
return dict(row)
|
||||||
|
|
||||||
|
|
||||||
|
def _record_pending_repair(
|
||||||
|
db_path, index_uid: str, task_uid: int, last_frame_id: str, count: int
|
||||||
|
) -> None:
|
||||||
|
with connect(db_path) as con:
|
||||||
|
con.execute(
|
||||||
|
"""
|
||||||
|
UPDATE caption_index_repair_checkpoints
|
||||||
|
SET pending_task_uid = ?, pending_last_frame_id = ?,
|
||||||
|
pending_count = ?, updated_at = CURRENT_TIMESTAMP
|
||||||
|
WHERE index_uid = ? AND pending_task_uid IS NULL
|
||||||
|
""",
|
||||||
|
(task_uid, last_frame_id, count, index_uid),
|
||||||
|
)
|
||||||
|
if con.execute("SELECT changes()").fetchone()[0] != 1:
|
||||||
|
raise RuntimeError("Repair checkpoint changed unexpectedly")
|
||||||
|
|
||||||
|
|
||||||
|
def _finish_pending_repair(db_path, index_uid: str, task_uid: int) -> int:
|
||||||
|
"""Advance the durable cursor only once Meilisearch confirms success."""
|
||||||
|
with connect(db_path) as con:
|
||||||
|
con.execute(
|
||||||
|
"""
|
||||||
|
UPDATE caption_index_repair_checkpoints
|
||||||
|
SET last_frame_id = pending_last_frame_id,
|
||||||
|
completed_documents = completed_documents + pending_count,
|
||||||
|
pending_task_uid = NULL, pending_last_frame_id = NULL,
|
||||||
|
pending_count = 0, updated_at = CURRENT_TIMESTAMP
|
||||||
|
WHERE index_uid = ? AND pending_task_uid = ?
|
||||||
|
""",
|
||||||
|
(index_uid, task_uid),
|
||||||
|
)
|
||||||
|
if con.execute("SELECT changes()").fetchone()[0] != 1:
|
||||||
|
raise RuntimeError("Repair checkpoint changed unexpectedly")
|
||||||
|
return int(con.execute(
|
||||||
|
"""
|
||||||
|
SELECT completed_documents FROM caption_index_repair_checkpoints
|
||||||
|
WHERE index_uid = ?
|
||||||
|
""", (index_uid,)
|
||||||
|
).fetchone()[0])
|
||||||
|
|
||||||
|
|
||||||
|
def repair_caption_documents(
|
||||||
|
*,
|
||||||
|
batch_size: int = 250,
|
||||||
|
progress: SyncProgress | None = None,
|
||||||
|
meili: Meili | None = None,
|
||||||
|
db_path=None,
|
||||||
|
resume_task_uid: int | None = None,
|
||||||
|
) -> int:
|
||||||
|
"""Resumable repair of only caption-affected Meilisearch documents.
|
||||||
|
|
||||||
|
A durable SQLite checkpoint records each pending Meilisearch task *before*
|
||||||
|
waiting on it. If interrupted, the next invocation waits for that same
|
||||||
|
task, then continues at the last verified frame. Slow queues never trigger
|
||||||
|
the 120-second caption-sync timeout.
|
||||||
|
"""
|
||||||
|
db_path = db_path or settings.db_path
|
||||||
|
init_db(db_path)
|
||||||
|
client = meili or _meili_client()
|
||||||
|
index_uid = str(client.index)
|
||||||
|
batch_size = max(1, int(batch_size))
|
||||||
|
restored_this_run = 0
|
||||||
|
checkpoint = _ensure_repair_checkpoint(db_path, index_uid)
|
||||||
|
|
||||||
|
if resume_task_uid is not None:
|
||||||
|
if checkpoint["pending_task_uid"] is not None:
|
||||||
|
raise ValueError("An existing repair task is already checkpointed")
|
||||||
|
if checkpoint["last_frame_id"] is not None:
|
||||||
|
raise ValueError(
|
||||||
|
"--resume-task is only for adopting the original first batch "
|
||||||
|
"of an older, non-resumable repair"
|
||||||
|
)
|
||||||
|
|
||||||
|
rows = _repair_batch(db_path, None, batch_size)
|
||||||
|
if not rows:
|
||||||
|
raise ValueError("No documents exist to associate with that task")
|
||||||
|
task = client.get_task(resume_task_uid)
|
||||||
|
if task.get("indexUid") != index_uid:
|
||||||
|
raise ValueError(
|
||||||
|
f"Task {resume_task_uid} belongs to index "
|
||||||
|
f"{task.get('indexUid')!r}, not {index_uid!r}"
|
||||||
|
)
|
||||||
|
if task.get("type") != "documentAdditionOrUpdate":
|
||||||
|
raise ValueError(
|
||||||
|
f"Task {resume_task_uid} is not a document update task"
|
||||||
|
)
|
||||||
|
if task.get("status") in {"failed", "canceled"}:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Task {resume_task_uid} already {task.get('status')}; "
|
||||||
|
"rerun without --resume-task to resubmit that batch"
|
||||||
|
)
|
||||||
|
details = task.get("details") or {}
|
||||||
|
expected = details.get("receivedDocuments")
|
||||||
|
if expected is not None and int(expected) != len(rows):
|
||||||
|
raise ValueError(
|
||||||
|
f"Task {resume_task_uid} contains {expected} documents, "
|
||||||
|
f"but the first repair batch contains {len(rows)}"
|
||||||
|
)
|
||||||
|
_record_pending_repair(
|
||||||
|
db_path, index_uid, resume_task_uid, str(rows[-1]["id"]), len(rows)
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
f"Adopted existing Meilisearch task {resume_task_uid} "
|
||||||
|
f"({len(rows)} documents); no duplicate batch submitted.",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
while True:
|
||||||
|
checkpoint = _ensure_repair_checkpoint(db_path, index_uid)
|
||||||
|
task_uid = checkpoint["pending_task_uid"]
|
||||||
|
|
||||||
|
if task_uid is None:
|
||||||
|
rows = _repair_batch(
|
||||||
|
db_path, checkpoint["last_frame_id"], batch_size
|
||||||
|
)
|
||||||
if not rows:
|
if not rows:
|
||||||
break
|
break
|
||||||
|
|
||||||
docs = [_complete_frame_document(row) for row in rows]
|
docs = [_complete_frame_document(row) for row in rows]
|
||||||
client.add_documents(docs, wait=True)
|
# Wait=False returns the asynchronous task id. Persist it before
|
||||||
total += len(docs)
|
# waiting so a container restart cannot forget queued work.
|
||||||
if progress is not None:
|
task_uid = client.add_documents(docs, wait=False)
|
||||||
progress({"stage": "repair", "documents_processed": total})
|
if task_uid is None:
|
||||||
return total
|
raise RuntimeError(
|
||||||
|
"Meilisearch did not return a task ID; cannot resume safely"
|
||||||
|
)
|
||||||
|
task_uid = int(task_uid)
|
||||||
|
_record_pending_repair(
|
||||||
|
db_path, index_uid, task_uid, str(rows[-1]["id"]), len(rows)
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
f"Queued repair task {task_uid}: {len(rows)} documents "
|
||||||
|
f"(already confirmed: {checkpoint['completed_documents']:,}).",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
last_update = time.monotonic() - 16
|
||||||
|
|
||||||
|
def on_poll(task: dict) -> None:
|
||||||
|
nonlocal last_update
|
||||||
|
now = time.monotonic()
|
||||||
|
if now - last_update >= 15:
|
||||||
|
status = str(task.get("status") or "unknown")
|
||||||
|
print(
|
||||||
|
f"Waiting for Meilisearch task {task_uid}: {status} "
|
||||||
|
f"(repair remains resumable with Ctrl+C).",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
last_update = now
|
||||||
|
|
||||||
|
# Unlike ordinary caption sync, a repair intentionally waits as long
|
||||||
|
# as the index needs. Re-run the command after Ctrl+C or a restart:
|
||||||
|
# the task UID is already stored in SQLite.
|
||||||
|
try:
|
||||||
|
client.wait_for_task(
|
||||||
|
int(task_uid),
|
||||||
|
timeout_seconds=None,
|
||||||
|
poll_seconds=3.0,
|
||||||
|
on_poll=on_poll,
|
||||||
|
)
|
||||||
|
except RuntimeError:
|
||||||
|
# Failed/canceled tasks are safe to resubmit. Preserve the
|
||||||
|
# checkpoint on API/network errors to avoid duplicate jobs.
|
||||||
|
try:
|
||||||
|
task = client.get_task(int(task_uid))
|
||||||
|
except Exception:
|
||||||
|
raise
|
||||||
|
if task.get("status") in {"failed", "canceled"}:
|
||||||
|
with connect(db_path) as con:
|
||||||
|
con.execute(
|
||||||
|
"""
|
||||||
|
UPDATE caption_index_repair_checkpoints
|
||||||
|
SET pending_task_uid = NULL, pending_last_frame_id = NULL,
|
||||||
|
pending_count = 0, updated_at = CURRENT_TIMESTAMP
|
||||||
|
WHERE index_uid = ? AND pending_task_uid = ?
|
||||||
|
""",
|
||||||
|
(index_uid, task_uid),
|
||||||
|
)
|
||||||
|
raise
|
||||||
|
|
||||||
|
count = int(checkpoint["pending_count"]) if checkpoint["pending_task_uid"] else len(rows)
|
||||||
|
confirmed_total = _finish_pending_repair(db_path, index_uid, int(task_uid))
|
||||||
|
restored_this_run += count
|
||||||
|
if progress is not None:
|
||||||
|
progress({
|
||||||
|
"stage": "repair",
|
||||||
|
"documents_processed": confirmed_total,
|
||||||
|
"completed_this_run": restored_this_run,
|
||||||
|
"task_uid": task_uid,
|
||||||
|
})
|
||||||
|
|
||||||
|
print(
|
||||||
|
f"Repair index {index_uid}: {checkpoint['completed_documents']:,} "
|
||||||
|
"documents previously confirmed before this run.",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
return restored_this_run
|
||||||
|
|
||||||
|
|
||||||
def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
|
def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
|
||||||
|
|||||||
Reference in New Issue
Block a user