From dd05c4bba9cd9d27f2f5d4be57f09c48ca89baf8 Mon Sep 17 00:00:00 2001 From: Sleeps Date: Fri, 9 Oct 2026 12:21:19 +0100 Subject: [PATCH] Add dry-run cleanup and targeted search sync for existing caption prefaces --- scene_search/caption_cleanup.py | 152 ++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 scene_search/caption_cleanup.py diff --git a/scene_search/caption_cleanup.py b/scene_search/caption_cleanup.py new file mode 100644 index 0000000..5e2fcd6 --- /dev/null +++ b/scene_search/caption_cleanup.py @@ -0,0 +1,152 @@ +"""Dry-run first cleanup of meta-prefaced captions already saved in SQLite. + +Only known introductory boilerplate is removed. No Ollama inference, +embedding regeneration or full Meilisearch rebuild is needed. +""" +from __future__ import annotations + +from pathlib import Path +from typing import Any, Callable + +from .caption_text import remove_image_preface +from .database import connect, init_db + + +def clean_stored_caption_prefaces( + db_path: Path, + *, + apply: bool = False, + limit: int = 0, + sync_fn: Callable[[list[str]], int] | None = None, +) -> dict[str, Any]: + if limit < 0: + raise ValueError("limit must be >= 0") + init_db(db_path) + with connect(db_path) as con: + # When the Caption Editor is installed, do not rewrite captions that + # the user has already manually corrected. + history_exists = con.execute( + """SELECT 1 FROM sqlite_master + WHERE type = 'table' AND name = 'caption_edit_history'""" + ).fetchone() is not None + + window_guard = ( + """AND NOT EXISTS ( + SELECT 1 FROM caption_edit_history h + WHERE h.caption_kind = 'window' AND h.caption_id = cw.id + )""" if history_exists else "" + ) + frame_guard = ( + """AND NOT EXISTS ( + SELECT 1 FROM caption_edit_history h + WHERE h.caption_kind = 'frame' AND h.caption_id = f.id + )""" if history_exists else "" + ) + sql = f""" + SELECT 'window' AS kind, cw.id, cw.media_id, cw.caption, + cw.representative_frame_id AS frame_id + FROM caption_windows cw + WHERE cw.caption IS NOT NULL AND TRIM(cw.caption) != '' + {window_guard} + + UNION ALL + + SELECT 'frame' AS kind, f.id, f.media_id, f.caption, + f.id AS frame_id + FROM frames f + WHERE f.caption IS NOT NULL AND TRIM(f.caption) != '' + AND NOT EXISTS ( + SELECT 1 FROM caption_window_frames cwf + WHERE cwf.frame_id = f.id + ) + {frame_guard} + """ + stats: dict[str, Any] = { + "scanned": 0, "matched": 0, "updated": 0, + "meili_synced": 0, "meili_errors": [], + "published_overviews_unchanged": 0, "examples": [], + } + sync = sync_fn + if apply and sync is None: + from .sync import sync_frame_metadata + sync = sync_frame_metadata + + def process_batch(edits: list[tuple[dict[str, Any], str]]) -> None: + if not edits or not apply: + return + frame_ids: list[str] = [] + with connect(db_path) as write_con: + for row, cleaned in edits: + table = "caption_windows" if row["kind"] == "window" else "frames" + extra = ", updated_at = CURRENT_TIMESTAMP" if row["kind"] == "window" else "" + changed = write_con.execute( + f"""UPDATE {table} SET caption = ?, meili_synced_at = NULL{extra} + WHERE id = ? AND caption = ?""", + (cleaned, row["id"], row["caption"]), + ).rowcount + if not changed: + continue + frame_ids.append(str(row["frame_id"])) + stats["updated"] += 1 + write_con.execute( + "UPDATE frames SET meili_synced_at = NULL WHERE id = ?", + (row["frame_id"],), + ) + if history_exists: + write_con.execute( + """INSERT INTO caption_edit_history + (media_id, caption_kind, caption_id, + previous_caption, corrected_caption) + VALUES (?, ?, ?, ?, ?)""", + (row["media_id"], row["kind"], row["id"], + row["caption"], cleaned), + ) + overview = write_con.execute( + """SELECT applied_overview FROM jellyfin_overview_drafts + WHERE media_id = ?""", + (row["media_id"],), + ).fetchone() + if overview and overview["applied_overview"] is not None: + stats["published_overviews_unchanged"] += 1 + else: + write_con.execute( + """DELETE FROM jellyfin_overview_drafts + WHERE media_id = ? AND applied_overview IS NULL""", + (row["media_id"],), + ) + if not frame_ids: + return + try: + assert sync is not None + stats["meili_synced"] += sync(list(dict.fromkeys(frame_ids))) + except Exception as exc: + # Partial updates were saved canonically with null sync markers: + # the usual caption catch-up can retry without regenerating. + stats["meili_errors"].append(str(exc)[:300]) + + with connect(db_path) as con: + cursor = con.execute(sql) + done = False + while not done: + rows = cursor.fetchmany(500) + if not rows: + break + batch: list[tuple[dict[str, Any], str]] = [] + for raw in rows: + row = dict(raw) + stats["scanned"] += 1 + new_caption = remove_image_preface(row["caption"]) + if new_caption == row["caption"]: + continue + stats["matched"] += 1 + if len(stats["examples"]) < 8: + stats["examples"].append({ + "before": row["caption"][:200], + "after": new_caption[:200], + }) + batch.append((row, new_caption)) + if limit and stats["matched"] >= limit: + done = True + break + process_batch(batch) + return stats