mirror of
https://github.com/MrSleeps/jellyfin-scene-search.git
synced 2026-11-20 00:33:36 +00:00
Introduce opt-in caption-derived Jellyfin Overview preview publish and restore
This commit is contained in:
@@ -0,0 +1,366 @@
|
||||
"""Preview, publish and restore opt-in Jellyfin visual overviews.
|
||||
|
||||
Existing canonical caption windows are used as evidence. No frame extraction,
|
||||
re-captioning or Meilisearch change is required. Metadata publishing is
|
||||
explicit and the original Jellyfin overview is retained for restoration.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from .config import settings
|
||||
from .database import connect, init_db
|
||||
from .jellyfin import JellyfinClient
|
||||
|
||||
|
||||
AI_OVERVIEW_HEADING = "AI visual overview (from sampled video frames; may include spoilers)"
|
||||
MAX_CAPTIONS = 64
|
||||
MAX_CAPTION_CHARS = 20000
|
||||
ALLOWED_KINDS = {"Movie", "Episode", "Video"}
|
||||
|
||||
|
||||
def _media(db_path: Path, *, media_id: str | None, title: str | None) -> dict:
|
||||
if bool(media_id) == bool(title):
|
||||
raise ValueError("Specify exactly one of --media-id or --title")
|
||||
|
||||
init_db(db_path)
|
||||
with connect(db_path) as con:
|
||||
if media_id:
|
||||
rows = con.execute(
|
||||
"SELECT * FROM media WHERE id = ? AND source_kind = 'jellyfin'",
|
||||
(media_id,),
|
||||
).fetchall()
|
||||
else:
|
||||
rows = con.execute(
|
||||
"SELECT * FROM media WHERE title = ? AND source_kind = 'jellyfin'",
|
||||
(title,),
|
||||
).fetchall()
|
||||
if not rows:
|
||||
raise ValueError("No indexed Jellyfin video matches that ID/title")
|
||||
if len(rows) > 1:
|
||||
raise ValueError(
|
||||
f"Title is ambiguous ({len(rows)} videos). Use --media-id instead."
|
||||
)
|
||||
row = dict(rows[0])
|
||||
if not row.get("jellyfin_item_id"):
|
||||
raise ValueError("The indexed video has no Jellyfin item ID")
|
||||
return row
|
||||
|
||||
|
||||
def _caption_evidence(db_path: Path, media_id: str) -> list[tuple[int, str]]:
|
||||
with connect(db_path) as con:
|
||||
rows = con.execute(
|
||||
"""
|
||||
SELECT cw.start_ms AS timestamp_ms, cw.caption
|
||||
FROM caption_windows cw
|
||||
WHERE cw.media_id = ?
|
||||
AND cw.caption IS NOT NULL AND TRIM(cw.caption) != ''
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT f.timestamp_ms, f.caption
|
||||
FROM frames f
|
||||
WHERE f.media_id = ?
|
||||
AND f.caption IS NOT NULL AND TRIM(f.caption) != ''
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM caption_window_frames cwf
|
||||
WHERE cwf.frame_id = f.id
|
||||
)
|
||||
ORDER BY timestamp_ms
|
||||
""",
|
||||
(media_id, media_id),
|
||||
).fetchall()
|
||||
return [
|
||||
(int(row["timestamp_ms"]), " ".join(str(row["caption"]).split()))
|
||||
for row in rows
|
||||
]
|
||||
|
||||
|
||||
def _sample(evidence: list[tuple[int, str]], limit: int) -> list[tuple[int, str]]:
|
||||
"""Sample throughout the title so we don't only describe its opening scenes."""
|
||||
limit = max(1, min(int(limit), MAX_CAPTIONS))
|
||||
if len(evidence) <= limit:
|
||||
return evidence
|
||||
if limit == 1:
|
||||
return [evidence[len(evidence) // 2]]
|
||||
indices = [
|
||||
(position * (len(evidence) - 1)) // (limit - 1)
|
||||
for position in range(limit)
|
||||
]
|
||||
return [evidence[index] for index in dict.fromkeys(indices)]
|
||||
|
||||
|
||||
def _chat_overview(
|
||||
captions: list[tuple[int, str]],
|
||||
*,
|
||||
model: str,
|
||||
ollama_url: str,
|
||||
timeout_seconds: int,
|
||||
) -> str:
|
||||
snippets: list[str] = []
|
||||
budget = MAX_CAPTION_CHARS
|
||||
for timestamp_ms, caption in captions:
|
||||
total_seconds = timestamp_ms // 1000
|
||||
hours, remainder = divmod(total_seconds, 3600)
|
||||
minutes, seconds = divmod(remainder, 60)
|
||||
line = f"[{hours:02d}:{minutes:02d}:{seconds:02d}] {caption}"
|
||||
if len(line) > budget:
|
||||
break
|
||||
snippets.append(line)
|
||||
budget -= len(line) + 1
|
||||
|
||||
if not snippets:
|
||||
raise ValueError("Not enough usable caption evidence")
|
||||
|
||||
prompt = (
|
||||
"Create a concise, natural-language visual overview for a video "
|
||||
"using ONLY the following time-ordered descriptions generated from "
|
||||
"sampled Trickplay images. Aim for 80–130 words, in one paragraph. "
|
||||
"Describe the recurring setting, visible people, actions and broad "
|
||||
"progression if supported. Do not invent character names, dialogue, "
|
||||
"motives, backstory, a plot, a genre or an ending. Avoid graphic detail. "
|
||||
"Do not treat individual disconnected frames as proof of a continuous "
|
||||
"event. If the descriptions are uncertain or inconsistent, be general. "
|
||||
"Do not include introductory labels, quotes or commentary.\n\n"
|
||||
+ "\n".join(snippets)
|
||||
)
|
||||
response = requests.post(
|
||||
f"{ollama_url.rstrip('/')}/api/chat",
|
||||
json={
|
||||
"model": model,
|
||||
"stream": False,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"options": {"temperature": 0.0, "num_predict": 300},
|
||||
},
|
||||
timeout=timeout_seconds,
|
||||
)
|
||||
if not response.ok:
|
||||
raise RuntimeError(
|
||||
f"Ollama {response.status_code}: {response.text[:300]}"
|
||||
)
|
||||
data = response.json()
|
||||
result = (data.get("message") or {}).get("content")
|
||||
if not isinstance(result, str) or not result.strip():
|
||||
raise RuntimeError("Ollama returned an empty visual overview")
|
||||
|
||||
cleaned = re.sub(r"\s+", " ", result).strip().strip('"')
|
||||
if len(cleaned) > 2200:
|
||||
raise RuntimeError(
|
||||
"LLM response was unusually long; review the prompt/model instead "
|
||||
"of silently truncating the description"
|
||||
)
|
||||
return cleaned
|
||||
|
||||
|
||||
def _draft(db_path: Path, media_id: str) -> dict | None:
|
||||
with connect(db_path) as con:
|
||||
row = con.execute(
|
||||
"SELECT * FROM jellyfin_overview_drafts WHERE media_id = ?",
|
||||
(media_id,),
|
||||
).fetchone()
|
||||
return dict(row) if row is not None else None
|
||||
|
||||
|
||||
def preview(
|
||||
*,
|
||||
db_path: Path,
|
||||
media_id: str | None = None,
|
||||
title: str | None = None,
|
||||
model: str | None = None,
|
||||
max_captions: int = MAX_CAPTIONS,
|
||||
regenerate: bool = False,
|
||||
chat_fn=None,
|
||||
) -> dict[str, Any]:
|
||||
"""Generate/cache one reviewable visual overview, never update Jellyfin."""
|
||||
media = _media(db_path, media_id=media_id, title=title)
|
||||
existing = _draft(db_path, media["id"])
|
||||
if existing and existing["applied_overview"] is not None:
|
||||
raise ValueError(
|
||||
"This video's AI overview is already applied. Restore it "
|
||||
"before generating a new version."
|
||||
)
|
||||
|
||||
evidence = _caption_evidence(db_path, media["id"])
|
||||
if not evidence:
|
||||
raise ValueError("No saved Trickplay captions exist for this video")
|
||||
chosen = _sample(evidence, max_captions)
|
||||
fingerprint = hashlib.sha256(
|
||||
json.dumps(chosen, ensure_ascii=False).encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
if (
|
||||
existing
|
||||
and not regenerate
|
||||
and existing["caption_fingerprint"] == fingerprint
|
||||
):
|
||||
return {**existing, "title": media["title"], "total_captions": len(evidence)}
|
||||
|
||||
selected_model = model or settings.overview_model
|
||||
if not selected_model:
|
||||
raise ValueError("OVERVIEW_MODEL is not configured")
|
||||
produce = chat_fn or _chat_overview
|
||||
generated = produce(
|
||||
chosen,
|
||||
model=selected_model,
|
||||
ollama_url=settings.ollama_url,
|
||||
timeout_seconds=settings.overview_timeout_seconds,
|
||||
)
|
||||
if not isinstance(generated, str) or not generated.strip():
|
||||
raise ValueError("LLM returned an empty overview")
|
||||
|
||||
with connect(db_path) as con:
|
||||
con.execute(
|
||||
"""
|
||||
INSERT INTO jellyfin_overview_drafts (
|
||||
media_id, jellyfin_item_id, overview, model,
|
||||
caption_fingerprint, sampled_captions, updated_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(media_id) DO UPDATE SET
|
||||
jellyfin_item_id = excluded.jellyfin_item_id,
|
||||
overview = excluded.overview,
|
||||
model = excluded.model,
|
||||
caption_fingerprint = excluded.caption_fingerprint,
|
||||
sampled_captions = excluded.sampled_captions,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(
|
||||
media["id"], media["jellyfin_item_id"], generated.strip(),
|
||||
selected_model, fingerprint, len(chosen),
|
||||
),
|
||||
)
|
||||
result = _draft(db_path, media["id"])
|
||||
assert result is not None
|
||||
return {**result, "title": media["title"], "total_captions": len(evidence)}
|
||||
|
||||
|
||||
def _jellyfin_client() -> JellyfinClient:
|
||||
return JellyfinClient(
|
||||
settings.jellyfin_url,
|
||||
settings.jellyfin_api_key,
|
||||
settings.jellyfin_timeout_seconds,
|
||||
)
|
||||
|
||||
|
||||
def _validate_remote(media: dict, item: dict) -> None:
|
||||
if str(item.get("Id") or "").replace("-", "").lower() != str(
|
||||
media["jellyfin_item_id"]
|
||||
).replace("-", "").lower():
|
||||
raise RuntimeError("Jellyfin returned a different item ID: refusing metadata update")
|
||||
if str(item.get("Type") or "") not in ALLOWED_KINDS:
|
||||
raise RuntimeError(
|
||||
f"Cannot update unsupported Jellyfin type {item.get('Type')!r}"
|
||||
)
|
||||
|
||||
|
||||
def _write_and_verify(client: JellyfinClient, item: dict, overview: str) -> None:
|
||||
client.update_item_overview(item, overview)
|
||||
after = client.item_for_update(str(item["Id"]))
|
||||
if (after.get("Overview") or "") != overview:
|
||||
raise RuntimeError(
|
||||
"Jellyfin accepted the metadata update but the Overview did not "
|
||||
"match on readback; review this item manually"
|
||||
)
|
||||
|
||||
|
||||
def publish(
|
||||
*,
|
||||
db_path: Path,
|
||||
media_id: str | None = None,
|
||||
title: str | None = None,
|
||||
client: JellyfinClient | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Append a labelled AI summary while retaining the original synopsis."""
|
||||
media = _media(db_path, media_id=media_id, title=title)
|
||||
draft = _draft(db_path, media["id"])
|
||||
if not draft:
|
||||
raise ValueError("Generate and review a draft before using --apply")
|
||||
if str(draft["jellyfin_item_id"]) != str(media["jellyfin_item_id"]):
|
||||
raise ValueError("Draft no longer matches the indexed Jellyfin item")
|
||||
|
||||
api = client or _jellyfin_client()
|
||||
item = api.item_for_update(str(media["jellyfin_item_id"]))
|
||||
_validate_remote(media, item)
|
||||
current = str(item.get("Overview") or "")
|
||||
if draft["applied_overview"] is not None:
|
||||
if current == draft["applied_overview"]:
|
||||
return {**draft, "title": media["title"], "status": "already_applied"}
|
||||
raise RuntimeError("Jellyfin Overview changed since we applied it; refusing to overwrite")
|
||||
|
||||
if AI_OVERVIEW_HEADING in current:
|
||||
raise RuntimeError(
|
||||
"Jellyfin Overview already contains an AI visual overview; "
|
||||
"refusing to append a duplicate"
|
||||
)
|
||||
|
||||
addition = AI_OVERVIEW_HEADING + "\n\n" + draft["overview"]
|
||||
proposed = current.rstrip() + ("\n\n" if current.strip() else "") + addition
|
||||
|
||||
# Backup the unmodified synopsis and intended replacement *before* the
|
||||
# external API call, so interrupted writes can still be restored.
|
||||
with connect(db_path) as con:
|
||||
con.execute(
|
||||
"""
|
||||
UPDATE jellyfin_overview_drafts
|
||||
SET original_overview = ?, applied_overview = ?,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE media_id = ?
|
||||
""",
|
||||
(current, proposed, media["id"]),
|
||||
)
|
||||
_write_and_verify(api, item, proposed)
|
||||
with connect(db_path) as con:
|
||||
con.execute(
|
||||
"""
|
||||
UPDATE jellyfin_overview_drafts
|
||||
SET applied_at = CURRENT_TIMESTAMP
|
||||
WHERE media_id = ?
|
||||
""",
|
||||
(media["id"],),
|
||||
)
|
||||
return {**_draft(db_path, media["id"]), "title": media["title"], "status": "applied"}
|
||||
|
||||
|
||||
def restore(
|
||||
*,
|
||||
db_path: Path,
|
||||
media_id: str | None = None,
|
||||
title: str | None = None,
|
||||
client: JellyfinClient | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Restore the exact old Overview only when the item is unchanged."""
|
||||
media = _media(db_path, media_id=media_id, title=title)
|
||||
draft = _draft(db_path, media["id"])
|
||||
if not draft or draft["applied_overview"] is None:
|
||||
raise ValueError("No previous Jellyfin Overview backup exists for this video")
|
||||
|
||||
api = client or _jellyfin_client()
|
||||
item = api.item_for_update(str(media["jellyfin_item_id"]))
|
||||
_validate_remote(media, item)
|
||||
current = str(item.get("Overview") or "")
|
||||
original = str(draft["original_overview"] or "")
|
||||
if current != draft["applied_overview"] and current != original:
|
||||
raise RuntimeError(
|
||||
"Jellyfin Overview was edited after publishing; refusing to overwrite "
|
||||
"those changes automatically"
|
||||
)
|
||||
if current != original:
|
||||
_write_and_verify(api, item, original)
|
||||
with connect(db_path) as con:
|
||||
con.execute(
|
||||
"""
|
||||
UPDATE jellyfin_overview_drafts
|
||||
SET original_overview = NULL, applied_overview = NULL,
|
||||
applied_at = NULL, updated_at = CURRENT_TIMESTAMP
|
||||
WHERE media_id = ?
|
||||
""",
|
||||
(media["id"],),
|
||||
)
|
||||
return {**_draft(db_path, media["id"]), "title": media["title"], "status": "restored"}
|
||||
Reference in New Issue
Block a user