Introduce opt-in caption-derived Jellyfin Overview preview publish and restore

This commit is contained in:
Sleeps
2026-10-08 15:44:50 +01:00
parent a12dad9521
commit 9a42b967b9
+366
View File
@@ -0,0 +1,366 @@
"""Preview, publish and restore opt-in Jellyfin visual overviews.
Existing canonical caption windows are used as evidence. No frame extraction,
re-captioning or Meilisearch change is required. Metadata publishing is
explicit and the original Jellyfin overview is retained for restoration.
"""
from __future__ import annotations
import hashlib
import json
import re
from pathlib import Path
from typing import Any
import requests
from .config import settings
from .database import connect, init_db
from .jellyfin import JellyfinClient
AI_OVERVIEW_HEADING = "AI visual overview (from sampled video frames; may include spoilers)"
MAX_CAPTIONS = 64
MAX_CAPTION_CHARS = 20000
ALLOWED_KINDS = {"Movie", "Episode", "Video"}
def _media(db_path: Path, *, media_id: str | None, title: str | None) -> dict:
if bool(media_id) == bool(title):
raise ValueError("Specify exactly one of --media-id or --title")
init_db(db_path)
with connect(db_path) as con:
if media_id:
rows = con.execute(
"SELECT * FROM media WHERE id = ? AND source_kind = 'jellyfin'",
(media_id,),
).fetchall()
else:
rows = con.execute(
"SELECT * FROM media WHERE title = ? AND source_kind = 'jellyfin'",
(title,),
).fetchall()
if not rows:
raise ValueError("No indexed Jellyfin video matches that ID/title")
if len(rows) > 1:
raise ValueError(
f"Title is ambiguous ({len(rows)} videos). Use --media-id instead."
)
row = dict(rows[0])
if not row.get("jellyfin_item_id"):
raise ValueError("The indexed video has no Jellyfin item ID")
return row
def _caption_evidence(db_path: Path, media_id: str) -> list[tuple[int, str]]:
with connect(db_path) as con:
rows = con.execute(
"""
SELECT cw.start_ms AS timestamp_ms, cw.caption
FROM caption_windows cw
WHERE cw.media_id = ?
AND cw.caption IS NOT NULL AND TRIM(cw.caption) != ''
UNION ALL
SELECT f.timestamp_ms, f.caption
FROM frames f
WHERE f.media_id = ?
AND f.caption IS NOT NULL AND TRIM(f.caption) != ''
AND NOT EXISTS (
SELECT 1
FROM caption_window_frames cwf
WHERE cwf.frame_id = f.id
)
ORDER BY timestamp_ms
""",
(media_id, media_id),
).fetchall()
return [
(int(row["timestamp_ms"]), " ".join(str(row["caption"]).split()))
for row in rows
]
def _sample(evidence: list[tuple[int, str]], limit: int) -> list[tuple[int, str]]:
"""Sample throughout the title so we don't only describe its opening scenes."""
limit = max(1, min(int(limit), MAX_CAPTIONS))
if len(evidence) <= limit:
return evidence
if limit == 1:
return [evidence[len(evidence) // 2]]
indices = [
(position * (len(evidence) - 1)) // (limit - 1)
for position in range(limit)
]
return [evidence[index] for index in dict.fromkeys(indices)]
def _chat_overview(
captions: list[tuple[int, str]],
*,
model: str,
ollama_url: str,
timeout_seconds: int,
) -> str:
snippets: list[str] = []
budget = MAX_CAPTION_CHARS
for timestamp_ms, caption in captions:
total_seconds = timestamp_ms // 1000
hours, remainder = divmod(total_seconds, 3600)
minutes, seconds = divmod(remainder, 60)
line = f"[{hours:02d}:{minutes:02d}:{seconds:02d}] {caption}"
if len(line) > budget:
break
snippets.append(line)
budget -= len(line) + 1
if not snippets:
raise ValueError("Not enough usable caption evidence")
prompt = (
"Create a concise, natural-language visual overview for a video "
"using ONLY the following time-ordered descriptions generated from "
"sampled Trickplay images. Aim for 80–130 words, in one paragraph. "
"Describe the recurring setting, visible people, actions and broad "
"progression if supported. Do not invent character names, dialogue, "
"motives, backstory, a plot, a genre or an ending. Avoid graphic detail. "
"Do not treat individual disconnected frames as proof of a continuous "
"event. If the descriptions are uncertain or inconsistent, be general. "
"Do not include introductory labels, quotes or commentary.\n\n"
+ "\n".join(snippets)
)
response = requests.post(
f"{ollama_url.rstrip('/')}/api/chat",
json={
"model": model,
"stream": False,
"messages": [{"role": "user", "content": prompt}],
"options": {"temperature": 0.0, "num_predict": 300},
},
timeout=timeout_seconds,
)
if not response.ok:
raise RuntimeError(
f"Ollama {response.status_code}: {response.text[:300]}"
)
data = response.json()
result = (data.get("message") or {}).get("content")
if not isinstance(result, str) or not result.strip():
raise RuntimeError("Ollama returned an empty visual overview")
cleaned = re.sub(r"\s+", " ", result).strip().strip('"')
if len(cleaned) > 2200:
raise RuntimeError(
"LLM response was unusually long; review the prompt/model instead "
"of silently truncating the description"
)
return cleaned
def _draft(db_path: Path, media_id: str) -> dict | None:
with connect(db_path) as con:
row = con.execute(
"SELECT * FROM jellyfin_overview_drafts WHERE media_id = ?",
(media_id,),
).fetchone()
return dict(row) if row is not None else None
def preview(
*,
db_path: Path,
media_id: str | None = None,
title: str | None = None,
model: str | None = None,
max_captions: int = MAX_CAPTIONS,
regenerate: bool = False,
chat_fn=None,
) -> dict[str, Any]:
"""Generate/cache one reviewable visual overview, never update Jellyfin."""
media = _media(db_path, media_id=media_id, title=title)
existing = _draft(db_path, media["id"])
if existing and existing["applied_overview"] is not None:
raise ValueError(
"This video's AI overview is already applied. Restore it "
"before generating a new version."
)
evidence = _caption_evidence(db_path, media["id"])
if not evidence:
raise ValueError("No saved Trickplay captions exist for this video")
chosen = _sample(evidence, max_captions)
fingerprint = hashlib.sha256(
json.dumps(chosen, ensure_ascii=False).encode("utf-8")
).hexdigest()
if (
existing
and not regenerate
and existing["caption_fingerprint"] == fingerprint
):
return {**existing, "title": media["title"], "total_captions": len(evidence)}
selected_model = model or settings.overview_model
if not selected_model:
raise ValueError("OVERVIEW_MODEL is not configured")
produce = chat_fn or _chat_overview
generated = produce(
chosen,
model=selected_model,
ollama_url=settings.ollama_url,
timeout_seconds=settings.overview_timeout_seconds,
)
if not isinstance(generated, str) or not generated.strip():
raise ValueError("LLM returned an empty overview")
with connect(db_path) as con:
con.execute(
"""
INSERT INTO jellyfin_overview_drafts (
media_id, jellyfin_item_id, overview, model,
caption_fingerprint, sampled_captions, updated_at
)
VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
ON CONFLICT(media_id) DO UPDATE SET
jellyfin_item_id = excluded.jellyfin_item_id,
overview = excluded.overview,
model = excluded.model,
caption_fingerprint = excluded.caption_fingerprint,
sampled_captions = excluded.sampled_captions,
updated_at = CURRENT_TIMESTAMP
""",
(
media["id"], media["jellyfin_item_id"], generated.strip(),
selected_model, fingerprint, len(chosen),
),
)
result = _draft(db_path, media["id"])
assert result is not None
return {**result, "title": media["title"], "total_captions": len(evidence)}
def _jellyfin_client() -> JellyfinClient:
return JellyfinClient(
settings.jellyfin_url,
settings.jellyfin_api_key,
settings.jellyfin_timeout_seconds,
)
def _validate_remote(media: dict, item: dict) -> None:
if str(item.get("Id") or "").replace("-", "").lower() != str(
media["jellyfin_item_id"]
).replace("-", "").lower():
raise RuntimeError("Jellyfin returned a different item ID: refusing metadata update")
if str(item.get("Type") or "") not in ALLOWED_KINDS:
raise RuntimeError(
f"Cannot update unsupported Jellyfin type {item.get('Type')!r}"
)
def _write_and_verify(client: JellyfinClient, item: dict, overview: str) -> None:
client.update_item_overview(item, overview)
after = client.item_for_update(str(item["Id"]))
if (after.get("Overview") or "") != overview:
raise RuntimeError(
"Jellyfin accepted the metadata update but the Overview did not "
"match on readback; review this item manually"
)
def publish(
*,
db_path: Path,
media_id: str | None = None,
title: str | None = None,
client: JellyfinClient | None = None,
) -> dict[str, Any]:
"""Append a labelled AI summary while retaining the original synopsis."""
media = _media(db_path, media_id=media_id, title=title)
draft = _draft(db_path, media["id"])
if not draft:
raise ValueError("Generate and review a draft before using --apply")
if str(draft["jellyfin_item_id"]) != str(media["jellyfin_item_id"]):
raise ValueError("Draft no longer matches the indexed Jellyfin item")
api = client or _jellyfin_client()
item = api.item_for_update(str(media["jellyfin_item_id"]))
_validate_remote(media, item)
current = str(item.get("Overview") or "")
if draft["applied_overview"] is not None:
if current == draft["applied_overview"]:
return {**draft, "title": media["title"], "status": "already_applied"}
raise RuntimeError("Jellyfin Overview changed since we applied it; refusing to overwrite")
if AI_OVERVIEW_HEADING in current:
raise RuntimeError(
"Jellyfin Overview already contains an AI visual overview; "
"refusing to append a duplicate"
)
addition = AI_OVERVIEW_HEADING + "\n\n" + draft["overview"]
proposed = current.rstrip() + ("\n\n" if current.strip() else "") + addition
# Backup the unmodified synopsis and intended replacement *before* the
# external API call, so interrupted writes can still be restored.
with connect(db_path) as con:
con.execute(
"""
UPDATE jellyfin_overview_drafts
SET original_overview = ?, applied_overview = ?,
updated_at = CURRENT_TIMESTAMP
WHERE media_id = ?
""",
(current, proposed, media["id"]),
)
_write_and_verify(api, item, proposed)
with connect(db_path) as con:
con.execute(
"""
UPDATE jellyfin_overview_drafts
SET applied_at = CURRENT_TIMESTAMP
WHERE media_id = ?
""",
(media["id"],),
)
return {**_draft(db_path, media["id"]), "title": media["title"], "status": "applied"}
def restore(
*,
db_path: Path,
media_id: str | None = None,
title: str | None = None,
client: JellyfinClient | None = None,
) -> dict[str, Any]:
"""Restore the exact old Overview only when the item is unchanged."""
media = _media(db_path, media_id=media_id, title=title)
draft = _draft(db_path, media["id"])
if not draft or draft["applied_overview"] is None:
raise ValueError("No previous Jellyfin Overview backup exists for this video")
api = client or _jellyfin_client()
item = api.item_for_update(str(media["jellyfin_item_id"]))
_validate_remote(media, item)
current = str(item.get("Overview") or "")
original = str(draft["original_overview"] or "")
if current != draft["applied_overview"] and current != original:
raise RuntimeError(
"Jellyfin Overview was edited after publishing; refusing to overwrite "
"those changes automatically"
)
if current != original:
_write_and_verify(api, item, original)
with connect(db_path) as con:
con.execute(
"""
UPDATE jellyfin_overview_drafts
SET original_overview = NULL, applied_overview = NULL,
applied_at = NULL, updated_at = CURRENT_TIMESTAMP
WHERE media_id = ?
""",
(media["id"],),
)
return {**_draft(db_path, media["id"]), "title": media["title"], "status": "restored"}