Files
jellyfin-scene-search/scene_search/overviews.py
T

367 lines
13 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Preview, publish and restore opt-in Jellyfin visual overviews.
Existing canonical caption windows are used as evidence. No frame extraction,
re-captioning or Meilisearch change is required. Metadata publishing is
explicit and the original Jellyfin overview is retained for restoration.
"""
from __future__ import annotations
import hashlib
import json
import re
from pathlib import Path
from typing import Any
import requests
from .config import settings
from .database import connect, init_db
from .jellyfin import JellyfinClient
AI_OVERVIEW_HEADING = "AI visual overview (from sampled video frames; may include spoilers)"
MAX_CAPTIONS = 64
MAX_CAPTION_CHARS = 20000
ALLOWED_KINDS = {"Movie", "Episode", "Video"}
def _media(db_path: Path, *, media_id: str | None, title: str | None) -> dict:
if bool(media_id) == bool(title):
raise ValueError("Specify exactly one of --media-id or --title")
init_db(db_path)
with connect(db_path) as con:
if media_id:
rows = con.execute(
"SELECT * FROM media WHERE id = ? AND source_kind = 'jellyfin'",
(media_id,),
).fetchall()
else:
rows = con.execute(
"SELECT * FROM media WHERE title = ? AND source_kind = 'jellyfin'",
(title,),
).fetchall()
if not rows:
raise ValueError("No indexed Jellyfin video matches that ID/title")
if len(rows) > 1:
raise ValueError(
f"Title is ambiguous ({len(rows)} videos). Use --media-id instead."
)
row = dict(rows[0])
if not row.get("jellyfin_item_id"):
raise ValueError("The indexed video has no Jellyfin item ID")
return row
def _caption_evidence(db_path: Path, media_id: str) -> list[tuple[int, str]]:
with connect(db_path) as con:
rows = con.execute(
"""
SELECT cw.start_ms AS timestamp_ms, cw.caption
FROM caption_windows cw
WHERE cw.media_id = ?
AND cw.caption IS NOT NULL AND TRIM(cw.caption) != ''
UNION ALL
SELECT f.timestamp_ms, f.caption
FROM frames f
WHERE f.media_id = ?
AND f.caption IS NOT NULL AND TRIM(f.caption) != ''
AND NOT EXISTS (
SELECT 1
FROM caption_window_frames cwf
WHERE cwf.frame_id = f.id
)
ORDER BY timestamp_ms
""",
(media_id, media_id),
).fetchall()
return [
(int(row["timestamp_ms"]), " ".join(str(row["caption"]).split()))
for row in rows
]
def _sample(evidence: list[tuple[int, str]], limit: int) -> list[tuple[int, str]]:
"""Sample throughout the title so we don't only describe its opening scenes."""
limit = max(1, min(int(limit), MAX_CAPTIONS))
if len(evidence) <= limit:
return evidence
if limit == 1:
return [evidence[len(evidence) // 2]]
indices = [
(position * (len(evidence) - 1)) // (limit - 1)
for position in range(limit)
]
return [evidence[index] for index in dict.fromkeys(indices)]
def _chat_overview(
captions: list[tuple[int, str]],
*,
model: str,
ollama_url: str,
timeout_seconds: int,
) -> str:
snippets: list[str] = []
budget = MAX_CAPTION_CHARS
for timestamp_ms, caption in captions:
total_seconds = timestamp_ms // 1000
hours, remainder = divmod(total_seconds, 3600)
minutes, seconds = divmod(remainder, 60)
line = f"[{hours:02d}:{minutes:02d}:{seconds:02d}] {caption}"
if len(line) > budget:
break
snippets.append(line)
budget -= len(line) + 1
if not snippets:
raise ValueError("Not enough usable caption evidence")
prompt = (
"Create a concise, natural-language visual overview for a video "
"using ONLY the following time-ordered descriptions generated from "
"sampled Trickplay images. Aim for 80–130 words, in one paragraph. "
"Describe the recurring setting, visible people, actions and broad "
"progression if supported. Do not invent character names, dialogue, "
"motives, backstory, a plot, a genre or an ending. Avoid graphic detail. "
"Do not treat individual disconnected frames as proof of a continuous "
"event. If the descriptions are uncertain or inconsistent, be general. "
"Do not include introductory labels, quotes or commentary.\n\n"
+ "\n".join(snippets)
)
response = requests.post(
f"{ollama_url.rstrip('/')}/api/chat",
json={
"model": model,
"stream": False,
"messages": [{"role": "user", "content": prompt}],
"options": {"temperature": 0.0, "num_predict": 300},
},
timeout=timeout_seconds,
)
if not response.ok:
raise RuntimeError(
f"Ollama {response.status_code}: {response.text[:300]}"
)
data = response.json()
result = (data.get("message") or {}).get("content")
if not isinstance(result, str) or not result.strip():
raise RuntimeError("Ollama returned an empty visual overview")
cleaned = re.sub(r"\s+", " ", result).strip().strip('"')
if len(cleaned) > 2200:
raise RuntimeError(
"LLM response was unusually long; review the prompt/model instead "
"of silently truncating the description"
)
return cleaned
def _draft(db_path: Path, media_id: str) -> dict | None:
with connect(db_path) as con:
row = con.execute(
"SELECT * FROM jellyfin_overview_drafts WHERE media_id = ?",
(media_id,),
).fetchone()
return dict(row) if row is not None else None
def preview(
*,
db_path: Path,
media_id: str | None = None,
title: str | None = None,
model: str | None = None,
max_captions: int = MAX_CAPTIONS,
regenerate: bool = False,
chat_fn=None,
) -> dict[str, Any]:
"""Generate/cache one reviewable visual overview, never update Jellyfin."""
media = _media(db_path, media_id=media_id, title=title)
existing = _draft(db_path, media["id"])
if existing and existing["applied_overview"] is not None:
raise ValueError(
"This video's AI overview is already applied. Restore it "
"before generating a new version."
)
evidence = _caption_evidence(db_path, media["id"])
if not evidence:
raise ValueError("No saved Trickplay captions exist for this video")
chosen = _sample(evidence, max_captions)
fingerprint = hashlib.sha256(
json.dumps(chosen, ensure_ascii=False).encode("utf-8")
).hexdigest()
if (
existing
and not regenerate
and existing["caption_fingerprint"] == fingerprint
):
return {**existing, "title": media["title"], "total_captions": len(evidence)}
selected_model = model or settings.overview_model
if not selected_model:
raise ValueError("OVERVIEW_MODEL is not configured")
produce = chat_fn or _chat_overview
generated = produce(
chosen,
model=selected_model,
ollama_url=settings.ollama_url,
timeout_seconds=settings.overview_timeout_seconds,
)
if not isinstance(generated, str) or not generated.strip():
raise ValueError("LLM returned an empty overview")
with connect(db_path) as con:
con.execute(
"""
INSERT INTO jellyfin_overview_drafts (
media_id, jellyfin_item_id, overview, model,
caption_fingerprint, sampled_captions, updated_at
)
VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
ON CONFLICT(media_id) DO UPDATE SET
jellyfin_item_id = excluded.jellyfin_item_id,
overview = excluded.overview,
model = excluded.model,
caption_fingerprint = excluded.caption_fingerprint,
sampled_captions = excluded.sampled_captions,
updated_at = CURRENT_TIMESTAMP
""",
(
media["id"], media["jellyfin_item_id"], generated.strip(),
selected_model, fingerprint, len(chosen),
),
)
result = _draft(db_path, media["id"])
assert result is not None
return {**result, "title": media["title"], "total_captions": len(evidence)}
def _jellyfin_client() -> JellyfinClient:
return JellyfinClient(
settings.jellyfin_url,
settings.jellyfin_api_key,
settings.jellyfin_timeout_seconds,
)
def _validate_remote(media: dict, item: dict) -> None:
if str(item.get("Id") or "").replace("-", "").lower() != str(
media["jellyfin_item_id"]
).replace("-", "").lower():
raise RuntimeError("Jellyfin returned a different item ID: refusing metadata update")
if str(item.get("Type") or "") not in ALLOWED_KINDS:
raise RuntimeError(
f"Cannot update unsupported Jellyfin type {item.get('Type')!r}"
)
def _write_and_verify(client: JellyfinClient, item: dict, overview: str) -> None:
client.update_item_overview(item, overview)
after = client.item_for_update(str(item["Id"]))
if (after.get("Overview") or "") != overview:
raise RuntimeError(
"Jellyfin accepted the metadata update but the Overview did not "
"match on readback; review this item manually"
)
def publish(
*,
db_path: Path,
media_id: str | None = None,
title: str | None = None,
client: JellyfinClient | None = None,
) -> dict[str, Any]:
"""Append a labelled AI summary while retaining the original synopsis."""
media = _media(db_path, media_id=media_id, title=title)
draft = _draft(db_path, media["id"])
if not draft:
raise ValueError("Generate and review a draft before using --apply")
if str(draft["jellyfin_item_id"]) != str(media["jellyfin_item_id"]):
raise ValueError("Draft no longer matches the indexed Jellyfin item")
api = client or _jellyfin_client()
item = api.item_for_update(str(media["jellyfin_item_id"]))
_validate_remote(media, item)
current = str(item.get("Overview") or "")
if draft["applied_overview"] is not None:
if current == draft["applied_overview"]:
return {**draft, "title": media["title"], "status": "already_applied"}
raise RuntimeError("Jellyfin Overview changed since we applied it; refusing to overwrite")
if AI_OVERVIEW_HEADING in current:
raise RuntimeError(
"Jellyfin Overview already contains an AI visual overview; "
"refusing to append a duplicate"
)
addition = AI_OVERVIEW_HEADING + "\n\n" + draft["overview"]
proposed = current.rstrip() + ("\n\n" if current.strip() else "") + addition
# Backup the unmodified synopsis and intended replacement *before* the
# external API call, so interrupted writes can still be restored.
with connect(db_path) as con:
con.execute(
"""
UPDATE jellyfin_overview_drafts
SET original_overview = ?, applied_overview = ?,
updated_at = CURRENT_TIMESTAMP
WHERE media_id = ?
""",
(current, proposed, media["id"]),
)
_write_and_verify(api, item, proposed)
with connect(db_path) as con:
con.execute(
"""
UPDATE jellyfin_overview_drafts
SET applied_at = CURRENT_TIMESTAMP
WHERE media_id = ?
""",
(media["id"],),
)
return {**_draft(db_path, media["id"]), "title": media["title"], "status": "applied"}
def restore(
*,
db_path: Path,
media_id: str | None = None,
title: str | None = None,
client: JellyfinClient | None = None,
) -> dict[str, Any]:
"""Restore the exact old Overview only when the item is unchanged."""
media = _media(db_path, media_id=media_id, title=title)
draft = _draft(db_path, media["id"])
if not draft or draft["applied_overview"] is None:
raise ValueError("No previous Jellyfin Overview backup exists for this video")
api = client or _jellyfin_client()
item = api.item_for_update(str(media["jellyfin_item_id"]))
_validate_remote(media, item)
current = str(item.get("Overview") or "")
original = str(draft["original_overview"] or "")
if current != draft["applied_overview"] and current != original:
raise RuntimeError(
"Jellyfin Overview was edited after publishing; refusing to overwrite "
"those changes automatically"
)
if current != original:
_write_and_verify(api, item, original)
with connect(db_path) as con:
con.execute(
"""
UPDATE jellyfin_overview_drafts
SET original_overview = NULL, applied_overview = NULL,
applied_at = NULL, updated_at = CURRENT_TIMESTAMP
WHERE media_id = ?
""",
(media["id"],),
)
return {**_draft(db_path, media["id"]), "title": media["title"], "status": "restored"}