mirror of
https://github.com/MrSleeps/jellyfin-scene-search.git
synced 2026-11-20 00:33:36 +00:00
Introduce opt-in caption-derived Jellyfin Overview preview publish and restore
This commit is contained in:
@@ -0,0 +1,366 @@
|
|||||||
|
"""Preview, publish and restore opt-in Jellyfin visual overviews.
|
||||||
|
|
||||||
|
Existing canonical caption windows are used as evidence. No frame extraction,
|
||||||
|
re-captioning or Meilisearch change is required. Metadata publishing is
|
||||||
|
explicit and the original Jellyfin overview is retained for restoration.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
from .config import settings
|
||||||
|
from .database import connect, init_db
|
||||||
|
from .jellyfin import JellyfinClient
|
||||||
|
|
||||||
|
|
||||||
|
AI_OVERVIEW_HEADING = "AI visual overview (from sampled video frames; may include spoilers)"
|
||||||
|
MAX_CAPTIONS = 64
|
||||||
|
MAX_CAPTION_CHARS = 20000
|
||||||
|
ALLOWED_KINDS = {"Movie", "Episode", "Video"}
|
||||||
|
|
||||||
|
|
||||||
|
def _media(db_path: Path, *, media_id: str | None, title: str | None) -> dict:
|
||||||
|
if bool(media_id) == bool(title):
|
||||||
|
raise ValueError("Specify exactly one of --media-id or --title")
|
||||||
|
|
||||||
|
init_db(db_path)
|
||||||
|
with connect(db_path) as con:
|
||||||
|
if media_id:
|
||||||
|
rows = con.execute(
|
||||||
|
"SELECT * FROM media WHERE id = ? AND source_kind = 'jellyfin'",
|
||||||
|
(media_id,),
|
||||||
|
).fetchall()
|
||||||
|
else:
|
||||||
|
rows = con.execute(
|
||||||
|
"SELECT * FROM media WHERE title = ? AND source_kind = 'jellyfin'",
|
||||||
|
(title,),
|
||||||
|
).fetchall()
|
||||||
|
if not rows:
|
||||||
|
raise ValueError("No indexed Jellyfin video matches that ID/title")
|
||||||
|
if len(rows) > 1:
|
||||||
|
raise ValueError(
|
||||||
|
f"Title is ambiguous ({len(rows)} videos). Use --media-id instead."
|
||||||
|
)
|
||||||
|
row = dict(rows[0])
|
||||||
|
if not row.get("jellyfin_item_id"):
|
||||||
|
raise ValueError("The indexed video has no Jellyfin item ID")
|
||||||
|
return row
|
||||||
|
|
||||||
|
|
||||||
|
def _caption_evidence(db_path: Path, media_id: str) -> list[tuple[int, str]]:
|
||||||
|
with connect(db_path) as con:
|
||||||
|
rows = con.execute(
|
||||||
|
"""
|
||||||
|
SELECT cw.start_ms AS timestamp_ms, cw.caption
|
||||||
|
FROM caption_windows cw
|
||||||
|
WHERE cw.media_id = ?
|
||||||
|
AND cw.caption IS NOT NULL AND TRIM(cw.caption) != ''
|
||||||
|
|
||||||
|
UNION ALL
|
||||||
|
|
||||||
|
SELECT f.timestamp_ms, f.caption
|
||||||
|
FROM frames f
|
||||||
|
WHERE f.media_id = ?
|
||||||
|
AND f.caption IS NOT NULL AND TRIM(f.caption) != ''
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM caption_window_frames cwf
|
||||||
|
WHERE cwf.frame_id = f.id
|
||||||
|
)
|
||||||
|
ORDER BY timestamp_ms
|
||||||
|
""",
|
||||||
|
(media_id, media_id),
|
||||||
|
).fetchall()
|
||||||
|
return [
|
||||||
|
(int(row["timestamp_ms"]), " ".join(str(row["caption"]).split()))
|
||||||
|
for row in rows
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _sample(evidence: list[tuple[int, str]], limit: int) -> list[tuple[int, str]]:
|
||||||
|
"""Sample throughout the title so we don't only describe its opening scenes."""
|
||||||
|
limit = max(1, min(int(limit), MAX_CAPTIONS))
|
||||||
|
if len(evidence) <= limit:
|
||||||
|
return evidence
|
||||||
|
if limit == 1:
|
||||||
|
return [evidence[len(evidence) // 2]]
|
||||||
|
indices = [
|
||||||
|
(position * (len(evidence) - 1)) // (limit - 1)
|
||||||
|
for position in range(limit)
|
||||||
|
]
|
||||||
|
return [evidence[index] for index in dict.fromkeys(indices)]
|
||||||
|
|
||||||
|
|
||||||
|
def _chat_overview(
|
||||||
|
captions: list[tuple[int, str]],
|
||||||
|
*,
|
||||||
|
model: str,
|
||||||
|
ollama_url: str,
|
||||||
|
timeout_seconds: int,
|
||||||
|
) -> str:
|
||||||
|
snippets: list[str] = []
|
||||||
|
budget = MAX_CAPTION_CHARS
|
||||||
|
for timestamp_ms, caption in captions:
|
||||||
|
total_seconds = timestamp_ms // 1000
|
||||||
|
hours, remainder = divmod(total_seconds, 3600)
|
||||||
|
minutes, seconds = divmod(remainder, 60)
|
||||||
|
line = f"[{hours:02d}:{minutes:02d}:{seconds:02d}] {caption}"
|
||||||
|
if len(line) > budget:
|
||||||
|
break
|
||||||
|
snippets.append(line)
|
||||||
|
budget -= len(line) + 1
|
||||||
|
|
||||||
|
if not snippets:
|
||||||
|
raise ValueError("Not enough usable caption evidence")
|
||||||
|
|
||||||
|
prompt = (
|
||||||
|
"Create a concise, natural-language visual overview for a video "
|
||||||
|
"using ONLY the following time-ordered descriptions generated from "
|
||||||
|
"sampled Trickplay images. Aim for 80–130 words, in one paragraph. "
|
||||||
|
"Describe the recurring setting, visible people, actions and broad "
|
||||||
|
"progression if supported. Do not invent character names, dialogue, "
|
||||||
|
"motives, backstory, a plot, a genre or an ending. Avoid graphic detail. "
|
||||||
|
"Do not treat individual disconnected frames as proof of a continuous "
|
||||||
|
"event. If the descriptions are uncertain or inconsistent, be general. "
|
||||||
|
"Do not include introductory labels, quotes or commentary.\n\n"
|
||||||
|
+ "\n".join(snippets)
|
||||||
|
)
|
||||||
|
response = requests.post(
|
||||||
|
f"{ollama_url.rstrip('/')}/api/chat",
|
||||||
|
json={
|
||||||
|
"model": model,
|
||||||
|
"stream": False,
|
||||||
|
"messages": [{"role": "user", "content": prompt}],
|
||||||
|
"options": {"temperature": 0.0, "num_predict": 300},
|
||||||
|
},
|
||||||
|
timeout=timeout_seconds,
|
||||||
|
)
|
||||||
|
if not response.ok:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Ollama {response.status_code}: {response.text[:300]}"
|
||||||
|
)
|
||||||
|
data = response.json()
|
||||||
|
result = (data.get("message") or {}).get("content")
|
||||||
|
if not isinstance(result, str) or not result.strip():
|
||||||
|
raise RuntimeError("Ollama returned an empty visual overview")
|
||||||
|
|
||||||
|
cleaned = re.sub(r"\s+", " ", result).strip().strip('"')
|
||||||
|
if len(cleaned) > 2200:
|
||||||
|
raise RuntimeError(
|
||||||
|
"LLM response was unusually long; review the prompt/model instead "
|
||||||
|
"of silently truncating the description"
|
||||||
|
)
|
||||||
|
return cleaned
|
||||||
|
|
||||||
|
|
||||||
|
def _draft(db_path: Path, media_id: str) -> dict | None:
|
||||||
|
with connect(db_path) as con:
|
||||||
|
row = con.execute(
|
||||||
|
"SELECT * FROM jellyfin_overview_drafts WHERE media_id = ?",
|
||||||
|
(media_id,),
|
||||||
|
).fetchone()
|
||||||
|
return dict(row) if row is not None else None
|
||||||
|
|
||||||
|
|
||||||
|
def preview(
|
||||||
|
*,
|
||||||
|
db_path: Path,
|
||||||
|
media_id: str | None = None,
|
||||||
|
title: str | None = None,
|
||||||
|
model: str | None = None,
|
||||||
|
max_captions: int = MAX_CAPTIONS,
|
||||||
|
regenerate: bool = False,
|
||||||
|
chat_fn=None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Generate/cache one reviewable visual overview, never update Jellyfin."""
|
||||||
|
media = _media(db_path, media_id=media_id, title=title)
|
||||||
|
existing = _draft(db_path, media["id"])
|
||||||
|
if existing and existing["applied_overview"] is not None:
|
||||||
|
raise ValueError(
|
||||||
|
"This video's AI overview is already applied. Restore it "
|
||||||
|
"before generating a new version."
|
||||||
|
)
|
||||||
|
|
||||||
|
evidence = _caption_evidence(db_path, media["id"])
|
||||||
|
if not evidence:
|
||||||
|
raise ValueError("No saved Trickplay captions exist for this video")
|
||||||
|
chosen = _sample(evidence, max_captions)
|
||||||
|
fingerprint = hashlib.sha256(
|
||||||
|
json.dumps(chosen, ensure_ascii=False).encode("utf-8")
|
||||||
|
).hexdigest()
|
||||||
|
|
||||||
|
if (
|
||||||
|
existing
|
||||||
|
and not regenerate
|
||||||
|
and existing["caption_fingerprint"] == fingerprint
|
||||||
|
):
|
||||||
|
return {**existing, "title": media["title"], "total_captions": len(evidence)}
|
||||||
|
|
||||||
|
selected_model = model or settings.overview_model
|
||||||
|
if not selected_model:
|
||||||
|
raise ValueError("OVERVIEW_MODEL is not configured")
|
||||||
|
produce = chat_fn or _chat_overview
|
||||||
|
generated = produce(
|
||||||
|
chosen,
|
||||||
|
model=selected_model,
|
||||||
|
ollama_url=settings.ollama_url,
|
||||||
|
timeout_seconds=settings.overview_timeout_seconds,
|
||||||
|
)
|
||||||
|
if not isinstance(generated, str) or not generated.strip():
|
||||||
|
raise ValueError("LLM returned an empty overview")
|
||||||
|
|
||||||
|
with connect(db_path) as con:
|
||||||
|
con.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO jellyfin_overview_drafts (
|
||||||
|
media_id, jellyfin_item_id, overview, model,
|
||||||
|
caption_fingerprint, sampled_captions, updated_at
|
||||||
|
)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
||||||
|
ON CONFLICT(media_id) DO UPDATE SET
|
||||||
|
jellyfin_item_id = excluded.jellyfin_item_id,
|
||||||
|
overview = excluded.overview,
|
||||||
|
model = excluded.model,
|
||||||
|
caption_fingerprint = excluded.caption_fingerprint,
|
||||||
|
sampled_captions = excluded.sampled_captions,
|
||||||
|
updated_at = CURRENT_TIMESTAMP
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
media["id"], media["jellyfin_item_id"], generated.strip(),
|
||||||
|
selected_model, fingerprint, len(chosen),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
result = _draft(db_path, media["id"])
|
||||||
|
assert result is not None
|
||||||
|
return {**result, "title": media["title"], "total_captions": len(evidence)}
|
||||||
|
|
||||||
|
|
||||||
|
def _jellyfin_client() -> JellyfinClient:
|
||||||
|
return JellyfinClient(
|
||||||
|
settings.jellyfin_url,
|
||||||
|
settings.jellyfin_api_key,
|
||||||
|
settings.jellyfin_timeout_seconds,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_remote(media: dict, item: dict) -> None:
|
||||||
|
if str(item.get("Id") or "").replace("-", "").lower() != str(
|
||||||
|
media["jellyfin_item_id"]
|
||||||
|
).replace("-", "").lower():
|
||||||
|
raise RuntimeError("Jellyfin returned a different item ID: refusing metadata update")
|
||||||
|
if str(item.get("Type") or "") not in ALLOWED_KINDS:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Cannot update unsupported Jellyfin type {item.get('Type')!r}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _write_and_verify(client: JellyfinClient, item: dict, overview: str) -> None:
|
||||||
|
client.update_item_overview(item, overview)
|
||||||
|
after = client.item_for_update(str(item["Id"]))
|
||||||
|
if (after.get("Overview") or "") != overview:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Jellyfin accepted the metadata update but the Overview did not "
|
||||||
|
"match on readback; review this item manually"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def publish(
|
||||||
|
*,
|
||||||
|
db_path: Path,
|
||||||
|
media_id: str | None = None,
|
||||||
|
title: str | None = None,
|
||||||
|
client: JellyfinClient | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Append a labelled AI summary while retaining the original synopsis."""
|
||||||
|
media = _media(db_path, media_id=media_id, title=title)
|
||||||
|
draft = _draft(db_path, media["id"])
|
||||||
|
if not draft:
|
||||||
|
raise ValueError("Generate and review a draft before using --apply")
|
||||||
|
if str(draft["jellyfin_item_id"]) != str(media["jellyfin_item_id"]):
|
||||||
|
raise ValueError("Draft no longer matches the indexed Jellyfin item")
|
||||||
|
|
||||||
|
api = client or _jellyfin_client()
|
||||||
|
item = api.item_for_update(str(media["jellyfin_item_id"]))
|
||||||
|
_validate_remote(media, item)
|
||||||
|
current = str(item.get("Overview") or "")
|
||||||
|
if draft["applied_overview"] is not None:
|
||||||
|
if current == draft["applied_overview"]:
|
||||||
|
return {**draft, "title": media["title"], "status": "already_applied"}
|
||||||
|
raise RuntimeError("Jellyfin Overview changed since we applied it; refusing to overwrite")
|
||||||
|
|
||||||
|
if AI_OVERVIEW_HEADING in current:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Jellyfin Overview already contains an AI visual overview; "
|
||||||
|
"refusing to append a duplicate"
|
||||||
|
)
|
||||||
|
|
||||||
|
addition = AI_OVERVIEW_HEADING + "\n\n" + draft["overview"]
|
||||||
|
proposed = current.rstrip() + ("\n\n" if current.strip() else "") + addition
|
||||||
|
|
||||||
|
# Backup the unmodified synopsis and intended replacement *before* the
|
||||||
|
# external API call, so interrupted writes can still be restored.
|
||||||
|
with connect(db_path) as con:
|
||||||
|
con.execute(
|
||||||
|
"""
|
||||||
|
UPDATE jellyfin_overview_drafts
|
||||||
|
SET original_overview = ?, applied_overview = ?,
|
||||||
|
updated_at = CURRENT_TIMESTAMP
|
||||||
|
WHERE media_id = ?
|
||||||
|
""",
|
||||||
|
(current, proposed, media["id"]),
|
||||||
|
)
|
||||||
|
_write_and_verify(api, item, proposed)
|
||||||
|
with connect(db_path) as con:
|
||||||
|
con.execute(
|
||||||
|
"""
|
||||||
|
UPDATE jellyfin_overview_drafts
|
||||||
|
SET applied_at = CURRENT_TIMESTAMP
|
||||||
|
WHERE media_id = ?
|
||||||
|
""",
|
||||||
|
(media["id"],),
|
||||||
|
)
|
||||||
|
return {**_draft(db_path, media["id"]), "title": media["title"], "status": "applied"}
|
||||||
|
|
||||||
|
|
||||||
|
def restore(
|
||||||
|
*,
|
||||||
|
db_path: Path,
|
||||||
|
media_id: str | None = None,
|
||||||
|
title: str | None = None,
|
||||||
|
client: JellyfinClient | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Restore the exact old Overview only when the item is unchanged."""
|
||||||
|
media = _media(db_path, media_id=media_id, title=title)
|
||||||
|
draft = _draft(db_path, media["id"])
|
||||||
|
if not draft or draft["applied_overview"] is None:
|
||||||
|
raise ValueError("No previous Jellyfin Overview backup exists for this video")
|
||||||
|
|
||||||
|
api = client or _jellyfin_client()
|
||||||
|
item = api.item_for_update(str(media["jellyfin_item_id"]))
|
||||||
|
_validate_remote(media, item)
|
||||||
|
current = str(item.get("Overview") or "")
|
||||||
|
original = str(draft["original_overview"] or "")
|
||||||
|
if current != draft["applied_overview"] and current != original:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Jellyfin Overview was edited after publishing; refusing to overwrite "
|
||||||
|
"those changes automatically"
|
||||||
|
)
|
||||||
|
if current != original:
|
||||||
|
_write_and_verify(api, item, original)
|
||||||
|
with connect(db_path) as con:
|
||||||
|
con.execute(
|
||||||
|
"""
|
||||||
|
UPDATE jellyfin_overview_drafts
|
||||||
|
SET original_overview = NULL, applied_overview = NULL,
|
||||||
|
applied_at = NULL, updated_at = CURRENT_TIMESTAMP
|
||||||
|
WHERE media_id = ?
|
||||||
|
""",
|
||||||
|
(media["id"],),
|
||||||
|
)
|
||||||
|
return {**_draft(db_path, media["id"]), "title": media["title"], "status": "restored"}
|
||||||
Reference in New Issue
Block a user