mirror of
https://github.com/MrSleeps/jellyfin-scene-search.git
synced 2026-11-19 00:03:36 +00:00
367 lines
13 KiB
Python
367 lines
13 KiB
Python
"""Preview, publish and restore opt-in Jellyfin visual overviews.
|
||
|
||
Existing canonical caption windows are used as evidence. No frame extraction,
|
||
re-captioning or Meilisearch change is required. Metadata publishing is
|
||
explicit and the original Jellyfin overview is retained for restoration.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import json
|
||
import re
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import requests
|
||
|
||
from .config import settings
|
||
from .database import connect, init_db
|
||
from .jellyfin import JellyfinClient
|
||
|
||
|
||
AI_OVERVIEW_HEADING = "AI visual overview (from sampled video frames; may include spoilers)"
|
||
MAX_CAPTIONS = 64
|
||
MAX_CAPTION_CHARS = 20000
|
||
ALLOWED_KINDS = {"Movie", "Episode", "Video"}
|
||
|
||
|
||
def _media(db_path: Path, *, media_id: str | None, title: str | None) -> dict:
|
||
if bool(media_id) == bool(title):
|
||
raise ValueError("Specify exactly one of --media-id or --title")
|
||
|
||
init_db(db_path)
|
||
with connect(db_path) as con:
|
||
if media_id:
|
||
rows = con.execute(
|
||
"SELECT * FROM media WHERE id = ? AND source_kind = 'jellyfin'",
|
||
(media_id,),
|
||
).fetchall()
|
||
else:
|
||
rows = con.execute(
|
||
"SELECT * FROM media WHERE title = ? AND source_kind = 'jellyfin'",
|
||
(title,),
|
||
).fetchall()
|
||
if not rows:
|
||
raise ValueError("No indexed Jellyfin video matches that ID/title")
|
||
if len(rows) > 1:
|
||
raise ValueError(
|
||
f"Title is ambiguous ({len(rows)} videos). Use --media-id instead."
|
||
)
|
||
row = dict(rows[0])
|
||
if not row.get("jellyfin_item_id"):
|
||
raise ValueError("The indexed video has no Jellyfin item ID")
|
||
return row
|
||
|
||
|
||
def _caption_evidence(db_path: Path, media_id: str) -> list[tuple[int, str]]:
|
||
with connect(db_path) as con:
|
||
rows = con.execute(
|
||
"""
|
||
SELECT cw.start_ms AS timestamp_ms, cw.caption
|
||
FROM caption_windows cw
|
||
WHERE cw.media_id = ?
|
||
AND cw.caption IS NOT NULL AND TRIM(cw.caption) != ''
|
||
|
||
UNION ALL
|
||
|
||
SELECT f.timestamp_ms, f.caption
|
||
FROM frames f
|
||
WHERE f.media_id = ?
|
||
AND f.caption IS NOT NULL AND TRIM(f.caption) != ''
|
||
AND NOT EXISTS (
|
||
SELECT 1
|
||
FROM caption_window_frames cwf
|
||
WHERE cwf.frame_id = f.id
|
||
)
|
||
ORDER BY timestamp_ms
|
||
""",
|
||
(media_id, media_id),
|
||
).fetchall()
|
||
return [
|
||
(int(row["timestamp_ms"]), " ".join(str(row["caption"]).split()))
|
||
for row in rows
|
||
]
|
||
|
||
|
||
def _sample(evidence: list[tuple[int, str]], limit: int) -> list[tuple[int, str]]:
|
||
"""Sample throughout the title so we don't only describe its opening scenes."""
|
||
limit = max(1, min(int(limit), MAX_CAPTIONS))
|
||
if len(evidence) <= limit:
|
||
return evidence
|
||
if limit == 1:
|
||
return [evidence[len(evidence) // 2]]
|
||
indices = [
|
||
(position * (len(evidence) - 1)) // (limit - 1)
|
||
for position in range(limit)
|
||
]
|
||
return [evidence[index] for index in dict.fromkeys(indices)]
|
||
|
||
|
||
def _chat_overview(
|
||
captions: list[tuple[int, str]],
|
||
*,
|
||
model: str,
|
||
ollama_url: str,
|
||
timeout_seconds: int,
|
||
) -> str:
|
||
snippets: list[str] = []
|
||
budget = MAX_CAPTION_CHARS
|
||
for timestamp_ms, caption in captions:
|
||
total_seconds = timestamp_ms // 1000
|
||
hours, remainder = divmod(total_seconds, 3600)
|
||
minutes, seconds = divmod(remainder, 60)
|
||
line = f"[{hours:02d}:{minutes:02d}:{seconds:02d}] {caption}"
|
||
if len(line) > budget:
|
||
break
|
||
snippets.append(line)
|
||
budget -= len(line) + 1
|
||
|
||
if not snippets:
|
||
raise ValueError("Not enough usable caption evidence")
|
||
|
||
prompt = (
|
||
"Create a concise, natural-language visual overview for a video "
|
||
"using ONLY the following time-ordered descriptions generated from "
|
||
"sampled Trickplay images. Aim for 80–130 words, in one paragraph. "
|
||
"Describe the recurring setting, visible people, actions and broad "
|
||
"progression if supported. Do not invent character names, dialogue, "
|
||
"motives, backstory, a plot, a genre or an ending. Avoid graphic detail. "
|
||
"Do not treat individual disconnected frames as proof of a continuous "
|
||
"event. If the descriptions are uncertain or inconsistent, be general. "
|
||
"Do not include introductory labels, quotes or commentary.\n\n"
|
||
+ "\n".join(snippets)
|
||
)
|
||
response = requests.post(
|
||
f"{ollama_url.rstrip('/')}/api/chat",
|
||
json={
|
||
"model": model,
|
||
"stream": False,
|
||
"messages": [{"role": "user", "content": prompt}],
|
||
"options": {"temperature": 0.0, "num_predict": 300},
|
||
},
|
||
timeout=timeout_seconds,
|
||
)
|
||
if not response.ok:
|
||
raise RuntimeError(
|
||
f"Ollama {response.status_code}: {response.text[:300]}"
|
||
)
|
||
data = response.json()
|
||
result = (data.get("message") or {}).get("content")
|
||
if not isinstance(result, str) or not result.strip():
|
||
raise RuntimeError("Ollama returned an empty visual overview")
|
||
|
||
cleaned = re.sub(r"\s+", " ", result).strip().strip('"')
|
||
if len(cleaned) > 2200:
|
||
raise RuntimeError(
|
||
"LLM response was unusually long; review the prompt/model instead "
|
||
"of silently truncating the description"
|
||
)
|
||
return cleaned
|
||
|
||
|
||
def _draft(db_path: Path, media_id: str) -> dict | None:
|
||
with connect(db_path) as con:
|
||
row = con.execute(
|
||
"SELECT * FROM jellyfin_overview_drafts WHERE media_id = ?",
|
||
(media_id,),
|
||
).fetchone()
|
||
return dict(row) if row is not None else None
|
||
|
||
|
||
def preview(
|
||
*,
|
||
db_path: Path,
|
||
media_id: str | None = None,
|
||
title: str | None = None,
|
||
model: str | None = None,
|
||
max_captions: int = MAX_CAPTIONS,
|
||
regenerate: bool = False,
|
||
chat_fn=None,
|
||
) -> dict[str, Any]:
|
||
"""Generate/cache one reviewable visual overview, never update Jellyfin."""
|
||
media = _media(db_path, media_id=media_id, title=title)
|
||
existing = _draft(db_path, media["id"])
|
||
if existing and existing["applied_overview"] is not None:
|
||
raise ValueError(
|
||
"This video's AI overview is already applied. Restore it "
|
||
"before generating a new version."
|
||
)
|
||
|
||
evidence = _caption_evidence(db_path, media["id"])
|
||
if not evidence:
|
||
raise ValueError("No saved Trickplay captions exist for this video")
|
||
chosen = _sample(evidence, max_captions)
|
||
fingerprint = hashlib.sha256(
|
||
json.dumps(chosen, ensure_ascii=False).encode("utf-8")
|
||
).hexdigest()
|
||
|
||
if (
|
||
existing
|
||
and not regenerate
|
||
and existing["caption_fingerprint"] == fingerprint
|
||
):
|
||
return {**existing, "title": media["title"], "total_captions": len(evidence)}
|
||
|
||
selected_model = model or settings.overview_model
|
||
if not selected_model:
|
||
raise ValueError("OVERVIEW_MODEL is not configured")
|
||
produce = chat_fn or _chat_overview
|
||
generated = produce(
|
||
chosen,
|
||
model=selected_model,
|
||
ollama_url=settings.ollama_url,
|
||
timeout_seconds=settings.overview_timeout_seconds,
|
||
)
|
||
if not isinstance(generated, str) or not generated.strip():
|
||
raise ValueError("LLM returned an empty overview")
|
||
|
||
with connect(db_path) as con:
|
||
con.execute(
|
||
"""
|
||
INSERT INTO jellyfin_overview_drafts (
|
||
media_id, jellyfin_item_id, overview, model,
|
||
caption_fingerprint, sampled_captions, updated_at
|
||
)
|
||
VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
||
ON CONFLICT(media_id) DO UPDATE SET
|
||
jellyfin_item_id = excluded.jellyfin_item_id,
|
||
overview = excluded.overview,
|
||
model = excluded.model,
|
||
caption_fingerprint = excluded.caption_fingerprint,
|
||
sampled_captions = excluded.sampled_captions,
|
||
updated_at = CURRENT_TIMESTAMP
|
||
""",
|
||
(
|
||
media["id"], media["jellyfin_item_id"], generated.strip(),
|
||
selected_model, fingerprint, len(chosen),
|
||
),
|
||
)
|
||
result = _draft(db_path, media["id"])
|
||
assert result is not None
|
||
return {**result, "title": media["title"], "total_captions": len(evidence)}
|
||
|
||
|
||
def _jellyfin_client() -> JellyfinClient:
|
||
return JellyfinClient(
|
||
settings.jellyfin_url,
|
||
settings.jellyfin_api_key,
|
||
settings.jellyfin_timeout_seconds,
|
||
)
|
||
|
||
|
||
def _validate_remote(media: dict, item: dict) -> None:
|
||
if str(item.get("Id") or "").replace("-", "").lower() != str(
|
||
media["jellyfin_item_id"]
|
||
).replace("-", "").lower():
|
||
raise RuntimeError("Jellyfin returned a different item ID: refusing metadata update")
|
||
if str(item.get("Type") or "") not in ALLOWED_KINDS:
|
||
raise RuntimeError(
|
||
f"Cannot update unsupported Jellyfin type {item.get('Type')!r}"
|
||
)
|
||
|
||
|
||
def _write_and_verify(client: JellyfinClient, item: dict, overview: str) -> None:
|
||
client.update_item_overview(item, overview)
|
||
after = client.item_for_update(str(item["Id"]))
|
||
if (after.get("Overview") or "") != overview:
|
||
raise RuntimeError(
|
||
"Jellyfin accepted the metadata update but the Overview did not "
|
||
"match on readback; review this item manually"
|
||
)
|
||
|
||
|
||
def publish(
|
||
*,
|
||
db_path: Path,
|
||
media_id: str | None = None,
|
||
title: str | None = None,
|
||
client: JellyfinClient | None = None,
|
||
) -> dict[str, Any]:
|
||
"""Append a labelled AI summary while retaining the original synopsis."""
|
||
media = _media(db_path, media_id=media_id, title=title)
|
||
draft = _draft(db_path, media["id"])
|
||
if not draft:
|
||
raise ValueError("Generate and review a draft before using --apply")
|
||
if str(draft["jellyfin_item_id"]) != str(media["jellyfin_item_id"]):
|
||
raise ValueError("Draft no longer matches the indexed Jellyfin item")
|
||
|
||
api = client or _jellyfin_client()
|
||
item = api.item_for_update(str(media["jellyfin_item_id"]))
|
||
_validate_remote(media, item)
|
||
current = str(item.get("Overview") or "")
|
||
if draft["applied_overview"] is not None:
|
||
if current == draft["applied_overview"]:
|
||
return {**draft, "title": media["title"], "status": "already_applied"}
|
||
raise RuntimeError("Jellyfin Overview changed since we applied it; refusing to overwrite")
|
||
|
||
if AI_OVERVIEW_HEADING in current:
|
||
raise RuntimeError(
|
||
"Jellyfin Overview already contains an AI visual overview; "
|
||
"refusing to append a duplicate"
|
||
)
|
||
|
||
addition = AI_OVERVIEW_HEADING + "\n\n" + draft["overview"]
|
||
proposed = current.rstrip() + ("\n\n" if current.strip() else "") + addition
|
||
|
||
# Backup the unmodified synopsis and intended replacement *before* the
|
||
# external API call, so interrupted writes can still be restored.
|
||
with connect(db_path) as con:
|
||
con.execute(
|
||
"""
|
||
UPDATE jellyfin_overview_drafts
|
||
SET original_overview = ?, applied_overview = ?,
|
||
updated_at = CURRENT_TIMESTAMP
|
||
WHERE media_id = ?
|
||
""",
|
||
(current, proposed, media["id"]),
|
||
)
|
||
_write_and_verify(api, item, proposed)
|
||
with connect(db_path) as con:
|
||
con.execute(
|
||
"""
|
||
UPDATE jellyfin_overview_drafts
|
||
SET applied_at = CURRENT_TIMESTAMP
|
||
WHERE media_id = ?
|
||
""",
|
||
(media["id"],),
|
||
)
|
||
return {**_draft(db_path, media["id"]), "title": media["title"], "status": "applied"}
|
||
|
||
|
||
def restore(
|
||
*,
|
||
db_path: Path,
|
||
media_id: str | None = None,
|
||
title: str | None = None,
|
||
client: JellyfinClient | None = None,
|
||
) -> dict[str, Any]:
|
||
"""Restore the exact old Overview only when the item is unchanged."""
|
||
media = _media(db_path, media_id=media_id, title=title)
|
||
draft = _draft(db_path, media["id"])
|
||
if not draft or draft["applied_overview"] is None:
|
||
raise ValueError("No previous Jellyfin Overview backup exists for this video")
|
||
|
||
api = client or _jellyfin_client()
|
||
item = api.item_for_update(str(media["jellyfin_item_id"]))
|
||
_validate_remote(media, item)
|
||
current = str(item.get("Overview") or "")
|
||
original = str(draft["original_overview"] or "")
|
||
if current != draft["applied_overview"] and current != original:
|
||
raise RuntimeError(
|
||
"Jellyfin Overview was edited after publishing; refusing to overwrite "
|
||
"those changes automatically"
|
||
)
|
||
if current != original:
|
||
_write_and_verify(api, item, original)
|
||
with connect(db_path) as con:
|
||
con.execute(
|
||
"""
|
||
UPDATE jellyfin_overview_drafts
|
||
SET original_overview = NULL, applied_overview = NULL,
|
||
applied_at = NULL, updated_at = CURRENT_TIMESTAMP
|
||
WHERE media_id = ?
|
||
""",
|
||
(media["id"],),
|
||
)
|
||
return {**_draft(db_path, media["id"]), "title": media["title"], "status": "restored"}
|