mirror of
https://github.com/MrSleeps/jellyfin-scene-search.git
synced 2026-11-19 00:03:36 +00:00
Route mature-caption overviews to specialist style and preserve full-video evidence coverage
This commit is contained in:
+151
-36
@@ -22,6 +22,7 @@ from .jellyfin import JellyfinClient
|
||||
AI_OVERVIEW_HEADING = "AI visual overview (from sampled video frames; may include spoilers)"
|
||||
MAX_CAPTIONS = 64
|
||||
MAX_CAPTION_CHARS = 20000
|
||||
OVERVIEW_PROMPT_VERSION = 2
|
||||
ALLOWED_KINDS = {"Movie", "Episode", "Video"}
|
||||
|
||||
|
||||
@@ -83,6 +84,87 @@ def _caption_evidence(db_path: Path, media_id: str) -> list[tuple[int, str]]:
|
||||
]
|
||||
|
||||
|
||||
def _caption_route_counts(db_path: Path, media_id: str) -> dict[str, int]:
|
||||
"""Count source captions by the captioner that actually produced them.
|
||||
|
||||
Use saved routing metadata, not a keyword scan of the text or the film
|
||||
title. A single contact-sheet window counts once, not once per frame.
|
||||
"""
|
||||
with connect(db_path) as con:
|
||||
rows = con.execute(
|
||||
"""
|
||||
SELECT route, COUNT(*) AS total FROM (
|
||||
SELECT cw.caption_route AS route
|
||||
FROM caption_windows cw
|
||||
WHERE cw.media_id = ?
|
||||
AND cw.caption IS NOT NULL AND TRIM(cw.caption) != ''
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT f.caption_route AS route
|
||||
FROM frames f
|
||||
WHERE f.media_id = ?
|
||||
AND f.caption IS NOT NULL AND TRIM(f.caption) != ''
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM caption_window_frames cwf
|
||||
WHERE cwf.frame_id = f.id
|
||||
)
|
||||
)
|
||||
GROUP BY route
|
||||
""",
|
||||
(media_id, media_id),
|
||||
).fetchall()
|
||||
counts = {"general": 0, "specialist": 0, "unknown": 0}
|
||||
for row in rows:
|
||||
route = str(row["route"] or "").lower()
|
||||
bucket = route if route in {"general", "specialist"} else "unknown"
|
||||
counts[bucket] += int(row["total"])
|
||||
return counts
|
||||
|
||||
|
||||
def _select_overview_style(
|
||||
counts: dict[str, int], requested: str = "auto"
|
||||
) -> str:
|
||||
if requested in {"general", "specialist"}:
|
||||
return requested
|
||||
if requested != "auto":
|
||||
raise ValueError("Overview style must be auto, general or specialist")
|
||||
total = sum(counts.values())
|
||||
specialist = counts.get("specialist", 0)
|
||||
# At least one-quarter of caption evidence must have specialist routing.
|
||||
# This tolerates mixed-content videos without switching on one stray frame.
|
||||
return "specialist" if total and 4 * specialist >= total else "general"
|
||||
|
||||
|
||||
def _format_caption_evidence(captions: list[tuple[int, str]]) -> str:
|
||||
"""Give every selected time sample space under the prompt character budget.
|
||||
|
||||
The old code stopped at the first long caption that exceeded the budget,
|
||||
silently dropping the rest of a video (particularly detailed specialist
|
||||
captions). Bound *each* caption instead, preserving temporal coverage.
|
||||
"""
|
||||
if not captions:
|
||||
raise ValueError("Not enough usable caption evidence")
|
||||
per_caption = max(
|
||||
60, min(700, (MAX_CAPTION_CHARS // len(captions)) - 18)
|
||||
)
|
||||
snippets: list[str] = []
|
||||
for timestamp_ms, caption in captions:
|
||||
total_seconds = timestamp_ms // 1000
|
||||
hours, remainder = divmod(total_seconds, 3600)
|
||||
minutes, seconds = divmod(remainder, 60)
|
||||
shortened = caption.strip()
|
||||
if len(shortened) > per_caption:
|
||||
shortened = shortened[:per_caption].rsplit(" ", 1)[0].rstrip(".,;: ")
|
||||
if not shortened:
|
||||
shortened = caption[:per_caption]
|
||||
shortened += "…"
|
||||
snippets.append(
|
||||
f"[{hours:02d}:{minutes:02d}:{seconds:02d}] {shortened}"
|
||||
)
|
||||
return "\n".join(snippets)
|
||||
|
||||
|
||||
def _sample(evidence: list[tuple[int, str]], limit: int) -> list[tuple[int, str]]:
|
||||
"""Sample throughout the title so we don't only describe its opening scenes."""
|
||||
limit = max(1, min(int(limit), MAX_CAPTIONS))
|
||||
@@ -103,40 +185,48 @@ def _chat_overview(
|
||||
model: str,
|
||||
ollama_url: str,
|
||||
timeout_seconds: int,
|
||||
style: str = "general",
|
||||
) -> str:
|
||||
snippets: list[str] = []
|
||||
budget = MAX_CAPTION_CHARS
|
||||
for timestamp_ms, caption in captions:
|
||||
total_seconds = timestamp_ms // 1000
|
||||
hours, remainder = divmod(total_seconds, 3600)
|
||||
minutes, seconds = divmod(remainder, 60)
|
||||
line = f"[{hours:02d}:{minutes:02d}:{seconds:02d}] {caption}"
|
||||
if len(line) > budget:
|
||||
break
|
||||
snippets.append(line)
|
||||
budget -= len(line) + 1
|
||||
evidence = _format_caption_evidence(captions)
|
||||
if style == "specialist":
|
||||
prompt = (
|
||||
"Write one concise, neutral visual overview (80–130 words) from "
|
||||
"time-ordered, AI-generated captions of a video. The source may "
|
||||
"contain mature or sensitive subject matter. Use specific but "
|
||||
"non-graphic language: describe recurring visible adults, "
|
||||
"appearance/clothing, setting, camera composition, and the broad "
|
||||
"types of activity that are plainly supported by the captions. "
|
||||
"If intimate or adult activity is central to the video, it is "
|
||||
"appropriate to acknowledge that plainly; avoid euphemisms that "
|
||||
"hide the main subject, but do not give explicit anatomical or "
|
||||
"step-by-step descriptions. Do not invent identities, ages, "
|
||||
"relationships, motives, dialogue, storylines, or an ending. "
|
||||
"Do not guess at details that the source captions contradict or "
|
||||
"do not establish. If the video contains different scenes, "
|
||||
"summarise the recurring elements without asserting continuity. "
|
||||
"Output only the summary paragraph.\n\n"
|
||||
)
|
||||
elif style == "general":
|
||||
prompt = (
|
||||
"Create a concise, natural-language visual overview for a video "
|
||||
"using ONLY the following time-ordered descriptions generated from "
|
||||
"sampled Trickplay images. Aim for 80–130 words, in one paragraph. "
|
||||
"Describe the recurring setting, visible people, actions and broad "
|
||||
"progression if supported. Do not invent character names, dialogue, "
|
||||
"motives, backstory, a plot, a genre or an ending. Avoid graphic detail. "
|
||||
"Do not treat individual disconnected frames as proof of a continuous "
|
||||
"event. If the descriptions are uncertain or inconsistent, be general. "
|
||||
"Do not include introductory labels, quotes or commentary.\n\n"
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unsupported overview style: {style}")
|
||||
|
||||
if not snippets:
|
||||
raise ValueError("Not enough usable caption evidence")
|
||||
|
||||
prompt = (
|
||||
"Create a concise, natural-language visual overview for a video "
|
||||
"using ONLY the following time-ordered descriptions generated from "
|
||||
"sampled Trickplay images. Aim for 80–130 words, in one paragraph. "
|
||||
"Describe the recurring setting, visible people, actions and broad "
|
||||
"progression if supported. Do not invent character names, dialogue, "
|
||||
"motives, backstory, a plot, a genre or an ending. Avoid graphic detail. "
|
||||
"Do not treat individual disconnected frames as proof of a continuous "
|
||||
"event. If the descriptions are uncertain or inconsistent, be general. "
|
||||
"Do not include introductory labels, quotes or commentary.\n\n"
|
||||
+ "\n".join(snippets)
|
||||
)
|
||||
response = requests.post(
|
||||
f"{ollama_url.rstrip('/')}/api/chat",
|
||||
json={
|
||||
"model": model,
|
||||
"stream": False,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"messages": [{"role": "user", "content": prompt + evidence}],
|
||||
"options": {"temperature": 0.0, "num_predict": 300},
|
||||
},
|
||||
timeout=timeout_seconds,
|
||||
@@ -176,6 +266,7 @@ def preview(
|
||||
model: str | None = None,
|
||||
max_captions: int = MAX_CAPTIONS,
|
||||
regenerate: bool = False,
|
||||
style: str = "auto",
|
||||
chat_fn=None,
|
||||
) -> dict[str, Any]:
|
||||
"""Generate/cache one reviewable visual overview, never update Jellyfin."""
|
||||
@@ -191,8 +282,26 @@ def preview(
|
||||
if not evidence:
|
||||
raise ValueError("No saved Trickplay captions exist for this video")
|
||||
chosen = _sample(evidence, max_captions)
|
||||
route_counts = _caption_route_counts(db_path, media["id"])
|
||||
selected_style = _select_overview_style(route_counts, style)
|
||||
selected_model = model or (
|
||||
settings.overview_specialist_model
|
||||
if selected_style == "specialist" and settings.overview_specialist_model
|
||||
else settings.overview_model
|
||||
)
|
||||
if not selected_model:
|
||||
raise ValueError("OVERVIEW_MODEL is not configured")
|
||||
fingerprint = hashlib.sha256(
|
||||
json.dumps(chosen, ensure_ascii=False).encode("utf-8")
|
||||
json.dumps(
|
||||
{
|
||||
"captions": chosen,
|
||||
"style": selected_style,
|
||||
"model": selected_model,
|
||||
"prompt_version": OVERVIEW_PROMPT_VERSION,
|
||||
"char_budget": MAX_CAPTION_CHARS,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
).encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
if (
|
||||
@@ -200,17 +309,19 @@ def preview(
|
||||
and not regenerate
|
||||
and existing["caption_fingerprint"] == fingerprint
|
||||
):
|
||||
return {**existing, "title": media["title"], "total_captions": len(evidence)}
|
||||
return {
|
||||
**existing, "title": media["title"],
|
||||
"total_captions": len(evidence),
|
||||
"route_counts": route_counts,
|
||||
}
|
||||
|
||||
selected_model = model or settings.overview_model
|
||||
if not selected_model:
|
||||
raise ValueError("OVERVIEW_MODEL is not configured")
|
||||
produce = chat_fn or _chat_overview
|
||||
generated = produce(
|
||||
chosen,
|
||||
model=selected_model,
|
||||
ollama_url=settings.ollama_url,
|
||||
timeout_seconds=settings.overview_timeout_seconds,
|
||||
style=selected_style,
|
||||
)
|
||||
if not isinstance(generated, str) or not generated.strip():
|
||||
raise ValueError("LLM returned an empty overview")
|
||||
@@ -219,26 +330,30 @@ def preview(
|
||||
con.execute(
|
||||
"""
|
||||
INSERT INTO jellyfin_overview_drafts (
|
||||
media_id, jellyfin_item_id, overview, model,
|
||||
media_id, jellyfin_item_id, overview, model, style,
|
||||
caption_fingerprint, sampled_captions, updated_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(media_id) DO UPDATE SET
|
||||
jellyfin_item_id = excluded.jellyfin_item_id,
|
||||
overview = excluded.overview,
|
||||
model = excluded.model,
|
||||
style = excluded.style,
|
||||
caption_fingerprint = excluded.caption_fingerprint,
|
||||
sampled_captions = excluded.sampled_captions,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(
|
||||
media["id"], media["jellyfin_item_id"], generated.strip(),
|
||||
selected_model, fingerprint, len(chosen),
|
||||
selected_model, selected_style, fingerprint, len(chosen),
|
||||
),
|
||||
)
|
||||
result = _draft(db_path, media["id"])
|
||||
assert result is not None
|
||||
return {**result, "title": media["title"], "total_captions": len(evidence)}
|
||||
return {
|
||||
**result, "title": media["title"], "total_captions": len(evidence),
|
||||
"route_counts": route_counts,
|
||||
}
|
||||
|
||||
|
||||
def _jellyfin_client() -> JellyfinClient:
|
||||
|
||||
Reference in New Issue
Block a user