Files
jellyfin-scene-search/scene_search/caption_windows.py
T
2026-10-07 15:29:12 +01:00

506 lines
14 KiB
Python

from __future__ import annotations
import hashlib
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from PIL import Image
from .database import connect
from .frame_source import FrameSource
RoutedFrame = tuple[Any, str, float, str]
@dataclass(frozen=True)
class CaptionWindow:
id: str
media_id: str
title: str
start_ms: int
end_ms: int
representative_row: Any
members: tuple[RoutedFrame, ...]
sample_rows: tuple[Any, ...]
route: str
score: float
reason: str
route_model: str
route_policy: str
window_policy: str
@property
def member_frame_ids(self) -> list[str]:
return [str(item[0]["id"]) for item in self.members]
def caption_window_policy(window_seconds: float, max_frames: int) -> str:
return (
"contact-sheet-v1|"
f"seconds={float(window_seconds):.3f}|"
f"max-frames={max(1, int(max_frames))}"
)
def _sample_indexes(count: int, max_frames: int) -> list[int]:
if count <= 0:
return []
max_frames = max(1, int(max_frames))
if count <= max_frames:
return list(range(count))
if max_frames == 1:
return [count // 2]
indexes = [
int(round(position * (count - 1) / (max_frames - 1)))
for position in range(max_frames)
]
return list(dict.fromkeys(indexes))
def _make_caption_window(
media_id: str,
bucket: int,
members: list[RoutedFrame],
*,
window_ms: int,
max_frames: int,
route_model: str,
route_policy: str,
window_policy: str,
) -> CaptionWindow:
members.sort(key=lambda item: int(item[0]["timestamp_ms"]))
start_ms = bucket * window_ms
last_row = members[-1][0]
interval_ms = max(1, int(last_row["interval_ms"] or window_ms))
end_ms = min(
start_ms + window_ms,
int(last_row["timestamp_ms"]) + interval_ms,
)
if end_ms <= start_ms:
end_ms = start_ms + window_ms
specialist_members = [item for item in members if item[1] == "specialist"]
route = "specialist" if specialist_members else "general"
scored_members = specialist_members or members
score = max(float(item[2]) for item in scored_members)
reason = (
"window-member-specialist"
if specialist_members
else "window-all-general"
)
sample_indexes = _sample_indexes(len(members), max_frames)
sample_rows = tuple(members[index][0] for index in sample_indexes)
representative_row = sample_rows[len(sample_rows) // 2]
window_id = hashlib.sha256(
f"{media_id}:{start_ms}:{end_ms}".encode()
).hexdigest()[:40]
return CaptionWindow(
id=window_id,
media_id=media_id,
title=str(members[0][0]["title"]),
start_ms=start_ms,
end_ms=end_ms,
representative_row=representative_row,
members=tuple(members),
sample_rows=sample_rows,
route=route,
score=score,
reason=reason,
route_model=route_model,
route_policy=route_policy,
window_policy=window_policy,
)
def iter_caption_windows(
routed_rows: list[RoutedFrame],
*,
window_seconds: float,
max_frames: int,
route_model: str,
route_policy: str,
):
"""Yield windows without materialising the full window set in memory.
routed_rows must be ordered by media id and timestamp. The caption
selection query guarantees that order for production calls.
"""
if not routed_rows:
return
window_ms = max(1, int(round(float(window_seconds) * 1000)))
policy = caption_window_policy(window_seconds, max_frames)
current_key: tuple[str, int] | None = None
members: list[RoutedFrame] = []
for item in routed_rows:
row = item[0]
media_id = str(row["media_id"])
bucket = int(row["timestamp_ms"]) // window_ms
key = (media_id, bucket)
if current_key is not None and key != current_key:
yield _make_caption_window(
current_key[0],
current_key[1],
members,
window_ms=window_ms,
max_frames=max_frames,
route_model=route_model,
route_policy=route_policy,
window_policy=policy,
)
members = []
current_key = key
members.append(item)
if current_key is not None and members:
yield _make_caption_window(
current_key[0],
current_key[1],
members,
window_ms=window_ms,
max_frames=max_frames,
route_model=route_model,
route_policy=route_policy,
window_policy=policy,
)
def caption_window_route_counts(
routed_rows: list[RoutedFrame],
*,
window_seconds: float,
max_frames: int,
route_model: str,
route_policy: str,
) -> tuple[int, int, int]:
general = 0
specialist = 0
for window in iter_caption_windows(
routed_rows,
window_seconds=window_seconds,
max_frames=max_frames,
route_model=route_model,
route_policy=route_policy,
):
if window.route == "specialist":
specialist += 1
else:
general += 1
return general + specialist, general, specialist
def group_caption_windows(
routed_rows: list[RoutedFrame],
*,
window_seconds: float,
max_frames: int,
route_model: str,
route_policy: str,
) -> list[CaptionWindow]:
"""Materialising helper kept for focused tests and small callers."""
return list(
iter_caption_windows(
routed_rows,
window_seconds=window_seconds,
max_frames=max_frames,
route_model=route_model,
route_policy=route_policy,
)
)
def build_contact_sheet(
source: FrameSource,
window: CaptionWindow,
*,
gap_px: int = 4,
max_height: int = 360,
) -> Image.Image:
images: list[Image.Image] = []
try:
for row in window.sample_rows:
images.append(source.image(row))
if not images:
raise RuntimeError(f"Caption window {window.id} has no source frames")
target_height = min(max(image.height for image in images), max_height)
widths = [
max(1, int(round(image.width * target_height / image.height)))
for image in images
]
canvas = Image.new(
"RGB",
(sum(widths) + gap_px * (len(images) - 1), target_height),
(0, 0, 0),
)
left = 0
for image, width in zip(images, widths, strict=True):
if image.size == (width, target_height):
scaled = image
close_scaled = False
else:
scaled = image.resize(
(width, target_height),
Image.Resampling.LANCZOS,
)
close_scaled = True
try:
canvas.paste(scaled, (left, 0))
finally:
if close_scaled:
scaled.close()
left += width + gap_px
return canvas
finally:
for image in images:
image.close()
def save_caption_window(
db_path: Path,
window: CaptionWindow,
*,
caption: str,
model: str,
) -> list[str]:
frame_ids = window.member_frame_ids
if not frame_ids:
raise RuntimeError("Cannot save a caption window without member frames")
placeholders = ",".join("?" for _ in frame_ids)
with connect(db_path) as con:
con.execute(
"""
INSERT INTO caption_windows (
id, media_id, start_ms, end_ms, representative_frame_id,
caption, caption_model, captioned_at,
caption_route, caption_route_reason, route_score,
route_model, route_policy, routed_at, window_policy,
meili_synced_at, updated_at
)
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?, ?, ?, ?, ?,
CURRENT_TIMESTAMP, ?, NULL, CURRENT_TIMESTAMP)
ON CONFLICT(id) DO UPDATE SET
media_id = excluded.media_id,
start_ms = excluded.start_ms,
end_ms = excluded.end_ms,
representative_frame_id = excluded.representative_frame_id,
caption = excluded.caption,
caption_model = excluded.caption_model,
captioned_at = CURRENT_TIMESTAMP,
caption_route = excluded.caption_route,
caption_route_reason = excluded.caption_route_reason,
route_score = excluded.route_score,
route_model = excluded.route_model,
route_policy = excluded.route_policy,
routed_at = CURRENT_TIMESTAMP,
window_policy = excluded.window_policy,
meili_synced_at = NULL,
updated_at = CURRENT_TIMESTAMP
""",
(
window.id,
window.media_id,
window.start_ms,
window.end_ms,
str(window.representative_row["id"]),
caption,
model,
window.route,
window.reason,
window.score,
window.route_model,
window.route_policy,
window.window_policy,
),
)
con.execute(
f"DELETE FROM caption_window_frames WHERE frame_id IN ({placeholders})",
frame_ids,
)
con.executemany(
"""
INSERT INTO caption_window_frames (window_id, frame_id, position)
VALUES (?, ?, ?)
""",
[
(window.id, frame_id, position)
for position, frame_id in enumerate(frame_ids)
],
)
con.execute(
f"""
UPDATE frames
SET caption = NULL,
caption_model = NULL,
captioned_at = NULL,
meili_synced_at = NULL
WHERE id IN ({placeholders})
""",
frame_ids,
)
con.execute(
"""
DELETE FROM caption_windows
WHERE media_id = ?
AND id != ?
AND NOT EXISTS (
SELECT 1
FROM caption_window_frames cwf
WHERE cwf.window_id = caption_windows.id
)
""",
(window.media_id, window.id),
)
return frame_ids
def count_valid_caption_windows(
db_path: Path,
library_ids: list[str],
*,
window_policy: str,
route_policy: str,
general_model: str,
specialist_model: str,
) -> int:
if not library_ids:
return 0
placeholders = ",".join("?" for _ in library_ids)
with connect(db_path) as con:
row = con.execute(
f"""
SELECT COUNT(*) AS total
FROM caption_windows cw
JOIN media m ON m.id = cw.media_id
WHERE m.source_kind = 'jellyfin'
AND m.jellyfin_library_id IN ({placeholders})
AND cw.caption IS NOT NULL
AND cw.window_policy = ?
AND cw.route_policy = ?
AND (
(cw.caption_route = 'general' AND cw.caption_model = ?)
OR (cw.caption_route = 'specialist' AND cw.caption_model = ?)
)
""",
[
*library_ids,
window_policy,
route_policy,
general_model,
specialist_model,
],
).fetchone()
return int(row["total"] or 0)
def load_media_caption_windows(db_path: Path, media_id: str) -> list[dict[str, Any]]:
with connect(db_path) as con:
rows = con.execute(
"""
SELECT *
FROM caption_windows
WHERE media_id = ?
ORDER BY start_ms
""",
(media_id,),
).fetchall()
snapshots: list[dict[str, Any]] = []
for row in rows:
members = con.execute(
"""
SELECT frame_id, position
FROM caption_window_frames
WHERE window_id = ?
ORDER BY position
""",
(row["id"],),
).fetchall()
snapshots.append(
{
"window": dict(row),
"members": [
(str(member["frame_id"]), int(member["position"]))
for member in members
],
}
)
return snapshots
def restore_media_caption_windows(
con: Any,
snapshots: list[dict[str, Any]],
frame_ids: set[str],
) -> int:
restored = 0
for snapshot in snapshots:
row = snapshot["window"]
representative = str(row["representative_frame_id"])
members = [
(frame_id, position)
for frame_id, position in snapshot["members"]
if frame_id in frame_ids
]
if representative not in frame_ids or not members:
continue
con.execute(
"""
INSERT INTO caption_windows (
id, media_id, start_ms, end_ms, representative_frame_id,
caption, caption_model, captioned_at,
caption_route, caption_route_reason, route_score,
route_model, route_policy, routed_at, window_policy,
meili_synced_at, created_at, updated_at
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?)
""",
(
row["id"],
row["media_id"],
row["start_ms"],
row["end_ms"],
representative,
row["caption"],
row["caption_model"],
row["captioned_at"],
row["caption_route"],
row["caption_route_reason"],
row["route_score"],
row["route_model"],
row["route_policy"],
row["routed_at"],
row["window_policy"],
row["created_at"],
row["updated_at"],
),
)
con.executemany(
"""
INSERT INTO caption_window_frames (window_id, frame_id, position)
VALUES (?, ?, ?)
""",
[(row["id"], frame_id, position) for frame_id, position in members],
)
restored += 1
return restored