mirror of
https://github.com/MrSleeps/jellyfin-scene-search.git
synced 2026-11-19 00:03:36 +00:00
506 lines
14 KiB
Python
506 lines
14 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from PIL import Image
|
|
|
|
from .database import connect
|
|
from .frame_source import FrameSource
|
|
|
|
|
|
RoutedFrame = tuple[Any, str, float, str]
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CaptionWindow:
|
|
id: str
|
|
media_id: str
|
|
title: str
|
|
start_ms: int
|
|
end_ms: int
|
|
representative_row: Any
|
|
members: tuple[RoutedFrame, ...]
|
|
sample_rows: tuple[Any, ...]
|
|
route: str
|
|
score: float
|
|
reason: str
|
|
route_model: str
|
|
route_policy: str
|
|
window_policy: str
|
|
|
|
@property
|
|
def member_frame_ids(self) -> list[str]:
|
|
return [str(item[0]["id"]) for item in self.members]
|
|
|
|
|
|
def caption_window_policy(window_seconds: float, max_frames: int) -> str:
|
|
return (
|
|
"contact-sheet-v1|"
|
|
f"seconds={float(window_seconds):.3f}|"
|
|
f"max-frames={max(1, int(max_frames))}"
|
|
)
|
|
|
|
|
|
def _sample_indexes(count: int, max_frames: int) -> list[int]:
|
|
if count <= 0:
|
|
return []
|
|
max_frames = max(1, int(max_frames))
|
|
if count <= max_frames:
|
|
return list(range(count))
|
|
if max_frames == 1:
|
|
return [count // 2]
|
|
|
|
indexes = [
|
|
int(round(position * (count - 1) / (max_frames - 1)))
|
|
for position in range(max_frames)
|
|
]
|
|
return list(dict.fromkeys(indexes))
|
|
|
|
|
|
def _make_caption_window(
|
|
media_id: str,
|
|
bucket: int,
|
|
members: list[RoutedFrame],
|
|
*,
|
|
window_ms: int,
|
|
max_frames: int,
|
|
route_model: str,
|
|
route_policy: str,
|
|
window_policy: str,
|
|
) -> CaptionWindow:
|
|
members.sort(key=lambda item: int(item[0]["timestamp_ms"]))
|
|
start_ms = bucket * window_ms
|
|
last_row = members[-1][0]
|
|
interval_ms = max(1, int(last_row["interval_ms"] or window_ms))
|
|
end_ms = min(
|
|
start_ms + window_ms,
|
|
int(last_row["timestamp_ms"]) + interval_ms,
|
|
)
|
|
if end_ms <= start_ms:
|
|
end_ms = start_ms + window_ms
|
|
|
|
specialist_members = [item for item in members if item[1] == "specialist"]
|
|
route = "specialist" if specialist_members else "general"
|
|
scored_members = specialist_members or members
|
|
score = max(float(item[2]) for item in scored_members)
|
|
reason = (
|
|
"window-member-specialist"
|
|
if specialist_members
|
|
else "window-all-general"
|
|
)
|
|
|
|
sample_indexes = _sample_indexes(len(members), max_frames)
|
|
sample_rows = tuple(members[index][0] for index in sample_indexes)
|
|
representative_row = sample_rows[len(sample_rows) // 2]
|
|
window_id = hashlib.sha256(
|
|
f"{media_id}:{start_ms}:{end_ms}".encode()
|
|
).hexdigest()[:40]
|
|
|
|
return CaptionWindow(
|
|
id=window_id,
|
|
media_id=media_id,
|
|
title=str(members[0][0]["title"]),
|
|
start_ms=start_ms,
|
|
end_ms=end_ms,
|
|
representative_row=representative_row,
|
|
members=tuple(members),
|
|
sample_rows=sample_rows,
|
|
route=route,
|
|
score=score,
|
|
reason=reason,
|
|
route_model=route_model,
|
|
route_policy=route_policy,
|
|
window_policy=window_policy,
|
|
)
|
|
|
|
|
|
def iter_caption_windows(
|
|
routed_rows: list[RoutedFrame],
|
|
*,
|
|
window_seconds: float,
|
|
max_frames: int,
|
|
route_model: str,
|
|
route_policy: str,
|
|
):
|
|
"""Yield windows without materialising the full window set in memory.
|
|
|
|
routed_rows must be ordered by media id and timestamp. The caption
|
|
selection query guarantees that order for production calls.
|
|
"""
|
|
if not routed_rows:
|
|
return
|
|
|
|
window_ms = max(1, int(round(float(window_seconds) * 1000)))
|
|
policy = caption_window_policy(window_seconds, max_frames)
|
|
current_key: tuple[str, int] | None = None
|
|
members: list[RoutedFrame] = []
|
|
|
|
for item in routed_rows:
|
|
row = item[0]
|
|
media_id = str(row["media_id"])
|
|
bucket = int(row["timestamp_ms"]) // window_ms
|
|
key = (media_id, bucket)
|
|
|
|
if current_key is not None and key != current_key:
|
|
yield _make_caption_window(
|
|
current_key[0],
|
|
current_key[1],
|
|
members,
|
|
window_ms=window_ms,
|
|
max_frames=max_frames,
|
|
route_model=route_model,
|
|
route_policy=route_policy,
|
|
window_policy=policy,
|
|
)
|
|
members = []
|
|
|
|
current_key = key
|
|
members.append(item)
|
|
|
|
if current_key is not None and members:
|
|
yield _make_caption_window(
|
|
current_key[0],
|
|
current_key[1],
|
|
members,
|
|
window_ms=window_ms,
|
|
max_frames=max_frames,
|
|
route_model=route_model,
|
|
route_policy=route_policy,
|
|
window_policy=policy,
|
|
)
|
|
|
|
|
|
def caption_window_route_counts(
|
|
routed_rows: list[RoutedFrame],
|
|
*,
|
|
window_seconds: float,
|
|
max_frames: int,
|
|
route_model: str,
|
|
route_policy: str,
|
|
) -> tuple[int, int, int]:
|
|
general = 0
|
|
specialist = 0
|
|
for window in iter_caption_windows(
|
|
routed_rows,
|
|
window_seconds=window_seconds,
|
|
max_frames=max_frames,
|
|
route_model=route_model,
|
|
route_policy=route_policy,
|
|
):
|
|
if window.route == "specialist":
|
|
specialist += 1
|
|
else:
|
|
general += 1
|
|
return general + specialist, general, specialist
|
|
|
|
|
|
def group_caption_windows(
|
|
routed_rows: list[RoutedFrame],
|
|
*,
|
|
window_seconds: float,
|
|
max_frames: int,
|
|
route_model: str,
|
|
route_policy: str,
|
|
) -> list[CaptionWindow]:
|
|
"""Materialising helper kept for focused tests and small callers."""
|
|
return list(
|
|
iter_caption_windows(
|
|
routed_rows,
|
|
window_seconds=window_seconds,
|
|
max_frames=max_frames,
|
|
route_model=route_model,
|
|
route_policy=route_policy,
|
|
)
|
|
)
|
|
|
|
|
|
def build_contact_sheet(
|
|
source: FrameSource,
|
|
window: CaptionWindow,
|
|
*,
|
|
gap_px: int = 4,
|
|
max_height: int = 360,
|
|
) -> Image.Image:
|
|
images: list[Image.Image] = []
|
|
try:
|
|
for row in window.sample_rows:
|
|
images.append(source.image(row))
|
|
|
|
if not images:
|
|
raise RuntimeError(f"Caption window {window.id} has no source frames")
|
|
|
|
target_height = min(max(image.height for image in images), max_height)
|
|
widths = [
|
|
max(1, int(round(image.width * target_height / image.height)))
|
|
for image in images
|
|
]
|
|
canvas = Image.new(
|
|
"RGB",
|
|
(sum(widths) + gap_px * (len(images) - 1), target_height),
|
|
(0, 0, 0),
|
|
)
|
|
|
|
left = 0
|
|
for image, width in zip(images, widths, strict=True):
|
|
if image.size == (width, target_height):
|
|
scaled = image
|
|
close_scaled = False
|
|
else:
|
|
scaled = image.resize(
|
|
(width, target_height),
|
|
Image.Resampling.LANCZOS,
|
|
)
|
|
close_scaled = True
|
|
try:
|
|
canvas.paste(scaled, (left, 0))
|
|
finally:
|
|
if close_scaled:
|
|
scaled.close()
|
|
left += width + gap_px
|
|
|
|
return canvas
|
|
finally:
|
|
for image in images:
|
|
image.close()
|
|
|
|
|
|
def save_caption_window(
|
|
db_path: Path,
|
|
window: CaptionWindow,
|
|
*,
|
|
caption: str,
|
|
model: str,
|
|
) -> list[str]:
|
|
frame_ids = window.member_frame_ids
|
|
if not frame_ids:
|
|
raise RuntimeError("Cannot save a caption window without member frames")
|
|
|
|
placeholders = ",".join("?" for _ in frame_ids)
|
|
with connect(db_path) as con:
|
|
con.execute(
|
|
"""
|
|
INSERT INTO caption_windows (
|
|
id, media_id, start_ms, end_ms, representative_frame_id,
|
|
caption, caption_model, captioned_at,
|
|
caption_route, caption_route_reason, route_score,
|
|
route_model, route_policy, routed_at, window_policy,
|
|
meili_synced_at, updated_at
|
|
)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?, ?, ?, ?, ?,
|
|
CURRENT_TIMESTAMP, ?, NULL, CURRENT_TIMESTAMP)
|
|
ON CONFLICT(id) DO UPDATE SET
|
|
media_id = excluded.media_id,
|
|
start_ms = excluded.start_ms,
|
|
end_ms = excluded.end_ms,
|
|
representative_frame_id = excluded.representative_frame_id,
|
|
caption = excluded.caption,
|
|
caption_model = excluded.caption_model,
|
|
captioned_at = CURRENT_TIMESTAMP,
|
|
caption_route = excluded.caption_route,
|
|
caption_route_reason = excluded.caption_route_reason,
|
|
route_score = excluded.route_score,
|
|
route_model = excluded.route_model,
|
|
route_policy = excluded.route_policy,
|
|
routed_at = CURRENT_TIMESTAMP,
|
|
window_policy = excluded.window_policy,
|
|
meili_synced_at = NULL,
|
|
updated_at = CURRENT_TIMESTAMP
|
|
""",
|
|
(
|
|
window.id,
|
|
window.media_id,
|
|
window.start_ms,
|
|
window.end_ms,
|
|
str(window.representative_row["id"]),
|
|
caption,
|
|
model,
|
|
window.route,
|
|
window.reason,
|
|
window.score,
|
|
window.route_model,
|
|
window.route_policy,
|
|
window.window_policy,
|
|
),
|
|
)
|
|
|
|
con.execute(
|
|
f"DELETE FROM caption_window_frames WHERE frame_id IN ({placeholders})",
|
|
frame_ids,
|
|
)
|
|
con.executemany(
|
|
"""
|
|
INSERT INTO caption_window_frames (window_id, frame_id, position)
|
|
VALUES (?, ?, ?)
|
|
""",
|
|
[
|
|
(window.id, frame_id, position)
|
|
for position, frame_id in enumerate(frame_ids)
|
|
],
|
|
)
|
|
|
|
con.execute(
|
|
f"""
|
|
UPDATE frames
|
|
SET caption = NULL,
|
|
caption_model = NULL,
|
|
captioned_at = NULL,
|
|
meili_synced_at = NULL
|
|
WHERE id IN ({placeholders})
|
|
""",
|
|
frame_ids,
|
|
)
|
|
|
|
con.execute(
|
|
"""
|
|
DELETE FROM caption_windows
|
|
WHERE media_id = ?
|
|
AND id != ?
|
|
AND NOT EXISTS (
|
|
SELECT 1
|
|
FROM caption_window_frames cwf
|
|
WHERE cwf.window_id = caption_windows.id
|
|
)
|
|
""",
|
|
(window.media_id, window.id),
|
|
)
|
|
|
|
return frame_ids
|
|
|
|
|
|
def count_valid_caption_windows(
|
|
db_path: Path,
|
|
library_ids: list[str],
|
|
*,
|
|
window_policy: str,
|
|
route_policy: str,
|
|
general_model: str,
|
|
specialist_model: str,
|
|
) -> int:
|
|
if not library_ids:
|
|
return 0
|
|
|
|
placeholders = ",".join("?" for _ in library_ids)
|
|
with connect(db_path) as con:
|
|
row = con.execute(
|
|
f"""
|
|
SELECT COUNT(*) AS total
|
|
FROM caption_windows cw
|
|
JOIN media m ON m.id = cw.media_id
|
|
WHERE m.source_kind = 'jellyfin'
|
|
AND m.jellyfin_library_id IN ({placeholders})
|
|
AND cw.caption IS NOT NULL
|
|
AND cw.window_policy = ?
|
|
AND cw.route_policy = ?
|
|
AND (
|
|
(cw.caption_route = 'general' AND cw.caption_model = ?)
|
|
OR (cw.caption_route = 'specialist' AND cw.caption_model = ?)
|
|
)
|
|
""",
|
|
[
|
|
*library_ids,
|
|
window_policy,
|
|
route_policy,
|
|
general_model,
|
|
specialist_model,
|
|
],
|
|
).fetchone()
|
|
return int(row["total"] or 0)
|
|
|
|
|
|
def load_media_caption_windows(db_path: Path, media_id: str) -> list[dict[str, Any]]:
|
|
with connect(db_path) as con:
|
|
rows = con.execute(
|
|
"""
|
|
SELECT *
|
|
FROM caption_windows
|
|
WHERE media_id = ?
|
|
ORDER BY start_ms
|
|
""",
|
|
(media_id,),
|
|
).fetchall()
|
|
|
|
snapshots: list[dict[str, Any]] = []
|
|
for row in rows:
|
|
members = con.execute(
|
|
"""
|
|
SELECT frame_id, position
|
|
FROM caption_window_frames
|
|
WHERE window_id = ?
|
|
ORDER BY position
|
|
""",
|
|
(row["id"],),
|
|
).fetchall()
|
|
snapshots.append(
|
|
{
|
|
"window": dict(row),
|
|
"members": [
|
|
(str(member["frame_id"]), int(member["position"]))
|
|
for member in members
|
|
],
|
|
}
|
|
)
|
|
return snapshots
|
|
|
|
|
|
def restore_media_caption_windows(
|
|
con: Any,
|
|
snapshots: list[dict[str, Any]],
|
|
frame_ids: set[str],
|
|
) -> int:
|
|
restored = 0
|
|
for snapshot in snapshots:
|
|
row = snapshot["window"]
|
|
representative = str(row["representative_frame_id"])
|
|
members = [
|
|
(frame_id, position)
|
|
for frame_id, position in snapshot["members"]
|
|
if frame_id in frame_ids
|
|
]
|
|
if representative not in frame_ids or not members:
|
|
continue
|
|
|
|
con.execute(
|
|
"""
|
|
INSERT INTO caption_windows (
|
|
id, media_id, start_ms, end_ms, representative_frame_id,
|
|
caption, caption_model, captioned_at,
|
|
caption_route, caption_route_reason, route_score,
|
|
route_model, route_policy, routed_at, window_policy,
|
|
meili_synced_at, created_at, updated_at
|
|
)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?)
|
|
""",
|
|
(
|
|
row["id"],
|
|
row["media_id"],
|
|
row["start_ms"],
|
|
row["end_ms"],
|
|
representative,
|
|
row["caption"],
|
|
row["caption_model"],
|
|
row["captioned_at"],
|
|
row["caption_route"],
|
|
row["caption_route_reason"],
|
|
row["route_score"],
|
|
row["route_model"],
|
|
row["route_policy"],
|
|
row["routed_at"],
|
|
row["window_policy"],
|
|
row["created_at"],
|
|
row["updated_at"],
|
|
),
|
|
)
|
|
con.executemany(
|
|
"""
|
|
INSERT INTO caption_window_frames (window_id, frame_id, position)
|
|
VALUES (?, ?, ?)
|
|
""",
|
|
[(row["id"], frame_id, position) for frame_id, position in members],
|
|
)
|
|
restored += 1
|
|
|
|
return restored
|