mirror of
https://github.com/MrSleeps/jellyfin-scene-search.git
synced 2026-11-19 00:03:36 +00:00
Caption chronological 30-second frame windows
This commit is contained in:
@@ -8,11 +8,19 @@ from typing import Any
|
||||
|
||||
from .captioner import ensure_caption_prompt_file, ensure_specialist_prompt_file
|
||||
from .cli import (
|
||||
_caption_queue,
|
||||
_caption_window_policy,
|
||||
_caption_window_queue,
|
||||
_route_model_name,
|
||||
_route_policy,
|
||||
_route_rows,
|
||||
_select_caption_rows,
|
||||
_unload_ollama_model,
|
||||
)
|
||||
from .caption_windows import (
|
||||
caption_window_route_counts,
|
||||
count_valid_caption_windows,
|
||||
iter_caption_windows,
|
||||
)
|
||||
from .config import settings
|
||||
from .database import connect, init_db
|
||||
from .frame_source import FrameSource
|
||||
@@ -227,8 +235,7 @@ class CaptionJobManager:
|
||||
placeholders = ",".join("?" for _ in library_ids)
|
||||
totals = con.execute(
|
||||
f"""
|
||||
SELECT COUNT(*) AS total,
|
||||
SUM(CASE WHEN f.caption IS NOT NULL THEN 1 ELSE 0 END) AS captioned
|
||||
SELECT COUNT(*) AS total
|
||||
FROM frames f
|
||||
JOIN media m ON m.id = f.media_id
|
||||
WHERE m.source_kind = 'jellyfin'
|
||||
@@ -238,7 +245,14 @@ class CaptionJobManager:
|
||||
).fetchone()
|
||||
|
||||
indexed_frames = int(totals["total"] or 0)
|
||||
already_captioned = int(totals["captioned"] or 0)
|
||||
already_captioned = count_valid_caption_windows(
|
||||
settings.db_path,
|
||||
library_ids,
|
||||
window_policy=_caption_window_policy(),
|
||||
route_policy=_route_policy(),
|
||||
general_model=settings.caption_general_model,
|
||||
specialist_model=settings.caption_specialist_model,
|
||||
)
|
||||
|
||||
self._progress(
|
||||
{
|
||||
@@ -374,42 +388,54 @@ class CaptionJobManager:
|
||||
settings.caption_specialist_prompt_file
|
||||
)
|
||||
|
||||
general_rows = [
|
||||
item
|
||||
for item in routed
|
||||
if item[1] == "general"
|
||||
and (
|
||||
args.force
|
||||
or item[0]["caption"] is None
|
||||
or item[0]["caption_model"] != settings.caption_general_model
|
||||
del rows
|
||||
route_model = _route_model_name()
|
||||
route_policy = _route_policy()
|
||||
windows_total, general_total, specialist_total = (
|
||||
caption_window_route_counts(
|
||||
routed,
|
||||
window_seconds=settings.caption_window_seconds,
|
||||
max_frames=settings.caption_window_max_frames,
|
||||
route_model=route_model,
|
||||
route_policy=route_policy,
|
||||
)
|
||||
]
|
||||
specialist_rows = [
|
||||
item
|
||||
for item in routed
|
||||
if item[1] == "specialist"
|
||||
and (
|
||||
args.force
|
||||
or item[0]["caption"] is None
|
||||
or item[0]["caption_model"] != settings.caption_specialist_model
|
||||
)
|
||||
|
||||
def windows_for(route: str):
|
||||
return (
|
||||
window
|
||||
for window in iter_caption_windows(
|
||||
routed,
|
||||
window_seconds=settings.caption_window_seconds,
|
||||
max_frames=settings.caption_window_max_frames,
|
||||
route_model=route_model,
|
||||
route_policy=route_policy,
|
||||
)
|
||||
if window.route == route
|
||||
)
|
||||
]
|
||||
retained = len(routed) - len(general_rows) - len(specialist_rows)
|
||||
frames_total = len(general_rows) + len(specialist_rows)
|
||||
|
||||
retained = (
|
||||
0
|
||||
if args.force
|
||||
else int(self.status().get("already_captioned") or 0)
|
||||
)
|
||||
|
||||
self._progress(
|
||||
{
|
||||
"stage": "captioning",
|
||||
"frames_total": frames_total,
|
||||
"frames_total": windows_total,
|
||||
"frames_completed": 0,
|
||||
"failed": missing,
|
||||
"retained": retained,
|
||||
"general_total": len(general_rows),
|
||||
"specialist_total": len(specialist_rows),
|
||||
"general_total": general_total,
|
||||
"specialist_total": specialist_total,
|
||||
"work_unit": "windows",
|
||||
"caption_frames_grouped": len(routed),
|
||||
"caption_window_seconds": settings.caption_window_seconds,
|
||||
}
|
||||
)
|
||||
|
||||
if general_rows:
|
||||
if general_total:
|
||||
_unload_ollama_model(settings.caption_specialist_model)
|
||||
|
||||
def general_progress(update: dict) -> None:
|
||||
@@ -418,19 +444,20 @@ class CaptionJobManager:
|
||||
self._progress(adjusted)
|
||||
|
||||
with FrameSource() as frame_source:
|
||||
general_done, general_failed = _caption_queue(
|
||||
general_done, general_failed = _caption_window_queue(
|
||||
"General queue",
|
||||
general_rows,
|
||||
windows_for("general"),
|
||||
settings.caption_general_model,
|
||||
general_prompt,
|
||||
progress=general_progress,
|
||||
total_frames=frames_total,
|
||||
queue_total=general_total,
|
||||
total_windows=windows_total,
|
||||
frame_source=frame_source,
|
||||
before_frame=lambda: self._wait_if_paused("captioning"),
|
||||
before_window=lambda: self._wait_if_paused("captioning"),
|
||||
after_saved=caption_sync.add,
|
||||
)
|
||||
|
||||
if specialist_rows:
|
||||
if specialist_total:
|
||||
_unload_ollama_model(settings.caption_general_model)
|
||||
|
||||
def specialist_progress(update: dict) -> None:
|
||||
@@ -442,21 +469,24 @@ class CaptionJobManager:
|
||||
)
|
||||
self._progress(adjusted)
|
||||
|
||||
specialist_done, specialist_failed = _caption_queue(
|
||||
specialist_done, specialist_failed = _caption_window_queue(
|
||||
"Specialist queue",
|
||||
specialist_rows,
|
||||
windows_for("specialist"),
|
||||
settings.caption_specialist_model,
|
||||
specialist_prompt,
|
||||
progress=specialist_progress,
|
||||
queue_total=specialist_total,
|
||||
completed_offset=general_done,
|
||||
total_frames=frames_total,
|
||||
total_windows=windows_total,
|
||||
frame_source=frame_source,
|
||||
before_frame=lambda: self._wait_if_paused("captioning"),
|
||||
before_window=lambda: self._wait_if_paused("captioning"),
|
||||
after_saved=caption_sync.add,
|
||||
)
|
||||
|
||||
completed = general_done + specialist_done
|
||||
failed = missing + general_failed + specialist_failed
|
||||
completed = general_done + specialist_done
|
||||
failed = missing + general_failed + specialist_failed
|
||||
|
||||
self._wait_if_paused("captioning")
|
||||
|
||||
|
||||
@@ -0,0 +1,505 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from .database import connect
|
||||
from .frame_source import FrameSource
|
||||
|
||||
|
||||
RoutedFrame = tuple[Any, str, float, str]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CaptionWindow:
|
||||
id: str
|
||||
media_id: str
|
||||
title: str
|
||||
start_ms: int
|
||||
end_ms: int
|
||||
representative_row: Any
|
||||
members: tuple[RoutedFrame, ...]
|
||||
sample_rows: tuple[Any, ...]
|
||||
route: str
|
||||
score: float
|
||||
reason: str
|
||||
route_model: str
|
||||
route_policy: str
|
||||
window_policy: str
|
||||
|
||||
@property
|
||||
def member_frame_ids(self) -> list[str]:
|
||||
return [str(item[0]["id"]) for item in self.members]
|
||||
|
||||
|
||||
def caption_window_policy(window_seconds: float, max_frames: int) -> str:
|
||||
return (
|
||||
"contact-sheet-v1|"
|
||||
f"seconds={float(window_seconds):.3f}|"
|
||||
f"max-frames={max(1, int(max_frames))}"
|
||||
)
|
||||
|
||||
|
||||
def _sample_indexes(count: int, max_frames: int) -> list[int]:
|
||||
if count <= 0:
|
||||
return []
|
||||
max_frames = max(1, int(max_frames))
|
||||
if count <= max_frames:
|
||||
return list(range(count))
|
||||
if max_frames == 1:
|
||||
return [count // 2]
|
||||
|
||||
indexes = [
|
||||
int(round(position * (count - 1) / (max_frames - 1)))
|
||||
for position in range(max_frames)
|
||||
]
|
||||
return list(dict.fromkeys(indexes))
|
||||
|
||||
|
||||
def _make_caption_window(
|
||||
media_id: str,
|
||||
bucket: int,
|
||||
members: list[RoutedFrame],
|
||||
*,
|
||||
window_ms: int,
|
||||
max_frames: int,
|
||||
route_model: str,
|
||||
route_policy: str,
|
||||
window_policy: str,
|
||||
) -> CaptionWindow:
|
||||
members.sort(key=lambda item: int(item[0]["timestamp_ms"]))
|
||||
start_ms = bucket * window_ms
|
||||
last_row = members[-1][0]
|
||||
interval_ms = max(1, int(last_row["interval_ms"] or window_ms))
|
||||
end_ms = min(
|
||||
start_ms + window_ms,
|
||||
int(last_row["timestamp_ms"]) + interval_ms,
|
||||
)
|
||||
if end_ms <= start_ms:
|
||||
end_ms = start_ms + window_ms
|
||||
|
||||
specialist_members = [item for item in members if item[1] == "specialist"]
|
||||
route = "specialist" if specialist_members else "general"
|
||||
scored_members = specialist_members or members
|
||||
score = max(float(item[2]) for item in scored_members)
|
||||
reason = (
|
||||
"window-member-specialist"
|
||||
if specialist_members
|
||||
else "window-all-general"
|
||||
)
|
||||
|
||||
sample_indexes = _sample_indexes(len(members), max_frames)
|
||||
sample_rows = tuple(members[index][0] for index in sample_indexes)
|
||||
representative_row = sample_rows[len(sample_rows) // 2]
|
||||
window_id = hashlib.sha256(
|
||||
f"{media_id}:{start_ms}:{end_ms}".encode()
|
||||
).hexdigest()[:40]
|
||||
|
||||
return CaptionWindow(
|
||||
id=window_id,
|
||||
media_id=media_id,
|
||||
title=str(members[0][0]["title"]),
|
||||
start_ms=start_ms,
|
||||
end_ms=end_ms,
|
||||
representative_row=representative_row,
|
||||
members=tuple(members),
|
||||
sample_rows=sample_rows,
|
||||
route=route,
|
||||
score=score,
|
||||
reason=reason,
|
||||
route_model=route_model,
|
||||
route_policy=route_policy,
|
||||
window_policy=window_policy,
|
||||
)
|
||||
|
||||
|
||||
def iter_caption_windows(
|
||||
routed_rows: list[RoutedFrame],
|
||||
*,
|
||||
window_seconds: float,
|
||||
max_frames: int,
|
||||
route_model: str,
|
||||
route_policy: str,
|
||||
):
|
||||
"""Yield windows without materialising the full window set in memory.
|
||||
|
||||
routed_rows must be ordered by media id and timestamp. The caption
|
||||
selection query guarantees that order for production calls.
|
||||
"""
|
||||
if not routed_rows:
|
||||
return
|
||||
|
||||
window_ms = max(1, int(round(float(window_seconds) * 1000)))
|
||||
policy = caption_window_policy(window_seconds, max_frames)
|
||||
current_key: tuple[str, int] | None = None
|
||||
members: list[RoutedFrame] = []
|
||||
|
||||
for item in routed_rows:
|
||||
row = item[0]
|
||||
media_id = str(row["media_id"])
|
||||
bucket = int(row["timestamp_ms"]) // window_ms
|
||||
key = (media_id, bucket)
|
||||
|
||||
if current_key is not None and key != current_key:
|
||||
yield _make_caption_window(
|
||||
current_key[0],
|
||||
current_key[1],
|
||||
members,
|
||||
window_ms=window_ms,
|
||||
max_frames=max_frames,
|
||||
route_model=route_model,
|
||||
route_policy=route_policy,
|
||||
window_policy=policy,
|
||||
)
|
||||
members = []
|
||||
|
||||
current_key = key
|
||||
members.append(item)
|
||||
|
||||
if current_key is not None and members:
|
||||
yield _make_caption_window(
|
||||
current_key[0],
|
||||
current_key[1],
|
||||
members,
|
||||
window_ms=window_ms,
|
||||
max_frames=max_frames,
|
||||
route_model=route_model,
|
||||
route_policy=route_policy,
|
||||
window_policy=policy,
|
||||
)
|
||||
|
||||
|
||||
def caption_window_route_counts(
|
||||
routed_rows: list[RoutedFrame],
|
||||
*,
|
||||
window_seconds: float,
|
||||
max_frames: int,
|
||||
route_model: str,
|
||||
route_policy: str,
|
||||
) -> tuple[int, int, int]:
|
||||
general = 0
|
||||
specialist = 0
|
||||
for window in iter_caption_windows(
|
||||
routed_rows,
|
||||
window_seconds=window_seconds,
|
||||
max_frames=max_frames,
|
||||
route_model=route_model,
|
||||
route_policy=route_policy,
|
||||
):
|
||||
if window.route == "specialist":
|
||||
specialist += 1
|
||||
else:
|
||||
general += 1
|
||||
return general + specialist, general, specialist
|
||||
|
||||
|
||||
def group_caption_windows(
|
||||
routed_rows: list[RoutedFrame],
|
||||
*,
|
||||
window_seconds: float,
|
||||
max_frames: int,
|
||||
route_model: str,
|
||||
route_policy: str,
|
||||
) -> list[CaptionWindow]:
|
||||
"""Materialising helper kept for focused tests and small callers."""
|
||||
return list(
|
||||
iter_caption_windows(
|
||||
routed_rows,
|
||||
window_seconds=window_seconds,
|
||||
max_frames=max_frames,
|
||||
route_model=route_model,
|
||||
route_policy=route_policy,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def build_contact_sheet(
|
||||
source: FrameSource,
|
||||
window: CaptionWindow,
|
||||
*,
|
||||
gap_px: int = 4,
|
||||
max_height: int = 360,
|
||||
) -> Image.Image:
|
||||
images: list[Image.Image] = []
|
||||
try:
|
||||
for row in window.sample_rows:
|
||||
images.append(source.image(row))
|
||||
|
||||
if not images:
|
||||
raise RuntimeError(f"Caption window {window.id} has no source frames")
|
||||
|
||||
target_height = min(max(image.height for image in images), max_height)
|
||||
widths = [
|
||||
max(1, int(round(image.width * target_height / image.height)))
|
||||
for image in images
|
||||
]
|
||||
canvas = Image.new(
|
||||
"RGB",
|
||||
(sum(widths) + gap_px * (len(images) - 1), target_height),
|
||||
(0, 0, 0),
|
||||
)
|
||||
|
||||
left = 0
|
||||
for image, width in zip(images, widths, strict=True):
|
||||
if image.size == (width, target_height):
|
||||
scaled = image
|
||||
close_scaled = False
|
||||
else:
|
||||
scaled = image.resize(
|
||||
(width, target_height),
|
||||
Image.Resampling.LANCZOS,
|
||||
)
|
||||
close_scaled = True
|
||||
try:
|
||||
canvas.paste(scaled, (left, 0))
|
||||
finally:
|
||||
if close_scaled:
|
||||
scaled.close()
|
||||
left += width + gap_px
|
||||
|
||||
return canvas
|
||||
finally:
|
||||
for image in images:
|
||||
image.close()
|
||||
|
||||
|
||||
def save_caption_window(
|
||||
db_path: Path,
|
||||
window: CaptionWindow,
|
||||
*,
|
||||
caption: str,
|
||||
model: str,
|
||||
) -> list[str]:
|
||||
frame_ids = window.member_frame_ids
|
||||
if not frame_ids:
|
||||
raise RuntimeError("Cannot save a caption window without member frames")
|
||||
|
||||
placeholders = ",".join("?" for _ in frame_ids)
|
||||
with connect(db_path) as con:
|
||||
con.execute(
|
||||
"""
|
||||
INSERT INTO caption_windows (
|
||||
id, media_id, start_ms, end_ms, representative_frame_id,
|
||||
caption, caption_model, captioned_at,
|
||||
caption_route, caption_route_reason, route_score,
|
||||
route_model, route_policy, routed_at, window_policy,
|
||||
meili_synced_at, updated_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?, ?, ?, ?, ?,
|
||||
CURRENT_TIMESTAMP, ?, NULL, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
media_id = excluded.media_id,
|
||||
start_ms = excluded.start_ms,
|
||||
end_ms = excluded.end_ms,
|
||||
representative_frame_id = excluded.representative_frame_id,
|
||||
caption = excluded.caption,
|
||||
caption_model = excluded.caption_model,
|
||||
captioned_at = CURRENT_TIMESTAMP,
|
||||
caption_route = excluded.caption_route,
|
||||
caption_route_reason = excluded.caption_route_reason,
|
||||
route_score = excluded.route_score,
|
||||
route_model = excluded.route_model,
|
||||
route_policy = excluded.route_policy,
|
||||
routed_at = CURRENT_TIMESTAMP,
|
||||
window_policy = excluded.window_policy,
|
||||
meili_synced_at = NULL,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(
|
||||
window.id,
|
||||
window.media_id,
|
||||
window.start_ms,
|
||||
window.end_ms,
|
||||
str(window.representative_row["id"]),
|
||||
caption,
|
||||
model,
|
||||
window.route,
|
||||
window.reason,
|
||||
window.score,
|
||||
window.route_model,
|
||||
window.route_policy,
|
||||
window.window_policy,
|
||||
),
|
||||
)
|
||||
|
||||
con.execute(
|
||||
f"DELETE FROM caption_window_frames WHERE frame_id IN ({placeholders})",
|
||||
frame_ids,
|
||||
)
|
||||
con.executemany(
|
||||
"""
|
||||
INSERT INTO caption_window_frames (window_id, frame_id, position)
|
||||
VALUES (?, ?, ?)
|
||||
""",
|
||||
[
|
||||
(window.id, frame_id, position)
|
||||
for position, frame_id in enumerate(frame_ids)
|
||||
],
|
||||
)
|
||||
|
||||
con.execute(
|
||||
f"""
|
||||
UPDATE frames
|
||||
SET caption = NULL,
|
||||
caption_model = NULL,
|
||||
captioned_at = NULL,
|
||||
meili_synced_at = NULL
|
||||
WHERE id IN ({placeholders})
|
||||
""",
|
||||
frame_ids,
|
||||
)
|
||||
|
||||
con.execute(
|
||||
"""
|
||||
DELETE FROM caption_windows
|
||||
WHERE media_id = ?
|
||||
AND id != ?
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM caption_window_frames cwf
|
||||
WHERE cwf.window_id = caption_windows.id
|
||||
)
|
||||
""",
|
||||
(window.media_id, window.id),
|
||||
)
|
||||
|
||||
return frame_ids
|
||||
|
||||
|
||||
def count_valid_caption_windows(
|
||||
db_path: Path,
|
||||
library_ids: list[str],
|
||||
*,
|
||||
window_policy: str,
|
||||
route_policy: str,
|
||||
general_model: str,
|
||||
specialist_model: str,
|
||||
) -> int:
|
||||
if not library_ids:
|
||||
return 0
|
||||
|
||||
placeholders = ",".join("?" for _ in library_ids)
|
||||
with connect(db_path) as con:
|
||||
row = con.execute(
|
||||
f"""
|
||||
SELECT COUNT(*) AS total
|
||||
FROM caption_windows cw
|
||||
JOIN media m ON m.id = cw.media_id
|
||||
WHERE m.source_kind = 'jellyfin'
|
||||
AND m.jellyfin_library_id IN ({placeholders})
|
||||
AND cw.caption IS NOT NULL
|
||||
AND cw.window_policy = ?
|
||||
AND cw.route_policy = ?
|
||||
AND (
|
||||
(cw.caption_route = 'general' AND cw.caption_model = ?)
|
||||
OR (cw.caption_route = 'specialist' AND cw.caption_model = ?)
|
||||
)
|
||||
""",
|
||||
[
|
||||
*library_ids,
|
||||
window_policy,
|
||||
route_policy,
|
||||
general_model,
|
||||
specialist_model,
|
||||
],
|
||||
).fetchone()
|
||||
return int(row["total"] or 0)
|
||||
|
||||
|
||||
def load_media_caption_windows(db_path: Path, media_id: str) -> list[dict[str, Any]]:
|
||||
with connect(db_path) as con:
|
||||
rows = con.execute(
|
||||
"""
|
||||
SELECT *
|
||||
FROM caption_windows
|
||||
WHERE media_id = ?
|
||||
ORDER BY start_ms
|
||||
""",
|
||||
(media_id,),
|
||||
).fetchall()
|
||||
|
||||
snapshots: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
members = con.execute(
|
||||
"""
|
||||
SELECT frame_id, position
|
||||
FROM caption_window_frames
|
||||
WHERE window_id = ?
|
||||
ORDER BY position
|
||||
""",
|
||||
(row["id"],),
|
||||
).fetchall()
|
||||
snapshots.append(
|
||||
{
|
||||
"window": dict(row),
|
||||
"members": [
|
||||
(str(member["frame_id"]), int(member["position"]))
|
||||
for member in members
|
||||
],
|
||||
}
|
||||
)
|
||||
return snapshots
|
||||
|
||||
|
||||
def restore_media_caption_windows(
|
||||
con: Any,
|
||||
snapshots: list[dict[str, Any]],
|
||||
frame_ids: set[str],
|
||||
) -> int:
|
||||
restored = 0
|
||||
for snapshot in snapshots:
|
||||
row = snapshot["window"]
|
||||
representative = str(row["representative_frame_id"])
|
||||
members = [
|
||||
(frame_id, position)
|
||||
for frame_id, position in snapshot["members"]
|
||||
if frame_id in frame_ids
|
||||
]
|
||||
if representative not in frame_ids or not members:
|
||||
continue
|
||||
|
||||
con.execute(
|
||||
"""
|
||||
INSERT INTO caption_windows (
|
||||
id, media_id, start_ms, end_ms, representative_frame_id,
|
||||
caption, caption_model, captioned_at,
|
||||
caption_route, caption_route_reason, route_score,
|
||||
route_model, route_policy, routed_at, window_policy,
|
||||
meili_synced_at, created_at, updated_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?)
|
||||
""",
|
||||
(
|
||||
row["id"],
|
||||
row["media_id"],
|
||||
row["start_ms"],
|
||||
row["end_ms"],
|
||||
representative,
|
||||
row["caption"],
|
||||
row["caption_model"],
|
||||
row["captioned_at"],
|
||||
row["caption_route"],
|
||||
row["caption_route_reason"],
|
||||
row["route_score"],
|
||||
row["route_model"],
|
||||
row["route_policy"],
|
||||
row["routed_at"],
|
||||
row["window_policy"],
|
||||
row["created_at"],
|
||||
row["updated_at"],
|
||||
),
|
||||
)
|
||||
con.executemany(
|
||||
"""
|
||||
INSERT INTO caption_window_frames (window_id, frame_id, position)
|
||||
VALUES (?, ?, ?)
|
||||
""",
|
||||
[(row["id"], frame_id, position) for frame_id, position in members],
|
||||
)
|
||||
restored += 1
|
||||
|
||||
return restored
|
||||
@@ -58,8 +58,23 @@ class OllamaCaptioner:
|
||||
self.max_tokens = max_tokens
|
||||
self.session = requests.Session()
|
||||
|
||||
def _caption_b64(self, image_b64: str, context_hint: str | None = None) -> str:
|
||||
def _caption_b64(
|
||||
self,
|
||||
image_b64: str,
|
||||
context_hint: str | None = None,
|
||||
*,
|
||||
chronological_sequence: bool = False,
|
||||
) -> str:
|
||||
prompt = self.prompt
|
||||
if chronological_sequence:
|
||||
prompt = (
|
||||
f"{prompt}\n\n"
|
||||
"The supplied image is one chronological contact sheet from a short "
|
||||
"video window. Read the panels from left to right as earlier to later "
|
||||
"moments. Describe the scene, people, actions and meaningful changes "
|
||||
"across the sequence as one continuous moment; do not treat the panels "
|
||||
"as simultaneous or unrelated images."
|
||||
)
|
||||
if context_hint:
|
||||
prompt = (
|
||||
f"{prompt}\n\nMetadata context for this video: {context_hint}\n"
|
||||
@@ -105,11 +120,17 @@ class OllamaCaptioner:
|
||||
self,
|
||||
image: Image.Image,
|
||||
context_hint: str | None = None,
|
||||
*,
|
||||
chronological_sequence: bool = False,
|
||||
) -> str:
|
||||
out = io.BytesIO()
|
||||
image.save(out, "JPEG", quality=90)
|
||||
image_b64 = base64.b64encode(out.getvalue()).decode("ascii")
|
||||
return self._caption_b64(image_b64, context_hint=context_hint)
|
||||
return self._caption_b64(
|
||||
image_b64,
|
||||
context_hint=context_hint,
|
||||
chronological_sequence=chronological_sequence,
|
||||
)
|
||||
|
||||
def unload(self) -> None:
|
||||
response = self.session.post(
|
||||
|
||||
+145
-89
@@ -2,7 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Iterable
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
@@ -12,6 +12,13 @@ from .captioner import (
|
||||
ensure_caption_prompt_file,
|
||||
ensure_specialist_prompt_file,
|
||||
)
|
||||
from .caption_windows import (
|
||||
build_contact_sheet,
|
||||
caption_window_policy,
|
||||
caption_window_route_counts,
|
||||
iter_caption_windows,
|
||||
save_caption_window,
|
||||
)
|
||||
from .config import settings
|
||||
from .database import connect, init_db, pack_vector, unpack_vector
|
||||
from .embedder import SiglipEmbedder
|
||||
@@ -188,6 +195,21 @@ def _route_policy() -> str:
|
||||
)
|
||||
|
||||
|
||||
def _caption_window_policy() -> str:
|
||||
return caption_window_policy(
|
||||
settings.caption_window_seconds,
|
||||
settings.caption_window_max_frames,
|
||||
)
|
||||
|
||||
|
||||
def _route_model_name() -> str:
|
||||
if settings.caption_force_specialist:
|
||||
return "forced-specialist"
|
||||
if settings.caption_router_enabled:
|
||||
return settings.caption_router_model
|
||||
return "disabled"
|
||||
|
||||
|
||||
CaptionProgress = Callable[[dict], None]
|
||||
|
||||
|
||||
@@ -200,21 +222,35 @@ def _select_caption_rows(
|
||||
params: list[object] = []
|
||||
|
||||
if only_needing_caption and not getattr(args, "force", False):
|
||||
route_policy = _route_policy()
|
||||
window_policy = _caption_window_policy()
|
||||
conditions.append(
|
||||
"("
|
||||
"f.caption IS NULL OR f.caption_model IS NULL "
|
||||
"OR f.caption_route IS NULL "
|
||||
"f.caption_route IS NULL "
|
||||
"OR f.caption_route NOT IN ('general', 'specialist') "
|
||||
"OR (f.caption_route = 'general' AND f.caption_model != ?) "
|
||||
"OR (f.caption_route = 'specialist' AND f.caption_model != ?) "
|
||||
"OR f.route_policy IS NULL OR f.route_policy != ?"
|
||||
"OR f.route_policy IS NULL OR f.route_policy != ? "
|
||||
"OR NOT EXISTS ("
|
||||
" SELECT 1 "
|
||||
" FROM caption_window_frames cwf "
|
||||
" JOIN caption_windows cw ON cw.id = cwf.window_id "
|
||||
" WHERE cwf.frame_id = f.id "
|
||||
" AND cw.caption IS NOT NULL "
|
||||
" AND cw.window_policy = ? "
|
||||
" AND cw.route_policy = ? "
|
||||
" AND ("
|
||||
" (cw.caption_route = 'general' AND cw.caption_model = ?) "
|
||||
" OR (cw.caption_route = 'specialist' AND cw.caption_model = ?)"
|
||||
" )"
|
||||
")"
|
||||
")"
|
||||
)
|
||||
params.extend(
|
||||
[
|
||||
route_policy,
|
||||
window_policy,
|
||||
route_policy,
|
||||
settings.caption_general_model,
|
||||
settings.caption_specialist_model,
|
||||
_route_policy(),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -244,13 +280,13 @@ def _select_caption_rows(
|
||||
f.thumb_path, f.caption, f.caption_model, f.caption_route,
|
||||
f.caption_route_reason, f.route_score, f.route_model,
|
||||
f.route_policy,
|
||||
m.title, m.source_kind, m.jellyfin_item_id,
|
||||
m.title, m.interval_ms, m.source_kind, m.jellyfin_item_id,
|
||||
m.jellyfin_media_source_id, m.jellyfin_tags,
|
||||
m.trickplay_width
|
||||
FROM frames f
|
||||
JOIN media m ON m.id = f.media_id
|
||||
{where}
|
||||
ORDER BY m.title, f.timestamp_ms
|
||||
ORDER BY m.title, f.media_id, f.timestamp_ms
|
||||
{limit}
|
||||
""",
|
||||
params,
|
||||
@@ -494,23 +530,24 @@ def _unload_ollama_model(model: str) -> None:
|
||||
print(f"WARN could not unload {model}: {exc}")
|
||||
|
||||
|
||||
def _caption_queue(
|
||||
def _caption_window_queue(
|
||||
label: str,
|
||||
routed_rows: list[tuple],
|
||||
windows: Iterable,
|
||||
model: str,
|
||||
prompt: str,
|
||||
*,
|
||||
progress: CaptionProgress | None = None,
|
||||
completed_offset: int = 0,
|
||||
total_frames: int | None = None,
|
||||
queue_total: int,
|
||||
total_windows: int | None = None,
|
||||
frame_source: FrameSource | None = None,
|
||||
before_frame: Callable[[], None] | None = None,
|
||||
before_window: Callable[[], None] | None = None,
|
||||
after_saved: Callable[[str], None] | None = None,
|
||||
) -> tuple[int, int]:
|
||||
if not routed_rows:
|
||||
if queue_total <= 0:
|
||||
return 0, 0
|
||||
|
||||
print(f"\n{label}: {len(routed_rows)} frame(s) with {model}")
|
||||
print(f"\n{label}: {queue_total} window(s) with {model}")
|
||||
captioner = OllamaCaptioner(
|
||||
settings.ollama_url,
|
||||
model,
|
||||
@@ -524,22 +561,28 @@ def _caption_queue(
|
||||
completed = 0
|
||||
failed = 0
|
||||
try:
|
||||
for row, route, score, reason in routed_rows:
|
||||
if before_frame is not None:
|
||||
before_frame()
|
||||
for window in windows:
|
||||
if before_window is not None:
|
||||
before_window()
|
||||
|
||||
row = window.representative_row
|
||||
if progress is not None:
|
||||
progress(
|
||||
{
|
||||
"stage": "captioning",
|
||||
"title": str(row["title"]),
|
||||
"timestamp_ms": int(row["timestamp_ms"]),
|
||||
"title": window.title,
|
||||
"timestamp_ms": window.start_ms,
|
||||
"window_start_ms": window.start_ms,
|
||||
"window_end_ms": window.end_ms,
|
||||
"model": model,
|
||||
"route": route,
|
||||
"frames_total": total_frames or len(routed_rows),
|
||||
"route": window.route,
|
||||
"frames_total": total_windows or queue_total,
|
||||
"frames_completed": completed_offset + completed,
|
||||
"failed": failed,
|
||||
"work_unit": "windows",
|
||||
}
|
||||
)
|
||||
|
||||
try:
|
||||
context_hint = None
|
||||
if settings.jellyfin_tag_context:
|
||||
@@ -551,67 +594,72 @@ def _caption_queue(
|
||||
settings.jellyfin_skip_tag,
|
||||
),
|
||||
)
|
||||
image = source.image(row)
|
||||
sheet = build_contact_sheet(source, window)
|
||||
try:
|
||||
caption = captioner.caption_image(
|
||||
image,
|
||||
sheet,
|
||||
context_hint=context_hint,
|
||||
chronological_sequence=True,
|
||||
)
|
||||
finally:
|
||||
image.close()
|
||||
sheet.close()
|
||||
except Exception as exc:
|
||||
failed += 1
|
||||
if progress is not None:
|
||||
progress(
|
||||
{
|
||||
"stage": "captioning",
|
||||
"title": str(row["title"]),
|
||||
"timestamp_ms": int(row["timestamp_ms"]),
|
||||
"title": window.title,
|
||||
"timestamp_ms": window.start_ms,
|
||||
"window_start_ms": window.start_ms,
|
||||
"window_end_ms": window.end_ms,
|
||||
"model": model,
|
||||
"route": route,
|
||||
"frames_total": total_frames or len(routed_rows),
|
||||
"route": window.route,
|
||||
"frames_total": total_windows or queue_total,
|
||||
"frames_completed": completed_offset + completed,
|
||||
"failed": failed,
|
||||
"work_unit": "windows",
|
||||
}
|
||||
)
|
||||
print(
|
||||
f"FAIL {row['title']} @ "
|
||||
f"{format_ts(int(row['timestamp_ms']))}: {exc}"
|
||||
f"FAIL {window.title} @ "
|
||||
f"{format_ts(window.start_ms)}-{format_ts(window.end_ms)}: {exc}"
|
||||
)
|
||||
continue
|
||||
|
||||
with connect(settings.db_path) as con:
|
||||
con.execute(
|
||||
"""
|
||||
UPDATE frames
|
||||
SET caption = ?, caption_model = ?,
|
||||
captioned_at = CURRENT_TIMESTAMP,
|
||||
meili_synced_at = NULL
|
||||
WHERE id = ?
|
||||
""",
|
||||
(caption, model, row["id"]),
|
||||
)
|
||||
|
||||
frame_ids = save_caption_window(
|
||||
settings.db_path,
|
||||
window,
|
||||
caption=caption,
|
||||
model=model,
|
||||
)
|
||||
completed += 1
|
||||
if after_saved is not None:
|
||||
after_saved(str(row["id"]))
|
||||
for frame_id in frame_ids:
|
||||
after_saved(frame_id)
|
||||
|
||||
if progress is not None:
|
||||
progress(
|
||||
{
|
||||
"stage": "captioning",
|
||||
"title": str(row["title"]),
|
||||
"timestamp_ms": int(row["timestamp_ms"]),
|
||||
"title": window.title,
|
||||
"timestamp_ms": window.start_ms,
|
||||
"window_start_ms": window.start_ms,
|
||||
"window_end_ms": window.end_ms,
|
||||
"model": model,
|
||||
"route": route,
|
||||
"frames_total": total_frames or len(routed_rows),
|
||||
"route": window.route,
|
||||
"frames_total": total_windows or queue_total,
|
||||
"frames_completed": completed_offset + completed,
|
||||
"failed": failed,
|
||||
"work_unit": "windows",
|
||||
}
|
||||
)
|
||||
print(
|
||||
f"{completed:4d}/{len(routed_rows)} {row['title']} @ "
|
||||
f"{format_ts(int(row['timestamp_ms']))} "
|
||||
f"[{route} {score:.3f}/{reason}]: {caption}"
|
||||
f"{completed:4d}/{queue_total} {window.title} @ "
|
||||
f"{format_ts(window.start_ms)}-{format_ts(window.end_ms)} "
|
||||
f"[{window.route} {window.score:.3f}/{window.reason}; "
|
||||
f"{len(window.sample_rows)} sampled/{len(window.members)} frames]: "
|
||||
f"{caption}"
|
||||
)
|
||||
finally:
|
||||
if own_source:
|
||||
@@ -641,46 +689,49 @@ def cmd_caption(args: argparse.Namespace) -> None:
|
||||
rows = _select_caption_rows(args, only_needing_caption=True)
|
||||
|
||||
if not rows:
|
||||
print("No frames need captioning.")
|
||||
print("No caption windows need processing.")
|
||||
return
|
||||
|
||||
routed, missing = _route_rows(rows)
|
||||
del rows
|
||||
if not routed:
|
||||
print(f"No captionable frames ({missing} missing thumbnails).")
|
||||
return
|
||||
|
||||
route_model = _route_model_name()
|
||||
route_policy = _route_policy()
|
||||
total_windows, general_total, specialist_total = caption_window_route_counts(
|
||||
routed,
|
||||
window_seconds=settings.caption_window_seconds,
|
||||
max_frames=settings.caption_window_max_frames,
|
||||
route_model=route_model,
|
||||
route_policy=route_policy,
|
||||
)
|
||||
|
||||
def windows_for(route: str):
|
||||
return (
|
||||
window
|
||||
for window in iter_caption_windows(
|
||||
routed,
|
||||
window_seconds=settings.caption_window_seconds,
|
||||
max_frames=settings.caption_window_max_frames,
|
||||
route_model=route_model,
|
||||
route_policy=route_policy,
|
||||
)
|
||||
if window.route == route
|
||||
)
|
||||
|
||||
print(
|
||||
f"Grouped {len(routed)} routed frame(s) into {total_windows} "
|
||||
f"~{settings.caption_window_seconds:g}s caption window(s), sampling up to "
|
||||
f"{settings.caption_window_max_frames} frame(s) per window."
|
||||
)
|
||||
|
||||
general_prompt = ensure_caption_prompt_file(settings.caption_prompt_file)
|
||||
specialist_prompt = ensure_specialist_prompt_file(
|
||||
settings.caption_specialist_prompt_file
|
||||
)
|
||||
|
||||
general_rows = [
|
||||
item
|
||||
for item in routed
|
||||
if item[1] == "general"
|
||||
and (
|
||||
args.force
|
||||
or item[0]["caption"] is None
|
||||
or item[0]["caption_model"] != settings.caption_general_model
|
||||
)
|
||||
]
|
||||
specialist_rows = [
|
||||
item
|
||||
for item in routed
|
||||
if item[1] == "specialist"
|
||||
and (
|
||||
args.force
|
||||
or item[0]["caption"] is None
|
||||
or item[0]["caption_model"] != settings.caption_specialist_model
|
||||
)
|
||||
]
|
||||
retained = len(routed) - len(general_rows) - len(specialist_rows)
|
||||
if retained:
|
||||
print(
|
||||
f"Keeping {retained} existing caption(s) because they already use "
|
||||
f"the model selected by the router."
|
||||
)
|
||||
|
||||
print(f"General prompt: {settings.caption_prompt_file}")
|
||||
print(f"Specialist prompt: {settings.caption_specialist_prompt_file}")
|
||||
print(f"Ollama: {settings.ollama_url}")
|
||||
@@ -697,38 +748,43 @@ def cmd_caption(args: argparse.Namespace) -> None:
|
||||
batch_size=settings.caption_meili_batch_size,
|
||||
)
|
||||
|
||||
if general_rows:
|
||||
if general_total:
|
||||
_unload_ollama_model(settings.caption_specialist_model)
|
||||
general_done, general_failed = _caption_queue(
|
||||
general_done, general_failed = _caption_window_queue(
|
||||
"General queue",
|
||||
general_rows,
|
||||
windows_for("general"),
|
||||
settings.caption_general_model,
|
||||
general_prompt,
|
||||
total_frames=len(general_rows) + len(specialist_rows),
|
||||
queue_total=general_total,
|
||||
total_windows=total_windows,
|
||||
after_saved=caption_sync.add if caption_sync is not None else None,
|
||||
)
|
||||
|
||||
if specialist_rows:
|
||||
if specialist_total:
|
||||
_unload_ollama_model(settings.caption_general_model)
|
||||
specialist_done, specialist_failed = _caption_queue(
|
||||
specialist_done, specialist_failed = _caption_window_queue(
|
||||
"Specialist queue",
|
||||
specialist_rows,
|
||||
windows_for("specialist"),
|
||||
settings.caption_specialist_model,
|
||||
specialist_prompt,
|
||||
queue_total=specialist_total,
|
||||
completed_offset=general_done,
|
||||
total_frames=len(general_rows) + len(specialist_rows),
|
||||
total_windows=total_windows,
|
||||
after_saved=caption_sync.add if caption_sync is not None else None,
|
||||
)
|
||||
|
||||
completed = general_done + specialist_done
|
||||
failed = general_failed + specialist_failed + missing
|
||||
print(f"\nCaptioning complete: {completed} saved, {failed} failed/skipped.")
|
||||
print(
|
||||
f"\nCaptioning complete: {completed} window(s) saved, "
|
||||
f"{failed} failed/skipped."
|
||||
)
|
||||
|
||||
if args.sync and caption_sync is not None:
|
||||
caption_sync.flush()
|
||||
print(
|
||||
f"Incremental Meilisearch caption updates complete: "
|
||||
f"{caption_sync.updated_total} document(s) queued."
|
||||
f"{caption_sync.updated_total} document(s) updated."
|
||||
)
|
||||
elif completed:
|
||||
print("Next: python -m scene_search.cli sync-meili")
|
||||
|
||||
@@ -59,6 +59,14 @@ class Settings:
|
||||
1,
|
||||
int(os.getenv("CAPTION_MEILI_BATCH_SIZE", "500")),
|
||||
)
|
||||
caption_window_seconds: float = max(
|
||||
1.0,
|
||||
float(os.getenv("CAPTION_WINDOW_SECONDS", "30")),
|
||||
)
|
||||
caption_window_max_frames: int = max(
|
||||
1,
|
||||
int(os.getenv("CAPTION_WINDOW_MAX_FRAMES", "3")),
|
||||
)
|
||||
caption_prompt_file: Path = Path(
|
||||
os.getenv("CAPTION_PROMPT_FILE", "./data/caption_prompt.txt")
|
||||
).expanduser().resolve()
|
||||
|
||||
@@ -57,6 +57,44 @@ CREATE TABLE IF NOT EXISTS frames (
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_frames_media_timestamp
|
||||
ON frames(media_id, timestamp_ms);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS caption_windows (
|
||||
id TEXT PRIMARY KEY,
|
||||
media_id TEXT NOT NULL REFERENCES media(id) ON DELETE CASCADE,
|
||||
start_ms INTEGER NOT NULL,
|
||||
end_ms INTEGER NOT NULL,
|
||||
representative_frame_id TEXT NOT NULL REFERENCES frames(id) ON DELETE CASCADE,
|
||||
caption TEXT,
|
||||
caption_model TEXT,
|
||||
captioned_at TEXT,
|
||||
caption_route TEXT,
|
||||
caption_route_reason TEXT,
|
||||
route_score REAL,
|
||||
route_model TEXT,
|
||||
route_policy TEXT,
|
||||
routed_at TEXT,
|
||||
window_policy TEXT NOT NULL,
|
||||
meili_synced_at TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(media_id, start_ms, end_ms, window_policy)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS caption_window_frames (
|
||||
window_id TEXT NOT NULL REFERENCES caption_windows(id) ON DELETE CASCADE,
|
||||
frame_id TEXT NOT NULL REFERENCES frames(id) ON DELETE CASCADE,
|
||||
position INTEGER NOT NULL,
|
||||
PRIMARY KEY(window_id, frame_id),
|
||||
UNIQUE(frame_id),
|
||||
UNIQUE(window_id, position)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_caption_windows_media_start
|
||||
ON caption_windows(media_id, start_ms);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_caption_windows_unsynced
|
||||
ON caption_windows(meili_synced_at)
|
||||
WHERE caption IS NOT NULL;
|
||||
"""
|
||||
|
||||
|
||||
|
||||
@@ -9,6 +9,10 @@ from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from .caption_windows import (
|
||||
load_media_caption_windows,
|
||||
restore_media_caption_windows,
|
||||
)
|
||||
from .config import settings
|
||||
from .database import connect, init_db, pack_vector
|
||||
from .embedder import SiglipEmbedder
|
||||
@@ -811,6 +815,7 @@ def _index_library(
|
||||
# same source. Frame ids are deterministic by media/timestamp/model,
|
||||
# so only exact surviving frames receive the old annotations.
|
||||
annotations: dict[str, tuple] = {}
|
||||
caption_window_snapshot: list[dict] = []
|
||||
if (
|
||||
existing
|
||||
and str(existing["source_path"] or "") == source_path_text
|
||||
@@ -821,6 +826,10 @@ def _index_library(
|
||||
and str(existing["jellyfin_tags"] or "") == tags_json
|
||||
):
|
||||
annotations = _load_frame_annotations(str(existing["id"]))
|
||||
caption_window_snapshot = load_media_caption_windows(
|
||||
settings.db_path,
|
||||
str(existing["id"]),
|
||||
)
|
||||
|
||||
with connect(settings.db_path) as con:
|
||||
con.execute("DELETE FROM media WHERE id = ?", (media_id,))
|
||||
@@ -867,17 +876,24 @@ def _index_library(
|
||||
""",
|
||||
frame_rows,
|
||||
)
|
||||
surviving_frame_ids = {str(row[0]) for row in frame_rows}
|
||||
preserved = _restore_frame_annotations(
|
||||
con,
|
||||
annotations,
|
||||
{str(row[0]) for row in frame_rows},
|
||||
surviving_frame_ids,
|
||||
)
|
||||
preserved_windows = restore_media_caption_windows(
|
||||
con,
|
||||
caption_window_snapshot,
|
||||
surviving_frame_ids,
|
||||
)
|
||||
|
||||
preserved_note = (
|
||||
f"; preserved {preserved} caption/route annotation(s)"
|
||||
if preserved
|
||||
else ""
|
||||
)
|
||||
preserved_note = ""
|
||||
if preserved or preserved_windows:
|
||||
preserved_note = (
|
||||
f"; preserved {preserved} frame annotation(s), "
|
||||
f"{preserved_windows} caption window(s)"
|
||||
)
|
||||
print(
|
||||
f" stored {len(frame_rows)} frames; tags={_display_tags(tags)}"
|
||||
f"{preserved_note}"
|
||||
@@ -893,6 +909,7 @@ def _index_library(
|
||||
"title": title,
|
||||
"frames": len(frame_rows),
|
||||
"preserved_annotations": preserved,
|
||||
"preserved_caption_windows": preserved_windows,
|
||||
"reason": "Indexed successfully",
|
||||
}
|
||||
)
|
||||
|
||||
@@ -122,6 +122,9 @@ class SearchService:
|
||||
"caption": hit.get("caption"),
|
||||
"caption_model": hit.get("caption_model"),
|
||||
"caption_route": hit.get("caption_route"),
|
||||
"caption_window_id": hit.get("caption_window_id"),
|
||||
"caption_window_start_ms": hit.get("caption_window_start_ms"),
|
||||
"caption_window_end_ms": hit.get("caption_window_end_ms"),
|
||||
"route_score": hit.get("route_score"),
|
||||
"tags": hit.get("jellyfin_tags") or [],
|
||||
"ranking_score": (
|
||||
|
||||
+111
-31
@@ -24,7 +24,7 @@ def sync_frame_metadata(
|
||||
*,
|
||||
meili: Meili | None = None,
|
||||
) -> int:
|
||||
"""Push caption/routing metadata for existing Meili documents only."""
|
||||
"""Push caption/window/routing metadata for existing Meili documents only."""
|
||||
unique_ids = list(dict.fromkeys(str(value) for value in frame_ids if value))
|
||||
if not unique_ids:
|
||||
return 0
|
||||
@@ -34,11 +34,37 @@ def sync_frame_metadata(
|
||||
with connect(settings.db_path) as con:
|
||||
rows = con.execute(
|
||||
f"""
|
||||
SELECT id, caption, caption_model, captioned_at,
|
||||
caption_route, caption_route_reason, route_score,
|
||||
route_model, route_policy, routed_at
|
||||
FROM frames
|
||||
WHERE id IN ({placeholders})
|
||||
SELECT
|
||||
f.id,
|
||||
CASE WHEN cw.representative_frame_id = f.id
|
||||
THEN cw.caption ELSE f.caption END AS served_caption,
|
||||
CASE WHEN cw.representative_frame_id = f.id
|
||||
THEN cw.caption_model ELSE f.caption_model END AS served_caption_model,
|
||||
CASE WHEN cw.representative_frame_id = f.id
|
||||
THEN cw.captioned_at ELSE f.captioned_at END AS served_captioned_at,
|
||||
CASE WHEN cw.representative_frame_id = f.id
|
||||
THEN cw.caption_route ELSE f.caption_route END AS served_caption_route,
|
||||
CASE WHEN cw.representative_frame_id = f.id
|
||||
THEN cw.caption_route_reason ELSE f.caption_route_reason
|
||||
END AS served_caption_route_reason,
|
||||
CASE WHEN cw.representative_frame_id = f.id
|
||||
THEN cw.route_score ELSE f.route_score END AS served_route_score,
|
||||
CASE WHEN cw.representative_frame_id = f.id
|
||||
THEN cw.route_model ELSE f.route_model END AS served_route_model,
|
||||
CASE WHEN cw.representative_frame_id = f.id
|
||||
THEN cw.route_policy ELSE f.route_policy END AS served_route_policy,
|
||||
CASE WHEN cw.representative_frame_id = f.id
|
||||
THEN cw.routed_at ELSE f.routed_at END AS served_routed_at,
|
||||
CASE WHEN cw.representative_frame_id = f.id
|
||||
THEN cw.id ELSE NULL END AS caption_window_id,
|
||||
CASE WHEN cw.representative_frame_id = f.id
|
||||
THEN cw.start_ms ELSE NULL END AS caption_window_start_ms,
|
||||
CASE WHEN cw.representative_frame_id = f.id
|
||||
THEN cw.end_ms ELSE NULL END AS caption_window_end_ms
|
||||
FROM frames f
|
||||
LEFT JOIN caption_window_frames cwf ON cwf.frame_id = f.id
|
||||
LEFT JOIN caption_windows cw ON cw.id = cwf.window_id
|
||||
WHERE f.id IN ({placeholders})
|
||||
""",
|
||||
unique_ids,
|
||||
).fetchall()
|
||||
@@ -49,15 +75,18 @@ def sync_frame_metadata(
|
||||
documents = [
|
||||
{
|
||||
"id": row["id"],
|
||||
"caption": row["caption"],
|
||||
"caption_model": row["caption_model"],
|
||||
"captioned_at": row["captioned_at"],
|
||||
"caption_route": row["caption_route"],
|
||||
"caption_route_reason": row["caption_route_reason"],
|
||||
"route_score": row["route_score"],
|
||||
"route_model": row["route_model"],
|
||||
"route_policy": row["route_policy"],
|
||||
"routed_at": row["routed_at"],
|
||||
"caption": row["served_caption"],
|
||||
"caption_model": row["served_caption_model"],
|
||||
"captioned_at": row["served_captioned_at"],
|
||||
"caption_route": row["served_caption_route"],
|
||||
"caption_route_reason": row["served_caption_route_reason"],
|
||||
"route_score": row["served_route_score"],
|
||||
"route_model": row["served_route_model"],
|
||||
"route_policy": row["served_route_policy"],
|
||||
"routed_at": row["served_routed_at"],
|
||||
"caption_window_id": row["caption_window_id"],
|
||||
"caption_window_start_ms": row["caption_window_start_ms"],
|
||||
"caption_window_end_ms": row["caption_window_end_ms"],
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
@@ -76,6 +105,15 @@ def sync_frame_metadata(
|
||||
""",
|
||||
synced_ids,
|
||||
)
|
||||
con.execute(
|
||||
f"""
|
||||
UPDATE caption_windows
|
||||
SET meili_synced_at = CURRENT_TIMESTAMP
|
||||
WHERE representative_frame_id IN ({placeholders})
|
||||
AND caption IS NOT NULL
|
||||
""",
|
||||
synced_ids,
|
||||
)
|
||||
|
||||
return len(synced_ids)
|
||||
|
||||
@@ -96,11 +134,13 @@ def unsynced_caption_frame_ids(
|
||||
SELECT f.id
|
||||
FROM frames f
|
||||
JOIN media m ON m.id = f.media_id
|
||||
LEFT JOIN caption_window_frames cwf ON cwf.frame_id = f.id
|
||||
LEFT JOIN caption_windows cw ON cw.id = cwf.window_id
|
||||
WHERE m.source_kind = 'jellyfin'
|
||||
AND m.jellyfin_library_id IN ({placeholders})
|
||||
AND f.caption IS NOT NULL
|
||||
AND f.meili_synced_at IS NULL
|
||||
ORDER BY f.captioned_at, f.id
|
||||
AND (f.caption IS NOT NULL OR cw.caption IS NOT NULL)
|
||||
ORDER BY COALESCE(cw.captioned_at, f.captioned_at), f.id
|
||||
LIMIT ?
|
||||
""",
|
||||
[*library_ids, limit],
|
||||
@@ -120,10 +160,12 @@ def count_unsynced_caption_frames(library_ids: list[str]) -> int:
|
||||
SELECT COUNT(*) AS total
|
||||
FROM frames f
|
||||
JOIN media m ON m.id = f.media_id
|
||||
LEFT JOIN caption_window_frames cwf ON cwf.frame_id = f.id
|
||||
LEFT JOIN caption_windows cw ON cw.id = cwf.window_id
|
||||
WHERE m.source_kind = 'jellyfin'
|
||||
AND m.jellyfin_library_id IN ({placeholders})
|
||||
AND f.caption IS NOT NULL
|
||||
AND f.meili_synced_at IS NULL
|
||||
AND (f.caption IS NOT NULL OR cw.caption IS NOT NULL)
|
||||
""",
|
||||
library_ids,
|
||||
).fetchone()
|
||||
@@ -247,16 +289,44 @@ def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
|
||||
|
||||
cursor = con.execute(
|
||||
"""
|
||||
SELECT f.id, f.media_id, f.timestamp_ms, f.thumb_path, f.embedding_f16,
|
||||
f.caption, f.caption_model, f.captioned_at,
|
||||
f.caption_route, f.caption_route_reason, f.route_score,
|
||||
f.route_model, f.route_policy, f.routed_at,
|
||||
SELECT
|
||||
f.id, f.media_id, f.timestamp_ms, f.thumb_path, f.embedding_f16,
|
||||
CASE WHEN cw.representative_frame_id = f.id
|
||||
THEN cw.caption ELSE f.caption END AS served_caption,
|
||||
CASE WHEN cw.representative_frame_id = f.id
|
||||
THEN cw.caption_model ELSE f.caption_model END
|
||||
AS served_caption_model,
|
||||
CASE WHEN cw.representative_frame_id = f.id
|
||||
THEN cw.captioned_at ELSE f.captioned_at END
|
||||
AS served_captioned_at,
|
||||
CASE WHEN cw.representative_frame_id = f.id
|
||||
THEN cw.caption_route ELSE f.caption_route END
|
||||
AS served_caption_route,
|
||||
CASE WHEN cw.representative_frame_id = f.id
|
||||
THEN cw.caption_route_reason ELSE f.caption_route_reason END
|
||||
AS served_caption_route_reason,
|
||||
CASE WHEN cw.representative_frame_id = f.id
|
||||
THEN cw.route_score ELSE f.route_score END AS served_route_score,
|
||||
CASE WHEN cw.representative_frame_id = f.id
|
||||
THEN cw.route_model ELSE f.route_model END AS served_route_model,
|
||||
CASE WHEN cw.representative_frame_id = f.id
|
||||
THEN cw.route_policy ELSE f.route_policy END AS served_route_policy,
|
||||
CASE WHEN cw.representative_frame_id = f.id
|
||||
THEN cw.routed_at ELSE f.routed_at END AS served_routed_at,
|
||||
CASE WHEN cw.representative_frame_id = f.id
|
||||
THEN cw.id ELSE NULL END AS caption_window_id,
|
||||
CASE WHEN cw.representative_frame_id = f.id
|
||||
THEN cw.start_ms ELSE NULL END AS caption_window_start_ms,
|
||||
CASE WHEN cw.representative_frame_id = f.id
|
||||
THEN cw.end_ms ELSE NULL END AS caption_window_end_ms,
|
||||
m.title, m.source_path, m.embedding_dim, m.model_id,
|
||||
m.source_kind, m.jellyfin_item_id, m.jellyfin_media_source_id,
|
||||
m.jellyfin_library_id, m.jellyfin_item_type, m.jellyfin_tags,
|
||||
m.trickplay_width
|
||||
FROM frames f
|
||||
JOIN media m ON m.id = f.media_id
|
||||
LEFT JOIN caption_window_frames cwf ON cwf.frame_id = f.id
|
||||
LEFT JOIN caption_windows cw ON cw.id = cwf.window_id
|
||||
ORDER BY f.id
|
||||
"""
|
||||
)
|
||||
@@ -282,15 +352,18 @@ def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
|
||||
"timestamp_ms": row["timestamp_ms"],
|
||||
"thumb_path": row["thumb_path"],
|
||||
"model_id": row["model_id"],
|
||||
"caption": row["caption"],
|
||||
"caption_model": row["caption_model"],
|
||||
"captioned_at": row["captioned_at"],
|
||||
"caption_route": row["caption_route"],
|
||||
"caption_route_reason": row["caption_route_reason"],
|
||||
"route_score": row["route_score"],
|
||||
"route_model": row["route_model"],
|
||||
"route_policy": row["route_policy"],
|
||||
"routed_at": row["routed_at"],
|
||||
"caption": row["served_caption"],
|
||||
"caption_model": row["served_caption_model"],
|
||||
"captioned_at": row["served_captioned_at"],
|
||||
"caption_route": row["served_caption_route"],
|
||||
"caption_route_reason": row["served_caption_route_reason"],
|
||||
"route_score": row["served_route_score"],
|
||||
"route_model": row["served_route_model"],
|
||||
"route_policy": row["served_route_policy"],
|
||||
"routed_at": row["served_routed_at"],
|
||||
"caption_window_id": row["caption_window_id"],
|
||||
"caption_window_start_ms": row["caption_window_start_ms"],
|
||||
"caption_window_end_ms": row["caption_window_end_ms"],
|
||||
"source_kind": row["source_kind"],
|
||||
"jellyfin_item_id": row["jellyfin_item_id"],
|
||||
"jellyfin_media_source_id": row["jellyfin_media_source_id"],
|
||||
@@ -319,6 +392,13 @@ def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
|
||||
SET meili_synced_at = CURRENT_TIMESTAMP
|
||||
"""
|
||||
)
|
||||
con.execute(
|
||||
"""
|
||||
UPDATE caption_windows
|
||||
SET meili_synced_at = CURRENT_TIMESTAMP
|
||||
WHERE caption IS NOT NULL
|
||||
"""
|
||||
)
|
||||
|
||||
if progress is not None:
|
||||
progress({"stage": "synced", "documents_processed": total})
|
||||
|
||||
@@ -360,7 +360,8 @@
|
||||
const percent = Number(s.percent || (s.stage === 'complete' ? 100 : 0));
|
||||
$('captionProgress').value = percent;
|
||||
$('captionText').textContent = percent.toFixed(1).replace('.0','') + '%';
|
||||
$('captionFrames').textContent = (s.frames_completed || 0) + ' / ' + (s.frames_total || 0) + ' frames';
|
||||
$('captionFrames').textContent = (s.frames_completed || 0) + ' / ' +
|
||||
(s.frames_total || 0) + ' ' + (s.work_unit === 'windows' ? 'windows' : 'frames');
|
||||
|
||||
if (s.running) {
|
||||
const starting = s.stage === 'starting';
|
||||
@@ -390,9 +391,12 @@
|
||||
$('captionStatus').textContent = 'Captioning paused';
|
||||
$('captionDetail').textContent =
|
||||
(s.title ? s.title + (s.timestamp_ms != null ? ' at ' + formatTs(s.timestamp_ms) : '') : '') +
|
||||
(s.frames_completed != null ? ' · ' + s.frames_completed + '/' + (s.frames_total || 0) + ' frames saved' : '');
|
||||
(s.frames_completed != null ? ' · ' + s.frames_completed + '/' +
|
||||
(s.frames_total || 0) + ' ' +
|
||||
(s.work_unit === 'windows' ? 'windows' : 'frames') + ' saved' : '');
|
||||
} else if (s.pause_requested) {
|
||||
$('captionStatus').textContent = 'Pause requested; finishing the current frame';
|
||||
$('captionStatus').textContent = 'Pause requested; finishing the current ' +
|
||||
(s.work_unit === 'windows' ? 'window' : 'frame');
|
||||
$('captionDetail').textContent = s.title || '';
|
||||
} else if (s.stage === 'routing') {
|
||||
$('captionStatus').textContent = 'Routing frames to caption models';
|
||||
|
||||
Reference in New Issue
Block a user