Caption chronological 30-second frame windows

This commit is contained in:
Sleeps
2026-10-07 15:29:12 +01:00
parent a5b657b3e7
commit 5610f3a0d9
18 changed files with 1384 additions and 180 deletions
+67 -37
View File
@@ -8,11 +8,19 @@ from typing import Any
from .captioner import ensure_caption_prompt_file, ensure_specialist_prompt_file
from .cli import (
_caption_queue,
_caption_window_policy,
_caption_window_queue,
_route_model_name,
_route_policy,
_route_rows,
_select_caption_rows,
_unload_ollama_model,
)
from .caption_windows import (
caption_window_route_counts,
count_valid_caption_windows,
iter_caption_windows,
)
from .config import settings
from .database import connect, init_db
from .frame_source import FrameSource
@@ -227,8 +235,7 @@ class CaptionJobManager:
placeholders = ",".join("?" for _ in library_ids)
totals = con.execute(
f"""
SELECT COUNT(*) AS total,
SUM(CASE WHEN f.caption IS NOT NULL THEN 1 ELSE 0 END) AS captioned
SELECT COUNT(*) AS total
FROM frames f
JOIN media m ON m.id = f.media_id
WHERE m.source_kind = 'jellyfin'
@@ -238,7 +245,14 @@ class CaptionJobManager:
).fetchone()
indexed_frames = int(totals["total"] or 0)
already_captioned = int(totals["captioned"] or 0)
already_captioned = count_valid_caption_windows(
settings.db_path,
library_ids,
window_policy=_caption_window_policy(),
route_policy=_route_policy(),
general_model=settings.caption_general_model,
specialist_model=settings.caption_specialist_model,
)
self._progress(
{
@@ -374,42 +388,54 @@ class CaptionJobManager:
settings.caption_specialist_prompt_file
)
general_rows = [
item
for item in routed
if item[1] == "general"
and (
args.force
or item[0]["caption"] is None
or item[0]["caption_model"] != settings.caption_general_model
del rows
route_model = _route_model_name()
route_policy = _route_policy()
windows_total, general_total, specialist_total = (
caption_window_route_counts(
routed,
window_seconds=settings.caption_window_seconds,
max_frames=settings.caption_window_max_frames,
route_model=route_model,
route_policy=route_policy,
)
]
specialist_rows = [
item
for item in routed
if item[1] == "specialist"
and (
args.force
or item[0]["caption"] is None
or item[0]["caption_model"] != settings.caption_specialist_model
)
def windows_for(route: str):
return (
window
for window in iter_caption_windows(
routed,
window_seconds=settings.caption_window_seconds,
max_frames=settings.caption_window_max_frames,
route_model=route_model,
route_policy=route_policy,
)
if window.route == route
)
]
retained = len(routed) - len(general_rows) - len(specialist_rows)
frames_total = len(general_rows) + len(specialist_rows)
retained = (
0
if args.force
else int(self.status().get("already_captioned") or 0)
)
self._progress(
{
"stage": "captioning",
"frames_total": frames_total,
"frames_total": windows_total,
"frames_completed": 0,
"failed": missing,
"retained": retained,
"general_total": len(general_rows),
"specialist_total": len(specialist_rows),
"general_total": general_total,
"specialist_total": specialist_total,
"work_unit": "windows",
"caption_frames_grouped": len(routed),
"caption_window_seconds": settings.caption_window_seconds,
}
)
if general_rows:
if general_total:
_unload_ollama_model(settings.caption_specialist_model)
def general_progress(update: dict) -> None:
@@ -418,19 +444,20 @@ class CaptionJobManager:
self._progress(adjusted)
with FrameSource() as frame_source:
general_done, general_failed = _caption_queue(
general_done, general_failed = _caption_window_queue(
"General queue",
general_rows,
windows_for("general"),
settings.caption_general_model,
general_prompt,
progress=general_progress,
total_frames=frames_total,
queue_total=general_total,
total_windows=windows_total,
frame_source=frame_source,
before_frame=lambda: self._wait_if_paused("captioning"),
before_window=lambda: self._wait_if_paused("captioning"),
after_saved=caption_sync.add,
)
if specialist_rows:
if specialist_total:
_unload_ollama_model(settings.caption_general_model)
def specialist_progress(update: dict) -> None:
@@ -442,21 +469,24 @@ class CaptionJobManager:
)
self._progress(adjusted)
specialist_done, specialist_failed = _caption_queue(
specialist_done, specialist_failed = _caption_window_queue(
"Specialist queue",
specialist_rows,
windows_for("specialist"),
settings.caption_specialist_model,
specialist_prompt,
progress=specialist_progress,
queue_total=specialist_total,
completed_offset=general_done,
total_frames=frames_total,
total_windows=windows_total,
frame_source=frame_source,
before_frame=lambda: self._wait_if_paused("captioning"),
before_window=lambda: self._wait_if_paused("captioning"),
after_saved=caption_sync.add,
)
completed = general_done + specialist_done
failed = missing + general_failed + specialist_failed
completed = general_done + specialist_done
failed = missing + general_failed + specialist_failed
self._wait_if_paused("captioning")
+505
View File
@@ -0,0 +1,505 @@
from __future__ import annotations
import hashlib
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from PIL import Image
from .database import connect
from .frame_source import FrameSource
RoutedFrame = tuple[Any, str, float, str]
@dataclass(frozen=True)
class CaptionWindow:
id: str
media_id: str
title: str
start_ms: int
end_ms: int
representative_row: Any
members: tuple[RoutedFrame, ...]
sample_rows: tuple[Any, ...]
route: str
score: float
reason: str
route_model: str
route_policy: str
window_policy: str
@property
def member_frame_ids(self) -> list[str]:
return [str(item[0]["id"]) for item in self.members]
def caption_window_policy(window_seconds: float, max_frames: int) -> str:
return (
"contact-sheet-v1|"
f"seconds={float(window_seconds):.3f}|"
f"max-frames={max(1, int(max_frames))}"
)
def _sample_indexes(count: int, max_frames: int) -> list[int]:
if count <= 0:
return []
max_frames = max(1, int(max_frames))
if count <= max_frames:
return list(range(count))
if max_frames == 1:
return [count // 2]
indexes = [
int(round(position * (count - 1) / (max_frames - 1)))
for position in range(max_frames)
]
return list(dict.fromkeys(indexes))
def _make_caption_window(
media_id: str,
bucket: int,
members: list[RoutedFrame],
*,
window_ms: int,
max_frames: int,
route_model: str,
route_policy: str,
window_policy: str,
) -> CaptionWindow:
members.sort(key=lambda item: int(item[0]["timestamp_ms"]))
start_ms = bucket * window_ms
last_row = members[-1][0]
interval_ms = max(1, int(last_row["interval_ms"] or window_ms))
end_ms = min(
start_ms + window_ms,
int(last_row["timestamp_ms"]) + interval_ms,
)
if end_ms <= start_ms:
end_ms = start_ms + window_ms
specialist_members = [item for item in members if item[1] == "specialist"]
route = "specialist" if specialist_members else "general"
scored_members = specialist_members or members
score = max(float(item[2]) for item in scored_members)
reason = (
"window-member-specialist"
if specialist_members
else "window-all-general"
)
sample_indexes = _sample_indexes(len(members), max_frames)
sample_rows = tuple(members[index][0] for index in sample_indexes)
representative_row = sample_rows[len(sample_rows) // 2]
window_id = hashlib.sha256(
f"{media_id}:{start_ms}:{end_ms}".encode()
).hexdigest()[:40]
return CaptionWindow(
id=window_id,
media_id=media_id,
title=str(members[0][0]["title"]),
start_ms=start_ms,
end_ms=end_ms,
representative_row=representative_row,
members=tuple(members),
sample_rows=sample_rows,
route=route,
score=score,
reason=reason,
route_model=route_model,
route_policy=route_policy,
window_policy=window_policy,
)
def iter_caption_windows(
routed_rows: list[RoutedFrame],
*,
window_seconds: float,
max_frames: int,
route_model: str,
route_policy: str,
):
"""Yield windows without materialising the full window set in memory.
routed_rows must be ordered by media id and timestamp. The caption
selection query guarantees that order for production calls.
"""
if not routed_rows:
return
window_ms = max(1, int(round(float(window_seconds) * 1000)))
policy = caption_window_policy(window_seconds, max_frames)
current_key: tuple[str, int] | None = None
members: list[RoutedFrame] = []
for item in routed_rows:
row = item[0]
media_id = str(row["media_id"])
bucket = int(row["timestamp_ms"]) // window_ms
key = (media_id, bucket)
if current_key is not None and key != current_key:
yield _make_caption_window(
current_key[0],
current_key[1],
members,
window_ms=window_ms,
max_frames=max_frames,
route_model=route_model,
route_policy=route_policy,
window_policy=policy,
)
members = []
current_key = key
members.append(item)
if current_key is not None and members:
yield _make_caption_window(
current_key[0],
current_key[1],
members,
window_ms=window_ms,
max_frames=max_frames,
route_model=route_model,
route_policy=route_policy,
window_policy=policy,
)
def caption_window_route_counts(
routed_rows: list[RoutedFrame],
*,
window_seconds: float,
max_frames: int,
route_model: str,
route_policy: str,
) -> tuple[int, int, int]:
general = 0
specialist = 0
for window in iter_caption_windows(
routed_rows,
window_seconds=window_seconds,
max_frames=max_frames,
route_model=route_model,
route_policy=route_policy,
):
if window.route == "specialist":
specialist += 1
else:
general += 1
return general + specialist, general, specialist
def group_caption_windows(
routed_rows: list[RoutedFrame],
*,
window_seconds: float,
max_frames: int,
route_model: str,
route_policy: str,
) -> list[CaptionWindow]:
"""Materialising helper kept for focused tests and small callers."""
return list(
iter_caption_windows(
routed_rows,
window_seconds=window_seconds,
max_frames=max_frames,
route_model=route_model,
route_policy=route_policy,
)
)
def build_contact_sheet(
source: FrameSource,
window: CaptionWindow,
*,
gap_px: int = 4,
max_height: int = 360,
) -> Image.Image:
images: list[Image.Image] = []
try:
for row in window.sample_rows:
images.append(source.image(row))
if not images:
raise RuntimeError(f"Caption window {window.id} has no source frames")
target_height = min(max(image.height for image in images), max_height)
widths = [
max(1, int(round(image.width * target_height / image.height)))
for image in images
]
canvas = Image.new(
"RGB",
(sum(widths) + gap_px * (len(images) - 1), target_height),
(0, 0, 0),
)
left = 0
for image, width in zip(images, widths, strict=True):
if image.size == (width, target_height):
scaled = image
close_scaled = False
else:
scaled = image.resize(
(width, target_height),
Image.Resampling.LANCZOS,
)
close_scaled = True
try:
canvas.paste(scaled, (left, 0))
finally:
if close_scaled:
scaled.close()
left += width + gap_px
return canvas
finally:
for image in images:
image.close()
def save_caption_window(
db_path: Path,
window: CaptionWindow,
*,
caption: str,
model: str,
) -> list[str]:
frame_ids = window.member_frame_ids
if not frame_ids:
raise RuntimeError("Cannot save a caption window without member frames")
placeholders = ",".join("?" for _ in frame_ids)
with connect(db_path) as con:
con.execute(
"""
INSERT INTO caption_windows (
id, media_id, start_ms, end_ms, representative_frame_id,
caption, caption_model, captioned_at,
caption_route, caption_route_reason, route_score,
route_model, route_policy, routed_at, window_policy,
meili_synced_at, updated_at
)
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?, ?, ?, ?, ?,
CURRENT_TIMESTAMP, ?, NULL, CURRENT_TIMESTAMP)
ON CONFLICT(id) DO UPDATE SET
media_id = excluded.media_id,
start_ms = excluded.start_ms,
end_ms = excluded.end_ms,
representative_frame_id = excluded.representative_frame_id,
caption = excluded.caption,
caption_model = excluded.caption_model,
captioned_at = CURRENT_TIMESTAMP,
caption_route = excluded.caption_route,
caption_route_reason = excluded.caption_route_reason,
route_score = excluded.route_score,
route_model = excluded.route_model,
route_policy = excluded.route_policy,
routed_at = CURRENT_TIMESTAMP,
window_policy = excluded.window_policy,
meili_synced_at = NULL,
updated_at = CURRENT_TIMESTAMP
""",
(
window.id,
window.media_id,
window.start_ms,
window.end_ms,
str(window.representative_row["id"]),
caption,
model,
window.route,
window.reason,
window.score,
window.route_model,
window.route_policy,
window.window_policy,
),
)
con.execute(
f"DELETE FROM caption_window_frames WHERE frame_id IN ({placeholders})",
frame_ids,
)
con.executemany(
"""
INSERT INTO caption_window_frames (window_id, frame_id, position)
VALUES (?, ?, ?)
""",
[
(window.id, frame_id, position)
for position, frame_id in enumerate(frame_ids)
],
)
con.execute(
f"""
UPDATE frames
SET caption = NULL,
caption_model = NULL,
captioned_at = NULL,
meili_synced_at = NULL
WHERE id IN ({placeholders})
""",
frame_ids,
)
con.execute(
"""
DELETE FROM caption_windows
WHERE media_id = ?
AND id != ?
AND NOT EXISTS (
SELECT 1
FROM caption_window_frames cwf
WHERE cwf.window_id = caption_windows.id
)
""",
(window.media_id, window.id),
)
return frame_ids
def count_valid_caption_windows(
db_path: Path,
library_ids: list[str],
*,
window_policy: str,
route_policy: str,
general_model: str,
specialist_model: str,
) -> int:
if not library_ids:
return 0
placeholders = ",".join("?" for _ in library_ids)
with connect(db_path) as con:
row = con.execute(
f"""
SELECT COUNT(*) AS total
FROM caption_windows cw
JOIN media m ON m.id = cw.media_id
WHERE m.source_kind = 'jellyfin'
AND m.jellyfin_library_id IN ({placeholders})
AND cw.caption IS NOT NULL
AND cw.window_policy = ?
AND cw.route_policy = ?
AND (
(cw.caption_route = 'general' AND cw.caption_model = ?)
OR (cw.caption_route = 'specialist' AND cw.caption_model = ?)
)
""",
[
*library_ids,
window_policy,
route_policy,
general_model,
specialist_model,
],
).fetchone()
return int(row["total"] or 0)
def load_media_caption_windows(db_path: Path, media_id: str) -> list[dict[str, Any]]:
with connect(db_path) as con:
rows = con.execute(
"""
SELECT *
FROM caption_windows
WHERE media_id = ?
ORDER BY start_ms
""",
(media_id,),
).fetchall()
snapshots: list[dict[str, Any]] = []
for row in rows:
members = con.execute(
"""
SELECT frame_id, position
FROM caption_window_frames
WHERE window_id = ?
ORDER BY position
""",
(row["id"],),
).fetchall()
snapshots.append(
{
"window": dict(row),
"members": [
(str(member["frame_id"]), int(member["position"]))
for member in members
],
}
)
return snapshots
def restore_media_caption_windows(
con: Any,
snapshots: list[dict[str, Any]],
frame_ids: set[str],
) -> int:
restored = 0
for snapshot in snapshots:
row = snapshot["window"]
representative = str(row["representative_frame_id"])
members = [
(frame_id, position)
for frame_id, position in snapshot["members"]
if frame_id in frame_ids
]
if representative not in frame_ids or not members:
continue
con.execute(
"""
INSERT INTO caption_windows (
id, media_id, start_ms, end_ms, representative_frame_id,
caption, caption_model, captioned_at,
caption_route, caption_route_reason, route_score,
route_model, route_policy, routed_at, window_policy,
meili_synced_at, created_at, updated_at
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?)
""",
(
row["id"],
row["media_id"],
row["start_ms"],
row["end_ms"],
representative,
row["caption"],
row["caption_model"],
row["captioned_at"],
row["caption_route"],
row["caption_route_reason"],
row["route_score"],
row["route_model"],
row["route_policy"],
row["routed_at"],
row["window_policy"],
row["created_at"],
row["updated_at"],
),
)
con.executemany(
"""
INSERT INTO caption_window_frames (window_id, frame_id, position)
VALUES (?, ?, ?)
""",
[(row["id"], frame_id, position) for frame_id, position in members],
)
restored += 1
return restored
+23 -2
View File
@@ -58,8 +58,23 @@ class OllamaCaptioner:
self.max_tokens = max_tokens
self.session = requests.Session()
def _caption_b64(self, image_b64: str, context_hint: str | None = None) -> str:
def _caption_b64(
self,
image_b64: str,
context_hint: str | None = None,
*,
chronological_sequence: bool = False,
) -> str:
prompt = self.prompt
if chronological_sequence:
prompt = (
f"{prompt}\n\n"
"The supplied image is one chronological contact sheet from a short "
"video window. Read the panels from left to right as earlier to later "
"moments. Describe the scene, people, actions and meaningful changes "
"across the sequence as one continuous moment; do not treat the panels "
"as simultaneous or unrelated images."
)
if context_hint:
prompt = (
f"{prompt}\n\nMetadata context for this video: {context_hint}\n"
@@ -105,11 +120,17 @@ class OllamaCaptioner:
self,
image: Image.Image,
context_hint: str | None = None,
*,
chronological_sequence: bool = False,
) -> str:
out = io.BytesIO()
image.save(out, "JPEG", quality=90)
image_b64 = base64.b64encode(out.getvalue()).decode("ascii")
return self._caption_b64(image_b64, context_hint=context_hint)
return self._caption_b64(
image_b64,
context_hint=context_hint,
chronological_sequence=chronological_sequence,
)
def unload(self) -> None:
response = self.session.post(
+145 -89
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
import argparse
import hashlib
from collections.abc import Callable
from collections.abc import Callable, Iterable
from pathlib import Path
from PIL import Image
@@ -12,6 +12,13 @@ from .captioner import (
ensure_caption_prompt_file,
ensure_specialist_prompt_file,
)
from .caption_windows import (
build_contact_sheet,
caption_window_policy,
caption_window_route_counts,
iter_caption_windows,
save_caption_window,
)
from .config import settings
from .database import connect, init_db, pack_vector, unpack_vector
from .embedder import SiglipEmbedder
@@ -188,6 +195,21 @@ def _route_policy() -> str:
)
def _caption_window_policy() -> str:
return caption_window_policy(
settings.caption_window_seconds,
settings.caption_window_max_frames,
)
def _route_model_name() -> str:
if settings.caption_force_specialist:
return "forced-specialist"
if settings.caption_router_enabled:
return settings.caption_router_model
return "disabled"
CaptionProgress = Callable[[dict], None]
@@ -200,21 +222,35 @@ def _select_caption_rows(
params: list[object] = []
if only_needing_caption and not getattr(args, "force", False):
route_policy = _route_policy()
window_policy = _caption_window_policy()
conditions.append(
"("
"f.caption IS NULL OR f.caption_model IS NULL "
"OR f.caption_route IS NULL "
"f.caption_route IS NULL "
"OR f.caption_route NOT IN ('general', 'specialist') "
"OR (f.caption_route = 'general' AND f.caption_model != ?) "
"OR (f.caption_route = 'specialist' AND f.caption_model != ?) "
"OR f.route_policy IS NULL OR f.route_policy != ?"
"OR f.route_policy IS NULL OR f.route_policy != ? "
"OR NOT EXISTS ("
" SELECT 1 "
" FROM caption_window_frames cwf "
" JOIN caption_windows cw ON cw.id = cwf.window_id "
" WHERE cwf.frame_id = f.id "
" AND cw.caption IS NOT NULL "
" AND cw.window_policy = ? "
" AND cw.route_policy = ? "
" AND ("
" (cw.caption_route = 'general' AND cw.caption_model = ?) "
" OR (cw.caption_route = 'specialist' AND cw.caption_model = ?)"
" )"
")"
")"
)
params.extend(
[
route_policy,
window_policy,
route_policy,
settings.caption_general_model,
settings.caption_specialist_model,
_route_policy(),
]
)
@@ -244,13 +280,13 @@ def _select_caption_rows(
f.thumb_path, f.caption, f.caption_model, f.caption_route,
f.caption_route_reason, f.route_score, f.route_model,
f.route_policy,
m.title, m.source_kind, m.jellyfin_item_id,
m.title, m.interval_ms, m.source_kind, m.jellyfin_item_id,
m.jellyfin_media_source_id, m.jellyfin_tags,
m.trickplay_width
FROM frames f
JOIN media m ON m.id = f.media_id
{where}
ORDER BY m.title, f.timestamp_ms
ORDER BY m.title, f.media_id, f.timestamp_ms
{limit}
""",
params,
@@ -494,23 +530,24 @@ def _unload_ollama_model(model: str) -> None:
print(f"WARN could not unload {model}: {exc}")
def _caption_queue(
def _caption_window_queue(
label: str,
routed_rows: list[tuple],
windows: Iterable,
model: str,
prompt: str,
*,
progress: CaptionProgress | None = None,
completed_offset: int = 0,
total_frames: int | None = None,
queue_total: int,
total_windows: int | None = None,
frame_source: FrameSource | None = None,
before_frame: Callable[[], None] | None = None,
before_window: Callable[[], None] | None = None,
after_saved: Callable[[str], None] | None = None,
) -> tuple[int, int]:
if not routed_rows:
if queue_total <= 0:
return 0, 0
print(f"\n{label}: {len(routed_rows)} frame(s) with {model}")
print(f"\n{label}: {queue_total} window(s) with {model}")
captioner = OllamaCaptioner(
settings.ollama_url,
model,
@@ -524,22 +561,28 @@ def _caption_queue(
completed = 0
failed = 0
try:
for row, route, score, reason in routed_rows:
if before_frame is not None:
before_frame()
for window in windows:
if before_window is not None:
before_window()
row = window.representative_row
if progress is not None:
progress(
{
"stage": "captioning",
"title": str(row["title"]),
"timestamp_ms": int(row["timestamp_ms"]),
"title": window.title,
"timestamp_ms": window.start_ms,
"window_start_ms": window.start_ms,
"window_end_ms": window.end_ms,
"model": model,
"route": route,
"frames_total": total_frames or len(routed_rows),
"route": window.route,
"frames_total": total_windows or queue_total,
"frames_completed": completed_offset + completed,
"failed": failed,
"work_unit": "windows",
}
)
try:
context_hint = None
if settings.jellyfin_tag_context:
@@ -551,67 +594,72 @@ def _caption_queue(
settings.jellyfin_skip_tag,
),
)
image = source.image(row)
sheet = build_contact_sheet(source, window)
try:
caption = captioner.caption_image(
image,
sheet,
context_hint=context_hint,
chronological_sequence=True,
)
finally:
image.close()
sheet.close()
except Exception as exc:
failed += 1
if progress is not None:
progress(
{
"stage": "captioning",
"title": str(row["title"]),
"timestamp_ms": int(row["timestamp_ms"]),
"title": window.title,
"timestamp_ms": window.start_ms,
"window_start_ms": window.start_ms,
"window_end_ms": window.end_ms,
"model": model,
"route": route,
"frames_total": total_frames or len(routed_rows),
"route": window.route,
"frames_total": total_windows or queue_total,
"frames_completed": completed_offset + completed,
"failed": failed,
"work_unit": "windows",
}
)
print(
f"FAIL {row['title']} @ "
f"{format_ts(int(row['timestamp_ms']))}: {exc}"
f"FAIL {window.title} @ "
f"{format_ts(window.start_ms)}-{format_ts(window.end_ms)}: {exc}"
)
continue
with connect(settings.db_path) as con:
con.execute(
"""
UPDATE frames
SET caption = ?, caption_model = ?,
captioned_at = CURRENT_TIMESTAMP,
meili_synced_at = NULL
WHERE id = ?
""",
(caption, model, row["id"]),
)
frame_ids = save_caption_window(
settings.db_path,
window,
caption=caption,
model=model,
)
completed += 1
if after_saved is not None:
after_saved(str(row["id"]))
for frame_id in frame_ids:
after_saved(frame_id)
if progress is not None:
progress(
{
"stage": "captioning",
"title": str(row["title"]),
"timestamp_ms": int(row["timestamp_ms"]),
"title": window.title,
"timestamp_ms": window.start_ms,
"window_start_ms": window.start_ms,
"window_end_ms": window.end_ms,
"model": model,
"route": route,
"frames_total": total_frames or len(routed_rows),
"route": window.route,
"frames_total": total_windows or queue_total,
"frames_completed": completed_offset + completed,
"failed": failed,
"work_unit": "windows",
}
)
print(
f"{completed:4d}/{len(routed_rows)} {row['title']} @ "
f"{format_ts(int(row['timestamp_ms']))} "
f"[{route} {score:.3f}/{reason}]: {caption}"
f"{completed:4d}/{queue_total} {window.title} @ "
f"{format_ts(window.start_ms)}-{format_ts(window.end_ms)} "
f"[{window.route} {window.score:.3f}/{window.reason}; "
f"{len(window.sample_rows)} sampled/{len(window.members)} frames]: "
f"{caption}"
)
finally:
if own_source:
@@ -641,46 +689,49 @@ def cmd_caption(args: argparse.Namespace) -> None:
rows = _select_caption_rows(args, only_needing_caption=True)
if not rows:
print("No frames need captioning.")
print("No caption windows need processing.")
return
routed, missing = _route_rows(rows)
del rows
if not routed:
print(f"No captionable frames ({missing} missing thumbnails).")
return
route_model = _route_model_name()
route_policy = _route_policy()
total_windows, general_total, specialist_total = caption_window_route_counts(
routed,
window_seconds=settings.caption_window_seconds,
max_frames=settings.caption_window_max_frames,
route_model=route_model,
route_policy=route_policy,
)
def windows_for(route: str):
return (
window
for window in iter_caption_windows(
routed,
window_seconds=settings.caption_window_seconds,
max_frames=settings.caption_window_max_frames,
route_model=route_model,
route_policy=route_policy,
)
if window.route == route
)
print(
f"Grouped {len(routed)} routed frame(s) into {total_windows} "
f"~{settings.caption_window_seconds:g}s caption window(s), sampling up to "
f"{settings.caption_window_max_frames} frame(s) per window."
)
general_prompt = ensure_caption_prompt_file(settings.caption_prompt_file)
specialist_prompt = ensure_specialist_prompt_file(
settings.caption_specialist_prompt_file
)
general_rows = [
item
for item in routed
if item[1] == "general"
and (
args.force
or item[0]["caption"] is None
or item[0]["caption_model"] != settings.caption_general_model
)
]
specialist_rows = [
item
for item in routed
if item[1] == "specialist"
and (
args.force
or item[0]["caption"] is None
or item[0]["caption_model"] != settings.caption_specialist_model
)
]
retained = len(routed) - len(general_rows) - len(specialist_rows)
if retained:
print(
f"Keeping {retained} existing caption(s) because they already use "
f"the model selected by the router."
)
print(f"General prompt: {settings.caption_prompt_file}")
print(f"Specialist prompt: {settings.caption_specialist_prompt_file}")
print(f"Ollama: {settings.ollama_url}")
@@ -697,38 +748,43 @@ def cmd_caption(args: argparse.Namespace) -> None:
batch_size=settings.caption_meili_batch_size,
)
if general_rows:
if general_total:
_unload_ollama_model(settings.caption_specialist_model)
general_done, general_failed = _caption_queue(
general_done, general_failed = _caption_window_queue(
"General queue",
general_rows,
windows_for("general"),
settings.caption_general_model,
general_prompt,
total_frames=len(general_rows) + len(specialist_rows),
queue_total=general_total,
total_windows=total_windows,
after_saved=caption_sync.add if caption_sync is not None else None,
)
if specialist_rows:
if specialist_total:
_unload_ollama_model(settings.caption_general_model)
specialist_done, specialist_failed = _caption_queue(
specialist_done, specialist_failed = _caption_window_queue(
"Specialist queue",
specialist_rows,
windows_for("specialist"),
settings.caption_specialist_model,
specialist_prompt,
queue_total=specialist_total,
completed_offset=general_done,
total_frames=len(general_rows) + len(specialist_rows),
total_windows=total_windows,
after_saved=caption_sync.add if caption_sync is not None else None,
)
completed = general_done + specialist_done
failed = general_failed + specialist_failed + missing
print(f"\nCaptioning complete: {completed} saved, {failed} failed/skipped.")
print(
f"\nCaptioning complete: {completed} window(s) saved, "
f"{failed} failed/skipped."
)
if args.sync and caption_sync is not None:
caption_sync.flush()
print(
f"Incremental Meilisearch caption updates complete: "
f"{caption_sync.updated_total} document(s) queued."
f"{caption_sync.updated_total} document(s) updated."
)
elif completed:
print("Next: python -m scene_search.cli sync-meili")
+8
View File
@@ -59,6 +59,14 @@ class Settings:
1,
int(os.getenv("CAPTION_MEILI_BATCH_SIZE", "500")),
)
caption_window_seconds: float = max(
1.0,
float(os.getenv("CAPTION_WINDOW_SECONDS", "30")),
)
caption_window_max_frames: int = max(
1,
int(os.getenv("CAPTION_WINDOW_MAX_FRAMES", "3")),
)
caption_prompt_file: Path = Path(
os.getenv("CAPTION_PROMPT_FILE", "./data/caption_prompt.txt")
).expanduser().resolve()
+38
View File
@@ -57,6 +57,44 @@ CREATE TABLE IF NOT EXISTS frames (
CREATE INDEX IF NOT EXISTS idx_frames_media_timestamp
ON frames(media_id, timestamp_ms);
CREATE TABLE IF NOT EXISTS caption_windows (
id TEXT PRIMARY KEY,
media_id TEXT NOT NULL REFERENCES media(id) ON DELETE CASCADE,
start_ms INTEGER NOT NULL,
end_ms INTEGER NOT NULL,
representative_frame_id TEXT NOT NULL REFERENCES frames(id) ON DELETE CASCADE,
caption TEXT,
caption_model TEXT,
captioned_at TEXT,
caption_route TEXT,
caption_route_reason TEXT,
route_score REAL,
route_model TEXT,
route_policy TEXT,
routed_at TEXT,
window_policy TEXT NOT NULL,
meili_synced_at TEXT,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
UNIQUE(media_id, start_ms, end_ms, window_policy)
);
CREATE TABLE IF NOT EXISTS caption_window_frames (
window_id TEXT NOT NULL REFERENCES caption_windows(id) ON DELETE CASCADE,
frame_id TEXT NOT NULL REFERENCES frames(id) ON DELETE CASCADE,
position INTEGER NOT NULL,
PRIMARY KEY(window_id, frame_id),
UNIQUE(frame_id),
UNIQUE(window_id, position)
);
CREATE INDEX IF NOT EXISTS idx_caption_windows_media_start
ON caption_windows(media_id, start_ms);
CREATE INDEX IF NOT EXISTS idx_caption_windows_unsynced
ON caption_windows(meili_synced_at)
WHERE caption IS NOT NULL;
"""
+23 -6
View File
@@ -9,6 +9,10 @@ from pathlib import Path
from PIL import Image
from .caption_windows import (
load_media_caption_windows,
restore_media_caption_windows,
)
from .config import settings
from .database import connect, init_db, pack_vector
from .embedder import SiglipEmbedder
@@ -811,6 +815,7 @@ def _index_library(
# same source. Frame ids are deterministic by media/timestamp/model,
# so only exact surviving frames receive the old annotations.
annotations: dict[str, tuple] = {}
caption_window_snapshot: list[dict] = []
if (
existing
and str(existing["source_path"] or "") == source_path_text
@@ -821,6 +826,10 @@ def _index_library(
and str(existing["jellyfin_tags"] or "") == tags_json
):
annotations = _load_frame_annotations(str(existing["id"]))
caption_window_snapshot = load_media_caption_windows(
settings.db_path,
str(existing["id"]),
)
with connect(settings.db_path) as con:
con.execute("DELETE FROM media WHERE id = ?", (media_id,))
@@ -867,17 +876,24 @@ def _index_library(
""",
frame_rows,
)
surviving_frame_ids = {str(row[0]) for row in frame_rows}
preserved = _restore_frame_annotations(
con,
annotations,
{str(row[0]) for row in frame_rows},
surviving_frame_ids,
)
preserved_windows = restore_media_caption_windows(
con,
caption_window_snapshot,
surviving_frame_ids,
)
preserved_note = (
f"; preserved {preserved} caption/route annotation(s)"
if preserved
else ""
)
preserved_note = ""
if preserved or preserved_windows:
preserved_note = (
f"; preserved {preserved} frame annotation(s), "
f"{preserved_windows} caption window(s)"
)
print(
f" stored {len(frame_rows)} frames; tags={_display_tags(tags)}"
f"{preserved_note}"
@@ -893,6 +909,7 @@ def _index_library(
"title": title,
"frames": len(frame_rows),
"preserved_annotations": preserved,
"preserved_caption_windows": preserved_windows,
"reason": "Indexed successfully",
}
)
+3
View File
@@ -122,6 +122,9 @@ class SearchService:
"caption": hit.get("caption"),
"caption_model": hit.get("caption_model"),
"caption_route": hit.get("caption_route"),
"caption_window_id": hit.get("caption_window_id"),
"caption_window_start_ms": hit.get("caption_window_start_ms"),
"caption_window_end_ms": hit.get("caption_window_end_ms"),
"route_score": hit.get("route_score"),
"tags": hit.get("jellyfin_tags") or [],
"ranking_score": (
+111 -31
View File
@@ -24,7 +24,7 @@ def sync_frame_metadata(
*,
meili: Meili | None = None,
) -> int:
"""Push caption/routing metadata for existing Meili documents only."""
"""Push caption/window/routing metadata for existing Meili documents only."""
unique_ids = list(dict.fromkeys(str(value) for value in frame_ids if value))
if not unique_ids:
return 0
@@ -34,11 +34,37 @@ def sync_frame_metadata(
with connect(settings.db_path) as con:
rows = con.execute(
f"""
SELECT id, caption, caption_model, captioned_at,
caption_route, caption_route_reason, route_score,
route_model, route_policy, routed_at
FROM frames
WHERE id IN ({placeholders})
SELECT
f.id,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.caption ELSE f.caption END AS served_caption,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.caption_model ELSE f.caption_model END AS served_caption_model,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.captioned_at ELSE f.captioned_at END AS served_captioned_at,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.caption_route ELSE f.caption_route END AS served_caption_route,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.caption_route_reason ELSE f.caption_route_reason
END AS served_caption_route_reason,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.route_score ELSE f.route_score END AS served_route_score,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.route_model ELSE f.route_model END AS served_route_model,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.route_policy ELSE f.route_policy END AS served_route_policy,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.routed_at ELSE f.routed_at END AS served_routed_at,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.id ELSE NULL END AS caption_window_id,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.start_ms ELSE NULL END AS caption_window_start_ms,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.end_ms ELSE NULL END AS caption_window_end_ms
FROM frames f
LEFT JOIN caption_window_frames cwf ON cwf.frame_id = f.id
LEFT JOIN caption_windows cw ON cw.id = cwf.window_id
WHERE f.id IN ({placeholders})
""",
unique_ids,
).fetchall()
@@ -49,15 +75,18 @@ def sync_frame_metadata(
documents = [
{
"id": row["id"],
"caption": row["caption"],
"caption_model": row["caption_model"],
"captioned_at": row["captioned_at"],
"caption_route": row["caption_route"],
"caption_route_reason": row["caption_route_reason"],
"route_score": row["route_score"],
"route_model": row["route_model"],
"route_policy": row["route_policy"],
"routed_at": row["routed_at"],
"caption": row["served_caption"],
"caption_model": row["served_caption_model"],
"captioned_at": row["served_captioned_at"],
"caption_route": row["served_caption_route"],
"caption_route_reason": row["served_caption_route_reason"],
"route_score": row["served_route_score"],
"route_model": row["served_route_model"],
"route_policy": row["served_route_policy"],
"routed_at": row["served_routed_at"],
"caption_window_id": row["caption_window_id"],
"caption_window_start_ms": row["caption_window_start_ms"],
"caption_window_end_ms": row["caption_window_end_ms"],
}
for row in rows
]
@@ -76,6 +105,15 @@ def sync_frame_metadata(
""",
synced_ids,
)
con.execute(
f"""
UPDATE caption_windows
SET meili_synced_at = CURRENT_TIMESTAMP
WHERE representative_frame_id IN ({placeholders})
AND caption IS NOT NULL
""",
synced_ids,
)
return len(synced_ids)
@@ -96,11 +134,13 @@ def unsynced_caption_frame_ids(
SELECT f.id
FROM frames f
JOIN media m ON m.id = f.media_id
LEFT JOIN caption_window_frames cwf ON cwf.frame_id = f.id
LEFT JOIN caption_windows cw ON cw.id = cwf.window_id
WHERE m.source_kind = 'jellyfin'
AND m.jellyfin_library_id IN ({placeholders})
AND f.caption IS NOT NULL
AND f.meili_synced_at IS NULL
ORDER BY f.captioned_at, f.id
AND (f.caption IS NOT NULL OR cw.caption IS NOT NULL)
ORDER BY COALESCE(cw.captioned_at, f.captioned_at), f.id
LIMIT ?
""",
[*library_ids, limit],
@@ -120,10 +160,12 @@ def count_unsynced_caption_frames(library_ids: list[str]) -> int:
SELECT COUNT(*) AS total
FROM frames f
JOIN media m ON m.id = f.media_id
LEFT JOIN caption_window_frames cwf ON cwf.frame_id = f.id
LEFT JOIN caption_windows cw ON cw.id = cwf.window_id
WHERE m.source_kind = 'jellyfin'
AND m.jellyfin_library_id IN ({placeholders})
AND f.caption IS NOT NULL
AND f.meili_synced_at IS NULL
AND (f.caption IS NOT NULL OR cw.caption IS NOT NULL)
""",
library_ids,
).fetchone()
@@ -247,16 +289,44 @@ def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
cursor = con.execute(
"""
SELECT f.id, f.media_id, f.timestamp_ms, f.thumb_path, f.embedding_f16,
f.caption, f.caption_model, f.captioned_at,
f.caption_route, f.caption_route_reason, f.route_score,
f.route_model, f.route_policy, f.routed_at,
SELECT
f.id, f.media_id, f.timestamp_ms, f.thumb_path, f.embedding_f16,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.caption ELSE f.caption END AS served_caption,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.caption_model ELSE f.caption_model END
AS served_caption_model,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.captioned_at ELSE f.captioned_at END
AS served_captioned_at,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.caption_route ELSE f.caption_route END
AS served_caption_route,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.caption_route_reason ELSE f.caption_route_reason END
AS served_caption_route_reason,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.route_score ELSE f.route_score END AS served_route_score,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.route_model ELSE f.route_model END AS served_route_model,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.route_policy ELSE f.route_policy END AS served_route_policy,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.routed_at ELSE f.routed_at END AS served_routed_at,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.id ELSE NULL END AS caption_window_id,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.start_ms ELSE NULL END AS caption_window_start_ms,
CASE WHEN cw.representative_frame_id = f.id
THEN cw.end_ms ELSE NULL END AS caption_window_end_ms,
m.title, m.source_path, m.embedding_dim, m.model_id,
m.source_kind, m.jellyfin_item_id, m.jellyfin_media_source_id,
m.jellyfin_library_id, m.jellyfin_item_type, m.jellyfin_tags,
m.trickplay_width
FROM frames f
JOIN media m ON m.id = f.media_id
LEFT JOIN caption_window_frames cwf ON cwf.frame_id = f.id
LEFT JOIN caption_windows cw ON cw.id = cwf.window_id
ORDER BY f.id
"""
)
@@ -282,15 +352,18 @@ def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
"timestamp_ms": row["timestamp_ms"],
"thumb_path": row["thumb_path"],
"model_id": row["model_id"],
"caption": row["caption"],
"caption_model": row["caption_model"],
"captioned_at": row["captioned_at"],
"caption_route": row["caption_route"],
"caption_route_reason": row["caption_route_reason"],
"route_score": row["route_score"],
"route_model": row["route_model"],
"route_policy": row["route_policy"],
"routed_at": row["routed_at"],
"caption": row["served_caption"],
"caption_model": row["served_caption_model"],
"captioned_at": row["served_captioned_at"],
"caption_route": row["served_caption_route"],
"caption_route_reason": row["served_caption_route_reason"],
"route_score": row["served_route_score"],
"route_model": row["served_route_model"],
"route_policy": row["served_route_policy"],
"routed_at": row["served_routed_at"],
"caption_window_id": row["caption_window_id"],
"caption_window_start_ms": row["caption_window_start_ms"],
"caption_window_end_ms": row["caption_window_end_ms"],
"source_kind": row["source_kind"],
"jellyfin_item_id": row["jellyfin_item_id"],
"jellyfin_media_source_id": row["jellyfin_media_source_id"],
@@ -319,6 +392,13 @@ def rebuild_meili(*, progress: SyncProgress | None = None) -> int:
SET meili_synced_at = CURRENT_TIMESTAMP
"""
)
con.execute(
"""
UPDATE caption_windows
SET meili_synced_at = CURRENT_TIMESTAMP
WHERE caption IS NOT NULL
"""
)
if progress is not None:
progress({"stage": "synced", "documents_processed": total})
+7 -3
View File
@@ -360,7 +360,8 @@
const percent = Number(s.percent || (s.stage === 'complete' ? 100 : 0));
$('captionProgress').value = percent;
$('captionText').textContent = percent.toFixed(1).replace('.0','') + '%';
$('captionFrames').textContent = (s.frames_completed || 0) + ' / ' + (s.frames_total || 0) + ' frames';
$('captionFrames').textContent = (s.frames_completed || 0) + ' / ' +
(s.frames_total || 0) + ' ' + (s.work_unit === 'windows' ? 'windows' : 'frames');
if (s.running) {
const starting = s.stage === 'starting';
@@ -390,9 +391,12 @@
$('captionStatus').textContent = 'Captioning paused';
$('captionDetail').textContent =
(s.title ? s.title + (s.timestamp_ms != null ? ' at ' + formatTs(s.timestamp_ms) : '') : '') +
(s.frames_completed != null ? ' · ' + s.frames_completed + '/' + (s.frames_total || 0) + ' frames saved' : '');
(s.frames_completed != null ? ' · ' + s.frames_completed + '/' +
(s.frames_total || 0) + ' ' +
(s.work_unit === 'windows' ? 'windows' : 'frames') + ' saved' : '');
} else if (s.pause_requested) {
$('captionStatus').textContent = 'Pause requested; finishing the current frame';
$('captionStatus').textContent = 'Pause requested; finishing the current ' +
(s.work_unit === 'windows' ? 'window' : 'frame');
$('captionDetail').textContent = s.title || '';
} else if (s.stage === 'routing') {
$('captionStatus').textContent = 'Routing frames to caption models';