Caption chronological 30-second frame windows

This commit is contained in:
Sleeps
2026-10-07 15:29:12 +01:00
parent a5b657b3e7
commit 5610f3a0d9
18 changed files with 1384 additions and 180 deletions
+78 -1
View File
@@ -6,7 +6,11 @@ from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
from scene_search.cli import _route_policy, _select_caption_rows
from scene_search.cli import (
_caption_window_policy,
_route_policy,
_select_caption_rows,
)
from scene_search.database import connect, init_db
@@ -22,6 +26,8 @@ class CaptionSelectionTests(unittest.TestCase):
caption_specialist_model="specialist-model",
caption_force_specialist=False,
caption_router_enabled=False,
caption_window_seconds=30.0,
caption_window_max_frames=3,
jellyfin_force_specialist_tag="scene-search:specialist",
jellyfin_force_general_tag="scene-search:general",
jellyfin_specialist_tags=(),
@@ -82,6 +88,41 @@ class CaptionSelectionTests(unittest.TestCase):
policy,
),
)
con.execute(
"""
INSERT INTO caption_windows (
id, media_id, start_ms, end_ms, representative_frame_id,
caption, caption_model, captioned_at, caption_route,
caption_route_reason, route_score, route_model,
route_policy, routed_at, window_policy
)
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?, ?, ?, ?,
?, CURRENT_TIMESTAMP, ?)
""",
(
"window-done",
"media-a",
0,
10000,
"done",
"already captioned",
"general-model",
"general",
"window-all-general",
0.0,
"disabled",
policy,
_caption_window_policy(),
),
)
con.execute(
"""
INSERT INTO caption_window_frames (window_id, frame_id, position)
VALUES (?, ?, ?)
""",
("window-done", "done", 0),
)
con.execute(
"""
INSERT INTO frames (
@@ -139,6 +180,8 @@ class CaptionSelectionTests(unittest.TestCase):
caption_specialist_model="specialist-model",
caption_force_specialist=False,
caption_router_enabled=True,
caption_window_seconds=30.0,
caption_window_max_frames=3,
caption_router_model="router-model",
caption_router_threshold=0.5,
caption_router_context_seconds=6,
@@ -196,6 +239,40 @@ class CaptionSelectionTests(unittest.TestCase):
policy,
),
)
con.execute(
"""
INSERT INTO caption_windows (
id, media_id, start_ms, end_ms, representative_frame_id,
caption, caption_model, captioned_at, caption_route,
caption_route_reason, route_score, route_model,
route_policy, routed_at, window_policy
)
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?, ?, ?, ?,
?, CURRENT_TIMESTAMP, ?)
""",
(
"window-wrong-model",
"media-a",
0,
10000,
"wrong-model",
"old general caption",
"general-model",
"specialist",
"window-member-specialist",
0.9,
"router-model",
policy,
_caption_window_policy(),
),
)
con.execute(
"""
INSERT INTO caption_window_frames (window_id, frame_id, position)
VALUES (?, ?, ?)
""",
("window-wrong-model", "wrong-model", 0),
)
args = SimpleNamespace(
force=False,
+246
View File
@@ -0,0 +1,246 @@
from __future__ import annotations
import tempfile
import unittest
from pathlib import Path
from PIL import Image
from scene_search.caption_windows import (
build_contact_sheet,
caption_window_policy,
group_caption_windows,
load_media_caption_windows,
restore_media_caption_windows,
save_caption_window,
)
from scene_search.database import connect, init_db
class FakeFrameSource:
def image(self, row):
value = int(row["frame_number"]) * 20
return Image.new("RGB", (20, 10), (value, value, value))
def routed_frame(index: int, *, route: str = "general"):
row = {
"id": f"frame-{index}",
"media_id": "media-1",
"frame_number": index,
"timestamp_ms": index * 5000,
"interval_ms": 5000,
"title": "Movie",
}
score = 0.9 if route == "specialist" else 0.1
return (row, route, score, "score")
class CaptionWindowTests(unittest.TestCase):
def test_groups_thirty_seconds_and_promotes_any_specialist_member(self) -> None:
routed = [routed_frame(index) for index in range(6)]
routed[3] = routed_frame(3, route="specialist")
windows = group_caption_windows(
routed,
window_seconds=30,
max_frames=3,
route_model="router-model",
route_policy="policy-v1",
)
self.assertEqual(len(windows), 1)
window = windows[0]
self.assertEqual(window.start_ms, 0)
self.assertEqual(window.end_ms, 30000)
self.assertEqual(window.route, "specialist")
self.assertEqual(len(window.members), 6)
self.assertEqual(len(window.sample_rows), 3)
self.assertEqual(window.sample_rows[0]["frame_number"], 0)
self.assertEqual(window.sample_rows[-1]["frame_number"], 5)
self.assertEqual(
window.window_policy,
caption_window_policy(30, 3),
)
def test_contact_sheet_is_chronological_horizontal_sequence(self) -> None:
window = group_caption_windows(
[routed_frame(index) for index in range(3)],
window_seconds=30,
max_frames=3,
route_model="router-model",
route_policy="policy-v1",
)[0]
sheet = build_contact_sheet(FakeFrameSource(), window, gap_px=4)
try:
self.assertEqual(sheet.size, (68, 10))
self.assertEqual(sheet.getpixel((0, 0)), (0, 0, 0))
self.assertEqual(sheet.getpixel((24, 0)), (20, 20, 20))
self.assertEqual(sheet.getpixel((48, 0)), (40, 40, 40))
finally:
sheet.close()
def test_save_is_window_canonical_and_clears_legacy_frame_captions(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
db_path = Path(tmp) / "scenes.sqlite3"
init_db(db_path)
with connect(db_path) as con:
con.execute(
"""
INSERT INTO media (
id, source_path, title, content_sha256, size_bytes,
mtime_ns, interval_ms, model_id, embedding_dim
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
"media-1", "/movie.mkv", "Movie", "hash", 0, 0,
5000, "embed-model", 1,
),
)
for index in range(3):
con.execute(
"""
INSERT INTO frames (
id, media_id, frame_number, timestamp_ms, thumb_path,
embedding_f16, caption, caption_model, captioned_at,
caption_route, route_policy
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?, ?)
""",
(
f"frame-{index}", "media-1", index, index * 5000,
f"/frame-{index}.jpg", b"\x00\x00",
f"legacy {index}", "old-model", "general", "policy-v1",
),
)
window = group_caption_windows(
[routed_frame(index) for index in range(3)],
window_seconds=30,
max_frames=3,
route_model="router-model",
route_policy="policy-v1",
)[0]
ids = save_caption_window(
db_path,
window,
caption="A person walks across the room.",
model="general-model",
)
self.assertEqual(ids, ["frame-0", "frame-1", "frame-2"])
with connect(db_path) as con:
stored = con.execute(
"SELECT * FROM caption_windows WHERE id = ?",
(window.id,),
).fetchone()
members = con.execute(
"""
SELECT frame_id
FROM caption_window_frames
WHERE window_id = ?
ORDER BY position
""",
(window.id,),
).fetchall()
frames = con.execute(
"SELECT caption, meili_synced_at FROM frames ORDER BY frame_number"
).fetchall()
self.assertEqual(stored["caption"], "A person walks across the room.")
self.assertEqual(
[row["frame_id"] for row in members],
["frame-0", "frame-1", "frame-2"],
)
self.assertTrue(all(row["caption"] is None for row in frames))
self.assertTrue(all(row["meili_synced_at"] is None for row in frames))
def test_snapshot_restore_preserves_window_across_media_rebuild(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
db_path = Path(tmp) / "scenes.sqlite3"
init_db(db_path)
with connect(db_path) as con:
con.execute(
"""
INSERT INTO media (
id, source_path, title, content_sha256, size_bytes,
mtime_ns, interval_ms, model_id, embedding_dim
)
VALUES ('media-1', '/movie.mkv', 'Movie', 'hash', 0, 0, 5000,
'embed-model', 1)
"""
)
for index in range(2):
con.execute(
"""
INSERT INTO frames (
id, media_id, frame_number, timestamp_ms,
thumb_path, embedding_f16
)
VALUES (?, 'media-1', ?, ?, ?, ?)
""",
(
f"frame-{index}", index, index * 5000,
f"/frame-{index}.jpg", b"\x00\x00",
),
)
window = group_caption_windows(
[routed_frame(0), routed_frame(1)],
window_seconds=30,
max_frames=3,
route_model="router-model",
route_policy="policy-v1",
)[0]
save_caption_window(
db_path,
window,
caption="Two moments from one scene.",
model="general-model",
)
snapshot = load_media_caption_windows(db_path, "media-1")
with connect(db_path) as con:
con.execute("DELETE FROM media WHERE id = 'media-1'")
con.execute(
"""
INSERT INTO media (
id, source_path, title, content_sha256, size_bytes,
mtime_ns, interval_ms, model_id, embedding_dim
)
VALUES ('media-1', '/movie.mkv', 'Movie', 'hash', 0, 0, 5000,
'embed-model', 1)
"""
)
for index in range(2):
con.execute(
"""
INSERT INTO frames (
id, media_id, frame_number, timestamp_ms,
thumb_path, embedding_f16
)
VALUES (?, 'media-1', ?, ?, ?, ?)
""",
(
f"frame-{index}", index, index * 5000,
f"/frame-{index}.jpg", b"\x00\x00",
),
)
restored = restore_media_caption_windows(
con,
snapshot,
{"frame-0", "frame-1"},
)
self.assertEqual(restored, 1)
with connect(db_path) as con:
row = con.execute(
"SELECT caption FROM caption_windows"
).fetchone()
self.assertEqual(row["caption"], "Two moments from one scene.")
if __name__ == "__main__":
unittest.main()
+26
View File
@@ -34,6 +34,32 @@ class OllamaCaptionerTests(unittest.TestCase):
self.assertEqual(payload["options"]["temperature"], 0.0)
self.assertEqual(payload["options"]["num_predict"], 120)
def test_chronological_contact_sheet_adds_sequence_instruction(self) -> None:
captioner = OllamaCaptioner("http://localhost:11434", "vision-test")
response = Mock()
response.ok = True
response.json.return_value = {
"message": {"content": "A person enters the room and sits down."}
}
captioner.session.post = Mock(return_value=response)
from PIL import Image
image = Image.new("RGB", (12, 4))
try:
result = captioner.caption_image(
image,
chronological_sequence=True,
)
finally:
image.close()
self.assertEqual(result, "A person enters the room and sits down.")
payload = captioner.session.post.call_args.kwargs["json"]
prompt = payload["messages"][0]["content"]
self.assertIn("chronological contact sheet", prompt)
self.assertIn("left to right", prompt)
def test_unload_releases_model_from_ollama(self) -> None:
captioner = OllamaCaptioner("http://localhost:11434", "vision-test")
response = Mock()
+74
View File
@@ -117,6 +117,80 @@ class IncrementalCaptionSyncTests(unittest.TestCase):
self.assertIsNone(row["meili_synced_at"])
def test_window_caption_is_exposed_only_on_representative_document(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
db_path = Path(tmp) / "scenes.sqlite3"
self._seed(db_path, count=2)
with connect(db_path) as con:
con.execute(
"""
UPDATE frames
SET caption = NULL, caption_model = NULL, captioned_at = NULL
"""
)
con.execute(
"""
INSERT INTO caption_windows (
id, media_id, start_ms, end_ms, representative_frame_id,
caption, caption_model, captioned_at,
caption_route, caption_route_reason, route_score,
route_model, route_policy, routed_at, window_policy
)
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?, ?, ?, ?, ?,
CURRENT_TIMESTAMP, ?)
""",
(
"window-1",
"media-1",
0,
20000,
"frame-1",
"A person crosses the room.",
"caption-model",
"specialist",
"window-member-specialist",
0.9,
"router-model",
"policy-v1",
"contact-sheet-v1|seconds=30.000|max-frames=3",
),
)
con.executemany(
"""
INSERT INTO caption_window_frames (window_id, frame_id, position)
VALUES (?, ?, ?)
""",
[
("window-1", "frame-0", 0),
("window-1", "frame-1", 1),
],
)
fake_settings = SimpleNamespace(db_path=db_path)
meili = MagicMock()
with patch("scene_search.sync.settings", fake_settings):
count = sync_frame_metadata(
["frame-0", "frame-1"],
meili=meili,
)
self.assertEqual(count, 2)
documents = meili.update_documents.call_args.args[0]
by_id = {document["id"]: document for document in documents}
self.assertIsNone(by_id["frame-0"]["caption"])
self.assertEqual(
by_id["frame-1"]["caption"],
"A person crosses the room.",
)
self.assertEqual(by_id["frame-1"]["caption_window_id"], "window-1")
with connect(db_path) as con:
window = con.execute(
"SELECT meili_synced_at FROM caption_windows WHERE id = ?",
("window-1",),
).fetchone()
self.assertIsNotNone(window["meili_synced_at"])
def test_batcher_catches_up_unsynced_captions_in_small_batches(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
db_path = Path(tmp) / "scenes.sqlite3"