mirror of
https://github.com/MrSleeps/jellyfin-scene-search.git
synced 2026-11-19 00:03:36 +00:00
Caption chronological 30-second frame windows
This commit is contained in:
@@ -6,7 +6,11 @@ from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from scene_search.cli import _route_policy, _select_caption_rows
|
||||
from scene_search.cli import (
|
||||
_caption_window_policy,
|
||||
_route_policy,
|
||||
_select_caption_rows,
|
||||
)
|
||||
from scene_search.database import connect, init_db
|
||||
|
||||
|
||||
@@ -22,6 +26,8 @@ class CaptionSelectionTests(unittest.TestCase):
|
||||
caption_specialist_model="specialist-model",
|
||||
caption_force_specialist=False,
|
||||
caption_router_enabled=False,
|
||||
caption_window_seconds=30.0,
|
||||
caption_window_max_frames=3,
|
||||
jellyfin_force_specialist_tag="scene-search:specialist",
|
||||
jellyfin_force_general_tag="scene-search:general",
|
||||
jellyfin_specialist_tags=(),
|
||||
@@ -82,6 +88,41 @@ class CaptionSelectionTests(unittest.TestCase):
|
||||
policy,
|
||||
),
|
||||
)
|
||||
con.execute(
|
||||
"""
|
||||
INSERT INTO caption_windows (
|
||||
id, media_id, start_ms, end_ms, representative_frame_id,
|
||||
caption, caption_model, captioned_at, caption_route,
|
||||
caption_route_reason, route_score, route_model,
|
||||
route_policy, routed_at, window_policy
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?, ?, ?, ?,
|
||||
?, CURRENT_TIMESTAMP, ?)
|
||||
""",
|
||||
(
|
||||
"window-done",
|
||||
"media-a",
|
||||
0,
|
||||
10000,
|
||||
"done",
|
||||
"already captioned",
|
||||
"general-model",
|
||||
"general",
|
||||
"window-all-general",
|
||||
0.0,
|
||||
"disabled",
|
||||
policy,
|
||||
_caption_window_policy(),
|
||||
),
|
||||
)
|
||||
con.execute(
|
||||
"""
|
||||
INSERT INTO caption_window_frames (window_id, frame_id, position)
|
||||
VALUES (?, ?, ?)
|
||||
""",
|
||||
("window-done", "done", 0),
|
||||
)
|
||||
|
||||
con.execute(
|
||||
"""
|
||||
INSERT INTO frames (
|
||||
@@ -139,6 +180,8 @@ class CaptionSelectionTests(unittest.TestCase):
|
||||
caption_specialist_model="specialist-model",
|
||||
caption_force_specialist=False,
|
||||
caption_router_enabled=True,
|
||||
caption_window_seconds=30.0,
|
||||
caption_window_max_frames=3,
|
||||
caption_router_model="router-model",
|
||||
caption_router_threshold=0.5,
|
||||
caption_router_context_seconds=6,
|
||||
@@ -196,6 +239,40 @@ class CaptionSelectionTests(unittest.TestCase):
|
||||
policy,
|
||||
),
|
||||
)
|
||||
con.execute(
|
||||
"""
|
||||
INSERT INTO caption_windows (
|
||||
id, media_id, start_ms, end_ms, representative_frame_id,
|
||||
caption, caption_model, captioned_at, caption_route,
|
||||
caption_route_reason, route_score, route_model,
|
||||
route_policy, routed_at, window_policy
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?, ?, ?, ?,
|
||||
?, CURRENT_TIMESTAMP, ?)
|
||||
""",
|
||||
(
|
||||
"window-wrong-model",
|
||||
"media-a",
|
||||
0,
|
||||
10000,
|
||||
"wrong-model",
|
||||
"old general caption",
|
||||
"general-model",
|
||||
"specialist",
|
||||
"window-member-specialist",
|
||||
0.9,
|
||||
"router-model",
|
||||
policy,
|
||||
_caption_window_policy(),
|
||||
),
|
||||
)
|
||||
con.execute(
|
||||
"""
|
||||
INSERT INTO caption_window_frames (window_id, frame_id, position)
|
||||
VALUES (?, ?, ?)
|
||||
""",
|
||||
("window-wrong-model", "wrong-model", 0),
|
||||
)
|
||||
|
||||
args = SimpleNamespace(
|
||||
force=False,
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from scene_search.caption_windows import (
|
||||
build_contact_sheet,
|
||||
caption_window_policy,
|
||||
group_caption_windows,
|
||||
load_media_caption_windows,
|
||||
restore_media_caption_windows,
|
||||
save_caption_window,
|
||||
)
|
||||
from scene_search.database import connect, init_db
|
||||
|
||||
|
||||
class FakeFrameSource:
|
||||
def image(self, row):
|
||||
value = int(row["frame_number"]) * 20
|
||||
return Image.new("RGB", (20, 10), (value, value, value))
|
||||
|
||||
|
||||
def routed_frame(index: int, *, route: str = "general"):
|
||||
row = {
|
||||
"id": f"frame-{index}",
|
||||
"media_id": "media-1",
|
||||
"frame_number": index,
|
||||
"timestamp_ms": index * 5000,
|
||||
"interval_ms": 5000,
|
||||
"title": "Movie",
|
||||
}
|
||||
score = 0.9 if route == "specialist" else 0.1
|
||||
return (row, route, score, "score")
|
||||
|
||||
|
||||
class CaptionWindowTests(unittest.TestCase):
|
||||
def test_groups_thirty_seconds_and_promotes_any_specialist_member(self) -> None:
|
||||
routed = [routed_frame(index) for index in range(6)]
|
||||
routed[3] = routed_frame(3, route="specialist")
|
||||
|
||||
windows = group_caption_windows(
|
||||
routed,
|
||||
window_seconds=30,
|
||||
max_frames=3,
|
||||
route_model="router-model",
|
||||
route_policy="policy-v1",
|
||||
)
|
||||
|
||||
self.assertEqual(len(windows), 1)
|
||||
window = windows[0]
|
||||
self.assertEqual(window.start_ms, 0)
|
||||
self.assertEqual(window.end_ms, 30000)
|
||||
self.assertEqual(window.route, "specialist")
|
||||
self.assertEqual(len(window.members), 6)
|
||||
self.assertEqual(len(window.sample_rows), 3)
|
||||
self.assertEqual(window.sample_rows[0]["frame_number"], 0)
|
||||
self.assertEqual(window.sample_rows[-1]["frame_number"], 5)
|
||||
self.assertEqual(
|
||||
window.window_policy,
|
||||
caption_window_policy(30, 3),
|
||||
)
|
||||
|
||||
def test_contact_sheet_is_chronological_horizontal_sequence(self) -> None:
|
||||
window = group_caption_windows(
|
||||
[routed_frame(index) for index in range(3)],
|
||||
window_seconds=30,
|
||||
max_frames=3,
|
||||
route_model="router-model",
|
||||
route_policy="policy-v1",
|
||||
)[0]
|
||||
|
||||
sheet = build_contact_sheet(FakeFrameSource(), window, gap_px=4)
|
||||
try:
|
||||
self.assertEqual(sheet.size, (68, 10))
|
||||
self.assertEqual(sheet.getpixel((0, 0)), (0, 0, 0))
|
||||
self.assertEqual(sheet.getpixel((24, 0)), (20, 20, 20))
|
||||
self.assertEqual(sheet.getpixel((48, 0)), (40, 40, 40))
|
||||
finally:
|
||||
sheet.close()
|
||||
|
||||
def test_save_is_window_canonical_and_clears_legacy_frame_captions(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db_path = Path(tmp) / "scenes.sqlite3"
|
||||
init_db(db_path)
|
||||
with connect(db_path) as con:
|
||||
con.execute(
|
||||
"""
|
||||
INSERT INTO media (
|
||||
id, source_path, title, content_sha256, size_bytes,
|
||||
mtime_ns, interval_ms, model_id, embedding_dim
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
"media-1", "/movie.mkv", "Movie", "hash", 0, 0,
|
||||
5000, "embed-model", 1,
|
||||
),
|
||||
)
|
||||
for index in range(3):
|
||||
con.execute(
|
||||
"""
|
||||
INSERT INTO frames (
|
||||
id, media_id, frame_number, timestamp_ms, thumb_path,
|
||||
embedding_f16, caption, caption_model, captioned_at,
|
||||
caption_route, route_policy
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?, ?)
|
||||
""",
|
||||
(
|
||||
f"frame-{index}", "media-1", index, index * 5000,
|
||||
f"/frame-{index}.jpg", b"\x00\x00",
|
||||
f"legacy {index}", "old-model", "general", "policy-v1",
|
||||
),
|
||||
)
|
||||
|
||||
window = group_caption_windows(
|
||||
[routed_frame(index) for index in range(3)],
|
||||
window_seconds=30,
|
||||
max_frames=3,
|
||||
route_model="router-model",
|
||||
route_policy="policy-v1",
|
||||
)[0]
|
||||
ids = save_caption_window(
|
||||
db_path,
|
||||
window,
|
||||
caption="A person walks across the room.",
|
||||
model="general-model",
|
||||
)
|
||||
|
||||
self.assertEqual(ids, ["frame-0", "frame-1", "frame-2"])
|
||||
with connect(db_path) as con:
|
||||
stored = con.execute(
|
||||
"SELECT * FROM caption_windows WHERE id = ?",
|
||||
(window.id,),
|
||||
).fetchone()
|
||||
members = con.execute(
|
||||
"""
|
||||
SELECT frame_id
|
||||
FROM caption_window_frames
|
||||
WHERE window_id = ?
|
||||
ORDER BY position
|
||||
""",
|
||||
(window.id,),
|
||||
).fetchall()
|
||||
frames = con.execute(
|
||||
"SELECT caption, meili_synced_at FROM frames ORDER BY frame_number"
|
||||
).fetchall()
|
||||
|
||||
self.assertEqual(stored["caption"], "A person walks across the room.")
|
||||
self.assertEqual(
|
||||
[row["frame_id"] for row in members],
|
||||
["frame-0", "frame-1", "frame-2"],
|
||||
)
|
||||
self.assertTrue(all(row["caption"] is None for row in frames))
|
||||
self.assertTrue(all(row["meili_synced_at"] is None for row in frames))
|
||||
|
||||
def test_snapshot_restore_preserves_window_across_media_rebuild(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db_path = Path(tmp) / "scenes.sqlite3"
|
||||
init_db(db_path)
|
||||
with connect(db_path) as con:
|
||||
con.execute(
|
||||
"""
|
||||
INSERT INTO media (
|
||||
id, source_path, title, content_sha256, size_bytes,
|
||||
mtime_ns, interval_ms, model_id, embedding_dim
|
||||
)
|
||||
VALUES ('media-1', '/movie.mkv', 'Movie', 'hash', 0, 0, 5000,
|
||||
'embed-model', 1)
|
||||
"""
|
||||
)
|
||||
for index in range(2):
|
||||
con.execute(
|
||||
"""
|
||||
INSERT INTO frames (
|
||||
id, media_id, frame_number, timestamp_ms,
|
||||
thumb_path, embedding_f16
|
||||
)
|
||||
VALUES (?, 'media-1', ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
f"frame-{index}", index, index * 5000,
|
||||
f"/frame-{index}.jpg", b"\x00\x00",
|
||||
),
|
||||
)
|
||||
|
||||
window = group_caption_windows(
|
||||
[routed_frame(0), routed_frame(1)],
|
||||
window_seconds=30,
|
||||
max_frames=3,
|
||||
route_model="router-model",
|
||||
route_policy="policy-v1",
|
||||
)[0]
|
||||
save_caption_window(
|
||||
db_path,
|
||||
window,
|
||||
caption="Two moments from one scene.",
|
||||
model="general-model",
|
||||
)
|
||||
snapshot = load_media_caption_windows(db_path, "media-1")
|
||||
|
||||
with connect(db_path) as con:
|
||||
con.execute("DELETE FROM media WHERE id = 'media-1'")
|
||||
con.execute(
|
||||
"""
|
||||
INSERT INTO media (
|
||||
id, source_path, title, content_sha256, size_bytes,
|
||||
mtime_ns, interval_ms, model_id, embedding_dim
|
||||
)
|
||||
VALUES ('media-1', '/movie.mkv', 'Movie', 'hash', 0, 0, 5000,
|
||||
'embed-model', 1)
|
||||
"""
|
||||
)
|
||||
for index in range(2):
|
||||
con.execute(
|
||||
"""
|
||||
INSERT INTO frames (
|
||||
id, media_id, frame_number, timestamp_ms,
|
||||
thumb_path, embedding_f16
|
||||
)
|
||||
VALUES (?, 'media-1', ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
f"frame-{index}", index, index * 5000,
|
||||
f"/frame-{index}.jpg", b"\x00\x00",
|
||||
),
|
||||
)
|
||||
restored = restore_media_caption_windows(
|
||||
con,
|
||||
snapshot,
|
||||
{"frame-0", "frame-1"},
|
||||
)
|
||||
|
||||
self.assertEqual(restored, 1)
|
||||
with connect(db_path) as con:
|
||||
row = con.execute(
|
||||
"SELECT caption FROM caption_windows"
|
||||
).fetchone()
|
||||
self.assertEqual(row["caption"], "Two moments from one scene.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -34,6 +34,32 @@ class OllamaCaptionerTests(unittest.TestCase):
|
||||
self.assertEqual(payload["options"]["temperature"], 0.0)
|
||||
self.assertEqual(payload["options"]["num_predict"], 120)
|
||||
|
||||
def test_chronological_contact_sheet_adds_sequence_instruction(self) -> None:
|
||||
captioner = OllamaCaptioner("http://localhost:11434", "vision-test")
|
||||
response = Mock()
|
||||
response.ok = True
|
||||
response.json.return_value = {
|
||||
"message": {"content": "A person enters the room and sits down."}
|
||||
}
|
||||
captioner.session.post = Mock(return_value=response)
|
||||
|
||||
from PIL import Image
|
||||
|
||||
image = Image.new("RGB", (12, 4))
|
||||
try:
|
||||
result = captioner.caption_image(
|
||||
image,
|
||||
chronological_sequence=True,
|
||||
)
|
||||
finally:
|
||||
image.close()
|
||||
|
||||
self.assertEqual(result, "A person enters the room and sits down.")
|
||||
payload = captioner.session.post.call_args.kwargs["json"]
|
||||
prompt = payload["messages"][0]["content"]
|
||||
self.assertIn("chronological contact sheet", prompt)
|
||||
self.assertIn("left to right", prompt)
|
||||
|
||||
def test_unload_releases_model_from_ollama(self) -> None:
|
||||
captioner = OllamaCaptioner("http://localhost:11434", "vision-test")
|
||||
response = Mock()
|
||||
|
||||
@@ -117,6 +117,80 @@ class IncrementalCaptionSyncTests(unittest.TestCase):
|
||||
|
||||
self.assertIsNone(row["meili_synced_at"])
|
||||
|
||||
def test_window_caption_is_exposed_only_on_representative_document(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db_path = Path(tmp) / "scenes.sqlite3"
|
||||
self._seed(db_path, count=2)
|
||||
with connect(db_path) as con:
|
||||
con.execute(
|
||||
"""
|
||||
UPDATE frames
|
||||
SET caption = NULL, caption_model = NULL, captioned_at = NULL
|
||||
"""
|
||||
)
|
||||
con.execute(
|
||||
"""
|
||||
INSERT INTO caption_windows (
|
||||
id, media_id, start_ms, end_ms, representative_frame_id,
|
||||
caption, caption_model, captioned_at,
|
||||
caption_route, caption_route_reason, route_score,
|
||||
route_model, route_policy, routed_at, window_policy
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?, ?, ?, ?, ?,
|
||||
CURRENT_TIMESTAMP, ?)
|
||||
""",
|
||||
(
|
||||
"window-1",
|
||||
"media-1",
|
||||
0,
|
||||
20000,
|
||||
"frame-1",
|
||||
"A person crosses the room.",
|
||||
"caption-model",
|
||||
"specialist",
|
||||
"window-member-specialist",
|
||||
0.9,
|
||||
"router-model",
|
||||
"policy-v1",
|
||||
"contact-sheet-v1|seconds=30.000|max-frames=3",
|
||||
),
|
||||
)
|
||||
con.executemany(
|
||||
"""
|
||||
INSERT INTO caption_window_frames (window_id, frame_id, position)
|
||||
VALUES (?, ?, ?)
|
||||
""",
|
||||
[
|
||||
("window-1", "frame-0", 0),
|
||||
("window-1", "frame-1", 1),
|
||||
],
|
||||
)
|
||||
|
||||
fake_settings = SimpleNamespace(db_path=db_path)
|
||||
meili = MagicMock()
|
||||
with patch("scene_search.sync.settings", fake_settings):
|
||||
count = sync_frame_metadata(
|
||||
["frame-0", "frame-1"],
|
||||
meili=meili,
|
||||
)
|
||||
|
||||
self.assertEqual(count, 2)
|
||||
documents = meili.update_documents.call_args.args[0]
|
||||
by_id = {document["id"]: document for document in documents}
|
||||
self.assertIsNone(by_id["frame-0"]["caption"])
|
||||
self.assertEqual(
|
||||
by_id["frame-1"]["caption"],
|
||||
"A person crosses the room.",
|
||||
)
|
||||
self.assertEqual(by_id["frame-1"]["caption_window_id"], "window-1")
|
||||
|
||||
with connect(db_path) as con:
|
||||
window = con.execute(
|
||||
"SELECT meili_synced_at FROM caption_windows WHERE id = ?",
|
||||
("window-1",),
|
||||
).fetchone()
|
||||
self.assertIsNotNone(window["meili_synced_at"])
|
||||
|
||||
def test_batcher_catches_up_unsynced_captions_in_small_batches(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db_path = Path(tmp) / "scenes.sqlite3"
|
||||
|
||||
Reference in New Issue
Block a user