Test preview/apply/restore safety and caption evidence selection

This commit is contained in:
Sleeps
2026-10-08 15:47:33 +01:00
parent 4e09a7989e
commit 9848bd238b
+232
View File
@@ -0,0 +1,232 @@
from __future__ import annotations
import tempfile
import unittest
from pathlib import Path
from unittest.mock import Mock, patch
from scene_search.database import connect, init_db
from scene_search.overviews import (
AI_OVERVIEW_HEADING,
_sample,
preview,
publish,
restore,
)
class VisualOverviewTests(unittest.TestCase):
@staticmethod
def seed(db_path: Path, *, with_window: bool = True) -> None:
init_db(db_path)
with connect(db_path) as con:
con.execute(
"""
INSERT INTO media (
id, source_path, title, content_sha256, size_bytes, mtime_ns,
interval_ms, model_id, embedding_dim, source_kind,
jellyfin_item_id, jellyfin_library_id
)
VALUES (
'media-1', 'jellyfin://film-1', 'My Film', 'hash', 0, 0,
10000, 'model', 2, 'jellyfin', 'film-1', 'library-1'
)
"""
)
for index in range(3):
con.execute(
"""
INSERT INTO frames (
id, media_id, frame_number, timestamp_ms,
thumb_path, embedding_f16, caption
)
VALUES (?, 'media-1', ?, ?, '', ?, ?)
""",
(
f"frame-{index}", index, index * 10000,
b"\0\0\0\0", f"Original legacy scene {index}",
),
)
if with_window:
con.execute(
"""
INSERT INTO caption_windows (
id, media_id, start_ms, end_ms, representative_frame_id,
caption, window_policy
)
VALUES ('cw-1', 'media-1', 0, 20000, 'frame-0',
'People crossing a plaza', 'test-policy')
"""
)
con.executemany(
"""
INSERT INTO caption_window_frames
(window_id, frame_id, position) VALUES ('cw-1', ?, ?)
""", [('frame-0', 0), ('frame-1', 1)],
)
@staticmethod
def client():
client = Mock()
remote = {
"Id": "film-1", "Type": "Movie", "Name": "My Film",
"Overview": "Original distributor synopsis.",
"Tags": ["Drama"], "Genres": ["Adventure"],
"ProviderIds": {"Tmdb": "1"},
}
live = {"item": dict(remote)}
def fetch(_):
return dict(live["item"])
def write(item, overview):
# A real Jellyfin write receives the entire DTO plus new Overview.
assert item["Tags"] == ["Drama"]
assert item["ProviderIds"] == {"Tmdb": "1"}
live["item"]["Overview"] = overview
client.item_for_update.side_effect = fetch
client.update_item_overview.side_effect = write
return client, live
def test_spaced_samples_cover_the_entire_video(self) -> None:
evidence = [(index * 1000, str(index)) for index in range(100)]
chosen = _sample(evidence, 5)
self.assertEqual(len(chosen), 5)
self.assertEqual(chosen[0], evidence[0])
self.assertEqual(chosen[-1], evidence[-1])
def test_preview_uses_window_captions_and_caches_without_jellyfin_write(self) -> None:
with tempfile.TemporaryDirectory() as d:
db_path = Path(d) / "scenes.sqlite3"
self.seed(db_path)
captured = []
def summarize(captions, **kwargs):
captured.extend(captions)
return "People gather outside and walk around a plaza."
with patch("scene_search.overviews._jellyfin_client") as jc:
first = preview(
db_path=db_path, title="My Film",
model="local-test-model", chat_fn=summarize,
)
second = preview(
db_path=db_path, media_id="media-1",
model="local-test-model",
chat_fn=lambda *a, **kw: self.fail("Unnecessary LLM call"),
)
jc.assert_not_called()
self.assertEqual(first["overview"], second["overview"])
self.assertEqual(first["sampled_captions"], 2)
self.assertEqual(first["total_captions"], 2)
self.assertEqual(captured[0][1], "People crossing a plaza")
self.assertIn("Original legacy scene 2", captured[1][1])
self.assertTrue(all("legacy scene 0" not in text for _, text in captured))
def test_publish_appends_only_after_preview_and_restore_returns_exact_original(self) -> None:
with tempfile.TemporaryDirectory() as d:
db_path = Path(d) / "scenes.sqlite3"
self.seed(db_path)
client, live = self.client()
with self.assertRaisesRegex(ValueError, "Generate and review"):
publish(db_path=db_path, title="My Film", client=client)
client.update_item_overview.assert_not_called()
preview(
db_path=db_path, title="My Film",
model="test", chat_fn=lambda *args, **kw: "People cross a plaza.",
)
published = publish(db_path=db_path, title="My Film", client=client)
self.assertEqual(published["status"], "applied")
self.assertTrue(
live["item"]["Overview"].startswith("Original distributor synopsis.")
)
self.assertIn(AI_OVERVIEW_HEADING, live["item"]["Overview"])
self.assertIn("People cross a plaza.", live["item"]["Overview"])
self.assertEqual(publish(
db_path=db_path, title="My Film", client=client
)["status"], "already_applied")
self.assertEqual(client.update_item_overview.call_count, 1)
result = restore(db_path=db_path, title="My Film", client=client)
self.assertEqual(result["status"], "restored")
self.assertEqual(live["item"]["Overview"], "Original distributor synopsis.")
self.assertEqual(client.update_item_overview.call_count, 2)
def test_out_of_band_metadata_edits_block_restore(self) -> None:
with tempfile.TemporaryDirectory() as d:
db_path = Path(d) / "scenes.sqlite3"
self.seed(db_path)
client, live = self.client()
preview(
db_path=db_path, title="My Film",
model="test", chat_fn=lambda *args, **kw: "A quiet courtyard.",
)
publish(db_path=db_path, title="My Film", client=client)
live["item"]["Overview"] += " New manual changes."
with self.assertRaisesRegex(RuntimeError, "edited after publishing"):
restore(db_path=db_path, title="My Film", client=client)
self.assertTrue(live["item"]["Overview"].endswith("New manual changes."))
def test_mismatched_remote_id_blocks_publish(self) -> None:
with tempfile.TemporaryDirectory() as d:
db_path = Path(d) / "scenes.sqlite3"
self.seed(db_path)
client, live = self.client()
live["item"]["Id"] = "wrong-item"
preview(
db_path=db_path, title="My Film", model="test",
chat_fn=lambda *args, **kw: "People passing through a courtyard.",
)
with self.assertRaisesRegex(RuntimeError, "different item ID"):
publish(db_path=db_path, title="My Film", client=client)
client.update_item_overview.assert_not_called()
def test_ambiguous_title_requires_media_id(self) -> None:
with tempfile.TemporaryDirectory() as d:
db_path = Path(d) / "scenes.sqlite3"
self.seed(db_path)
with connect(db_path) as con:
con.execute(
"""
INSERT INTO media (
id, source_path, title, content_sha256, size_bytes,
mtime_ns, interval_ms, model_id, embedding_dim,
source_kind, jellyfin_item_id
)
VALUES ('media-2', 'jellyfin://film-2', 'My Film',
'hash2', 0, 0, 10000, 'model', 2,
'jellyfin', 'film-2')
"""
)
with self.assertRaisesRegex(ValueError, "ambiguous"):
preview(db_path=db_path, title="My Film", model="test")
def test_partial_api_failure_preserves_restore_backup(self) -> None:
with tempfile.TemporaryDirectory() as d:
db_path = Path(d) / "scenes.sqlite3"
self.seed(db_path)
client, live = self.client()
preview(
db_path=db_path, title="My Film", model="test",
chat_fn=lambda *args, **kw: "Courtyard with passers-by.",
)
client.update_item_overview.side_effect = RuntimeError("network failure")
with self.assertRaisesRegex(RuntimeError, "network failure"):
publish(db_path=db_path, title="My Film", client=client)
with connect(db_path) as con:
row = con.execute(
"SELECT original_overview, applied_overview "
"FROM jellyfin_overview_drafts WHERE media_id = 'media-1'"
).fetchone()
self.assertEqual(row["original_overview"], "Original distributor synopsis.")
self.assertIn("Courtyard with passers-by.", row["applied_overview"])
# Restoring a write that never occurred is a safe no-op.
restore(db_path=db_path, title="My Film", client=client)
self.assertEqual(live["item"]["Overview"], "Original distributor synopsis.")
if __name__ == "__main__":
unittest.main()