from __future__ import annotations import tempfile import unittest from pathlib import Path from types import SimpleNamespace from unittest.mock import MagicMock, patch from scene_search.database import connect, init_db from scene_search.sync import ( CaptionMeiliSync, count_caption_repair_documents, repair_caption_documents, sync_frame_metadata, ) class IncrementalCaptionSyncTests(unittest.TestCase): def _seed(self, db_path: Path, *, count: int = 2) -> None: init_db(db_path) with connect(db_path) as con: con.execute( """ INSERT INTO media ( id, source_path, title, content_sha256, size_bytes, mtime_ns, interval_ms, model_id, embedding_dim, source_kind, jellyfin_item_id, jellyfin_library_id ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( "media-1", "jellyfin://item-1", "Movie", "hash", 0, 0, 10000, "model", 2, "jellyfin", "item-1", "lib-1", ), ) for index in range(count): con.execute( """ INSERT INTO frames ( id, media_id, frame_number, timestamp_ms, thumb_path, embedding_f16, caption, caption_model, captioned_at, caption_route, caption_route_reason, route_score, route_model, route_policy, routed_at, meili_synced_at ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, NULL) """, ( f"frame-{index}", "media-1", index, index * 10000, f"jellyfin://item-1/trickplay/320/{index}", b"\x00\x00\x00\x00", f"Caption {index}", "caption-model", "specialist", "score", 0.9, "router-model", "policy-v1", ), ) def test_sync_updates_only_caption_metadata_and_marks_frame_synced(self) -> None: with tempfile.TemporaryDirectory() as tmp: db_path = Path(tmp) / "scenes.sqlite3" self._seed(db_path, count=1) fake_settings = SimpleNamespace(db_path=db_path) meili = MagicMock() with patch("scene_search.sync.settings", fake_settings): count = sync_frame_metadata(["frame-0"], meili=meili) self.assertEqual(count, 1) meili.update_documents.assert_called_once() args, kwargs = meili.update_documents.call_args self.assertTrue(kwargs["wait"]) self.assertEqual(args[0][0]["id"], "frame-0") self.assertEqual(args[0][0]["caption"], "Caption 0") self.assertNotIn("_vectors", args[0][0]) with connect(db_path) as con: row = con.execute( "SELECT meili_synced_at FROM frames WHERE id = ?", ("frame-0",), ).fetchone() self.assertIsNotNone(row["meili_synced_at"]) def test_failed_meili_update_leaves_frame_unsynced(self) -> None: with tempfile.TemporaryDirectory() as tmp: db_path = Path(tmp) / "scenes.sqlite3" self._seed(db_path, count=1) fake_settings = SimpleNamespace(db_path=db_path) meili = MagicMock() meili.update_documents.side_effect = RuntimeError("Meili unavailable") with ( patch("scene_search.sync.settings", fake_settings), self.assertRaisesRegex(RuntimeError, "Meili unavailable"), ): sync_frame_metadata(["frame-0"], meili=meili) with connect(db_path) as con: row = con.execute( "SELECT meili_synced_at FROM frames WHERE id = ?", ("frame-0",), ).fetchone() self.assertIsNone(row["meili_synced_at"]) def test_window_caption_is_exposed_only_on_representative_document(self) -> None: with tempfile.TemporaryDirectory() as tmp: db_path = Path(tmp) / "scenes.sqlite3" self._seed(db_path, count=2) with connect(db_path) as con: con.execute( """ UPDATE frames SET caption = NULL, caption_model = NULL, captioned_at = NULL """ ) con.execute( """ INSERT INTO caption_windows ( id, media_id, start_ms, end_ms, representative_frame_id, caption, caption_model, captioned_at, caption_route, caption_route_reason, route_score, route_model, route_policy, routed_at, window_policy ) VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?) """, ( "window-1", "media-1", 0, 20000, "frame-1", "A person crosses the room.", "caption-model", "specialist", "window-member-specialist", 0.9, "router-model", "policy-v1", "contact-sheet-v1|seconds=30.000|max-frames=3", ), ) con.executemany( """ INSERT INTO caption_window_frames (window_id, frame_id, position) VALUES (?, ?, ?) """, [ ("window-1", "frame-0", 0), ("window-1", "frame-1", 1), ], ) fake_settings = SimpleNamespace(db_path=db_path) meili = MagicMock() with patch("scene_search.sync.settings", fake_settings): count = sync_frame_metadata( ["frame-0", "frame-1"], meili=meili, ) self.assertEqual(count, 2) documents = meili.update_documents.call_args.args[0] by_id = {document["id"]: document for document in documents} self.assertIsNone(by_id["frame-0"]["caption"]) self.assertEqual( by_id["frame-1"]["caption"], "A person crosses the room.", ) self.assertEqual(by_id["frame-1"]["caption_window_id"], "window-1") with connect(db_path) as con: window = con.execute( "SELECT meili_synced_at FROM caption_windows WHERE id = ?", ("window-1",), ).fetchone() self.assertIsNotNone(window["meili_synced_at"]) def test_batcher_catches_up_unsynced_captions_in_small_batches(self) -> None: with tempfile.TemporaryDirectory() as tmp: db_path = Path(tmp) / "scenes.sqlite3" self._seed(db_path, count=2) fake_settings = SimpleNamespace(db_path=db_path) meili = MagicMock() progress: list[dict] = [] with patch("scene_search.sync.settings", fake_settings): syncer = CaptionMeiliSync( ["lib-1"], meili=meili, batch_size=1, progress=progress.append, ) caught_up = syncer.catch_up() self.assertEqual(caught_up, 2) self.assertEqual(syncer.updated_total, 2) self.assertEqual(meili.update_documents.call_count, 2) self.assertEqual(progress[-1]["meili_sync_pending"], 0) def test_repair_restores_identity_vectors_and_window_caption_without_reindex(self) -> None: with tempfile.TemporaryDirectory() as tmp: db_path = Path(tmp) / "scenes.sqlite3" self._seed(db_path, count=3) with connect(db_path) as con: con.execute( """ UPDATE media SET jellyfin_item_type = 'Movie', jellyfin_media_source_id = 'source-1' WHERE id = 'media-1' """ ) con.execute( """ UPDATE frames SET caption = NULL, caption_model = NULL, captioned_at = NULL WHERE id IN ('frame-0', 'frame-1') """ ) con.execute( """ INSERT INTO caption_windows ( id, media_id, start_ms, end_ms, representative_frame_id, caption, caption_model, captioned_at, caption_route, caption_route_reason, route_score, route_model, route_policy, routed_at, window_policy ) VALUES ( 'window-1', 'media-1', 0, 20000, 'frame-1', 'A distinct red umbrella.', 'caption-model', CURRENT_TIMESTAMP, 'specialist', 'reason', 0.9, 'router-model', 'policy-v1', CURRENT_TIMESTAMP, 'test-window-policy' ) """ ) con.executemany( """ INSERT INTO caption_window_frames (window_id, frame_id, position) VALUES (?, ?, ?) """, [('window-1', 'frame-0', 0), ('window-1', 'frame-1', 1)], ) # frame-2 is a legacy captioned frame, so it needs repair too. self.assertEqual(count_caption_repair_documents(db_path), 3) meili = MagicMock() meili.index = "scenes" meili.add_documents.side_effect = [101, 102] progress = [] restored = repair_caption_documents( db_path=db_path, meili=meili, batch_size=2, progress=progress.append, ) self.assertEqual(restored, 3) self.assertEqual(meili.add_documents.call_count, 2) self.assertTrue(all( call.kwargs["wait"] is False for call in meili.add_documents.call_args_list )) self.assertEqual(meili.wait_for_task.call_count, 2) docs = [ document for call in meili.add_documents.call_args_list for document in call.args[0] ] by_id = {doc["id"]: doc for doc in docs} self.assertEqual(len(by_id), 3) self.assertEqual(by_id["frame-1"]["caption"], "A distinct red umbrella.") self.assertEqual(by_id["frame-1"]["caption_window_id"], "window-1") self.assertIsNone(by_id["frame-0"]["caption"]) self.assertEqual(by_id["frame-2"]["caption"], "Caption 2") for doc in docs: self.assertEqual(doc["jellyfin_item_id"], "item-1") self.assertEqual(doc["jellyfin_item_type"], "Movie") self.assertEqual(doc["jellyfin_library_id"], "lib-1") self.assertEqual(doc["title"], "Movie") self.assertIn("siglip2", doc["_vectors"]) self.assertEqual(len(doc["_vectors"]["siglip2"]), 2) self.assertEqual(progress[-1]["documents_processed"], 3) with connect(db_path) as con: checkpoint = con.execute( "SELECT * FROM caption_index_repair_checkpoints " "WHERE index_uid = 'scenes'" ).fetchone() self.assertEqual(checkpoint["completed_documents"], 3) self.assertIsNone(checkpoint["pending_task_uid"]) def test_repair_resumes_pending_task_without_submitting_again(self) -> None: with tempfile.TemporaryDirectory() as tmp: db_path = Path(tmp) / "scenes.sqlite3" self._seed(db_path, count=2) meili = MagicMock() meili.index = "scenes" meili.add_documents.return_value = 710 meili.wait_for_task.side_effect = KeyboardInterrupt() with self.assertRaises(KeyboardInterrupt): repair_caption_documents( db_path=db_path, meili=meili, batch_size=2 ) with connect(db_path) as con: checkpoint = con.execute( "SELECT * FROM caption_index_repair_checkpoints" ).fetchone() self.assertEqual(checkpoint["pending_task_uid"], 710) self.assertEqual(checkpoint["completed_documents"], 0) meili.wait_for_task.side_effect = None restored = repair_caption_documents( db_path=db_path, meili=meili, batch_size=2 ) self.assertEqual(restored, 2) self.assertEqual(meili.add_documents.call_count, 1) self.assertEqual(meili.wait_for_task.call_count, 2) with connect(db_path) as con: checkpoint = con.execute( "SELECT * FROM caption_index_repair_checkpoints" ).fetchone() self.assertIsNone(checkpoint["pending_task_uid"]) self.assertEqual(checkpoint["completed_documents"], 2) # Subsequent runs skip everything already confirmed. self.assertEqual( repair_caption_documents( db_path=db_path, meili=meili, batch_size=2 ), 0 ) self.assertEqual(meili.add_documents.call_count, 1) def test_adopt_old_first_batch_avoids_duplicating_task(self) -> None: with tempfile.TemporaryDirectory() as tmp: db_path = Path(tmp) / "scenes.sqlite3" self._seed(db_path, count=2) meili = MagicMock() meili.index = "scenes" meili.get_task.return_value = { "uid": 41758, "indexUid": "scenes", "type": "documentAdditionOrUpdate", "status": "succeeded", "details": {"receivedDocuments": 2}, } restored = repair_caption_documents( db_path=db_path, meili=meili, batch_size=2, resume_task_uid=41758, ) self.assertEqual(restored, 2) meili.add_documents.assert_not_called() meili.wait_for_task.assert_called_once() with connect(db_path) as con: row = con.execute( "SELECT * FROM caption_index_repair_checkpoints" ).fetchone() self.assertEqual(row["completed_documents"], 2) def test_pending_task_error_is_not_silently_duplicated(self) -> None: with tempfile.TemporaryDirectory() as tmp: db_path = Path(tmp) / "scenes.sqlite3" self._seed(db_path, count=1) meili = MagicMock() meili.index = "scenes" meili.add_documents.return_value = 512 meili.wait_for_task.side_effect = RuntimeError( "API temporarily unavailable" ) meili.get_task.return_value = { "status": "processing", } with self.assertRaisesRegex( RuntimeError, "API temporarily unavailable" ): repair_caption_documents( db_path=db_path, meili=meili, batch_size=1 ) with connect(db_path) as con: row = con.execute( "SELECT * FROM caption_index_repair_checkpoints" ).fetchone() self.assertEqual(row["pending_task_uid"], 512) self.assertEqual(row["completed_documents"], 0) meili.wait_for_task.side_effect = None self.assertEqual( repair_caption_documents( db_path=db_path, meili=meili, batch_size=1 ), 1 ) self.assertEqual(meili.add_documents.call_count, 1) if __name__ == "__main__": unittest.main()