mirror of
https://github.com/MrSleeps/jellyfin-scene-search.git
synced 2026-11-19 00:03:36 +00:00
396 lines
16 KiB
Python
396 lines
16 KiB
Python
from __future__ import annotations
|
|
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import Mock
|
|
|
|
import numpy as np
|
|
|
|
from scene_search.database import connect, init_db, pack_vector
|
|
from scene_search.search_service import SearchService
|
|
|
|
|
|
class FakeEmbedder:
|
|
def embed_text(self, text: str) -> np.ndarray:
|
|
self.last_text = text
|
|
return np.asarray([1.0, 0.0], dtype=np.float32)
|
|
|
|
def match_probabilities(
|
|
self,
|
|
text_vector: np.ndarray,
|
|
image_vectors: np.ndarray,
|
|
) -> np.ndarray:
|
|
count = len(image_vectors)
|
|
if count == 0:
|
|
return np.asarray([], dtype=np.float32)
|
|
return np.linspace(0.9, 0.5, count, dtype=np.float32)
|
|
|
|
|
|
class SearchServiceTests(unittest.TestCase):
|
|
def test_search_returns_scenes_and_grouped_jellyfin_items(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
db_path = Path(tmp) / "scenes.sqlite3"
|
|
init_db(db_path)
|
|
|
|
with connect(db_path) as con:
|
|
con.execute(
|
|
"""
|
|
INSERT INTO media (
|
|
id, source_path, title, content_sha256, size_bytes, mtime_ns,
|
|
interval_ms, model_id, embedding_dim, source_kind,
|
|
jellyfin_item_id, jellyfin_library_id, jellyfin_item_type,
|
|
jellyfin_tags
|
|
)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
(
|
|
"media-1",
|
|
"/media/example.mkv",
|
|
"Example Movie",
|
|
"hash",
|
|
1,
|
|
0,
|
|
10000,
|
|
"model",
|
|
2,
|
|
"jellyfin",
|
|
"jf-1",
|
|
"lib-1",
|
|
"Movie",
|
|
'["tag one"]',
|
|
),
|
|
)
|
|
for number, timestamp in enumerate((0, 10000, 20000)):
|
|
con.execute(
|
|
"""
|
|
INSERT INTO frames (
|
|
id, media_id, frame_number, timestamp_ms,
|
|
thumb_path, embedding_f16, caption
|
|
)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
(
|
|
f"frame-{number}",
|
|
"media-1",
|
|
number,
|
|
timestamp,
|
|
f"/tmp/frame-{number}.jpg",
|
|
pack_vector(np.asarray([1.0, 0.0], dtype=np.float32)),
|
|
f"caption {number}",
|
|
),
|
|
)
|
|
|
|
service = SearchService(
|
|
db_path=db_path,
|
|
meili_url="http://meili.test",
|
|
meili_master_key="key",
|
|
meili_index="scenes",
|
|
model_id="model",
|
|
device="cpu",
|
|
semantic_ratio=0.8,
|
|
)
|
|
service._embedder = FakeEmbedder() # type: ignore[assignment]
|
|
service.meili = Mock()
|
|
service.meili.search_caption_text.return_value = {"hits": []}
|
|
service.meili.search_vector.return_value = {
|
|
"hits": [
|
|
{
|
|
"id": "frame-0",
|
|
"media_id": "media-1",
|
|
"jellyfin_item_id": "jf-1",
|
|
"jellyfin_library_id": "lib-1",
|
|
"jellyfin_item_type": "Movie",
|
|
"title": "Example Movie",
|
|
"timestamp_ms": 0,
|
|
"caption": "caption 0",
|
|
"jellyfin_tags": ["tag one"],
|
|
"_rankingScore": 0.95,
|
|
},
|
|
{
|
|
"id": "frame-1",
|
|
"media_id": "media-1",
|
|
"jellyfin_item_id": "jf-1",
|
|
"jellyfin_library_id": "lib-1",
|
|
"jellyfin_item_type": "Movie",
|
|
"title": "Example Movie",
|
|
"timestamp_ms": 10000,
|
|
"caption": "caption 1",
|
|
"jellyfin_tags": ["tag one"],
|
|
"_rankingScore": 0.85,
|
|
},
|
|
{
|
|
"id": "frame-2",
|
|
"media_id": "media-1",
|
|
"jellyfin_item_id": "jf-1",
|
|
"jellyfin_library_id": "lib-1",
|
|
"jellyfin_item_type": "Movie",
|
|
"title": "Example Movie",
|
|
"timestamp_ms": 20000,
|
|
"caption": "caption 2",
|
|
"jellyfin_tags": ["tag one"],
|
|
"_rankingScore": 0.75,
|
|
},
|
|
]
|
|
}
|
|
|
|
result = service.search("person outside", scene_limit=10, item_limit=5)
|
|
|
|
self.assertEqual(result["scene_count"], 3)
|
|
self.assertEqual(result["item_count"], 1)
|
|
self.assertEqual(len(result["items"]), 1)
|
|
self.assertEqual(result["items"][0]["jellyfin_item_id"], "jf-1")
|
|
self.assertEqual(result["items"][0]["best_timestamp_ms"], 0)
|
|
self.assertEqual(result["items"][0]["scene_matches"], 3)
|
|
self.assertEqual(len(result["items"][0]["matching_scenes"]), 3)
|
|
self.assertEqual(
|
|
result["items"][0]["matching_scenes"][1]["timestamp_ms"],
|
|
10000,
|
|
)
|
|
self.assertEqual(result["candidate_scene_count"], 3)
|
|
self.assertAlmostEqual(result["scenes"][0]["siglip_score"], 0.9, places=5)
|
|
|
|
call = service.meili.search_vector.call_args
|
|
self.assertEqual(call.args[1], "person outside")
|
|
self.assertEqual(
|
|
call.kwargs["filter_expression"],
|
|
'source_kind = "jellyfin"',
|
|
)
|
|
|
|
def test_search_expands_candidates_until_unique_item_limit_is_met(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
db_path = Path(tmp) / "scenes.sqlite3"
|
|
init_db(db_path)
|
|
|
|
with connect(db_path) as con:
|
|
for media_number in range(1, 4):
|
|
con.execute(
|
|
"""
|
|
INSERT INTO media (
|
|
id, source_path, title, content_sha256, size_bytes, mtime_ns,
|
|
interval_ms, model_id, embedding_dim, source_kind,
|
|
jellyfin_item_id, jellyfin_library_id, jellyfin_item_type,
|
|
jellyfin_tags
|
|
)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
(
|
|
f"media-{media_number}",
|
|
f"/media/example-{media_number}.mkv",
|
|
f"Movie {media_number}",
|
|
f"hash-{media_number}",
|
|
1,
|
|
0,
|
|
10000,
|
|
"model",
|
|
2,
|
|
"jellyfin",
|
|
f"jf-{media_number}",
|
|
"lib-1",
|
|
"Movie",
|
|
"[]",
|
|
),
|
|
)
|
|
|
|
frame_rows = [
|
|
(f"frame-{number}", "media-1", number, number * 10000)
|
|
for number in range(6)
|
|
] + [
|
|
("frame-6", "media-2", 0, 0),
|
|
("frame-7", "media-3", 0, 0),
|
|
]
|
|
for frame_id, media_id, frame_number, timestamp_ms in frame_rows:
|
|
con.execute(
|
|
"""
|
|
INSERT INTO frames (
|
|
id, media_id, frame_number, timestamp_ms,
|
|
thumb_path, embedding_f16, caption
|
|
)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
(
|
|
frame_id,
|
|
media_id,
|
|
frame_number,
|
|
timestamp_ms,
|
|
f"/tmp/{frame_id}.jpg",
|
|
pack_vector(np.asarray([1.0, 0.0], dtype=np.float32)),
|
|
f"caption {frame_id}",
|
|
),
|
|
)
|
|
|
|
def hit(frame_id: str, media_number: int, rank: float) -> dict:
|
|
return {
|
|
"id": frame_id,
|
|
"media_id": f"media-{media_number}",
|
|
"jellyfin_item_id": f"jf-{media_number}",
|
|
"jellyfin_library_id": "lib-1",
|
|
"jellyfin_item_type": "Movie",
|
|
"title": f"Movie {media_number}",
|
|
"timestamp_ms": 0,
|
|
"caption": f"caption {frame_id}",
|
|
"jellyfin_tags": [],
|
|
"_rankingScore": rank,
|
|
}
|
|
|
|
first_hits = [
|
|
hit(f"frame-{number}", 1, 0.99 - number * 0.01)
|
|
for number in range(6)
|
|
]
|
|
expanded_hits = first_hits + [
|
|
hit("frame-6", 2, 0.80),
|
|
hit("frame-7", 3, 0.79),
|
|
]
|
|
|
|
service = SearchService(
|
|
db_path=db_path,
|
|
meili_url="http://meili.test",
|
|
meili_master_key="key",
|
|
meili_index="scenes",
|
|
model_id="model",
|
|
device="cpu",
|
|
semantic_ratio=0.8,
|
|
)
|
|
service._embedder = FakeEmbedder() # type: ignore[assignment]
|
|
service.meili = Mock()
|
|
service.meili.search_caption_text.return_value = {"hits": []}
|
|
service.meili.search_vector.side_effect = [
|
|
{"hits": first_hits},
|
|
{"hits": expanded_hits},
|
|
]
|
|
|
|
result = service.search("person outside", scene_limit=2, item_limit=3)
|
|
|
|
self.assertEqual(
|
|
[call.args[2] for call in service.meili.search_vector.call_args_list],
|
|
[6, 12],
|
|
)
|
|
self.assertEqual(result["item_count"], 3)
|
|
self.assertEqual(
|
|
[item["jellyfin_item_id"] for item in result["items"]],
|
|
["jf-1", "jf-2", "jf-3"],
|
|
)
|
|
self.assertEqual(result["scene_count"], 2)
|
|
self.assertEqual(result["candidate_scene_count"], 8)
|
|
self.assertEqual(result["items"][0]["scene_matches"], 6)
|
|
self.assertEqual(
|
|
len(result["items"][0]["matching_scenes"]),
|
|
SearchService.SCENES_PER_ITEM,
|
|
)
|
|
|
|
|
|
def test_exact_caption_matches_are_reserved_without_losing_semantic_hits(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
db_path = Path(tmp) / "scenes.sqlite3"
|
|
init_db(db_path)
|
|
with connect(db_path) as con:
|
|
for number in range(1, 4):
|
|
con.execute(
|
|
"""
|
|
INSERT INTO media (
|
|
id, source_path, title, content_sha256,
|
|
size_bytes, mtime_ns, interval_ms, model_id,
|
|
embedding_dim, source_kind, jellyfin_item_id,
|
|
jellyfin_library_id, jellyfin_item_type
|
|
)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
(
|
|
f"media-{number}", f"/media/movie-{number}.mkv",
|
|
f"Movie {number}", f"hash-{number}", 1, 0,
|
|
10000, "model", 2, "jellyfin",
|
|
f"jf-{number}", "lib-1", "Movie",
|
|
),
|
|
)
|
|
con.execute(
|
|
"""
|
|
INSERT INTO frames (
|
|
id, media_id, frame_number, timestamp_ms,
|
|
thumb_path, embedding_f16
|
|
) VALUES (?, ?, ?, ?, ?, ?)
|
|
""",
|
|
(
|
|
f"frame-{number}", f"media-{number}",
|
|
0, 0, f"/tmp/frame-{number}.jpg",
|
|
pack_vector(np.asarray([1.0, 0.0], dtype=np.float32)),
|
|
),
|
|
)
|
|
|
|
def hit(number: int, *, caption: str | None = None) -> dict:
|
|
return {
|
|
"id": f"frame-{number}",
|
|
"jellyfin_item_id": f"jf-{number}",
|
|
"jellyfin_item_type": "Movie",
|
|
"title": f"Movie {number}",
|
|
"caption": caption,
|
|
"_rankingScore": 0.7,
|
|
}
|
|
|
|
service = SearchService(
|
|
db_path=db_path,
|
|
meili_url="http://meili.test",
|
|
meili_master_key="key",
|
|
meili_index="scenes",
|
|
model_id="model",
|
|
device="cpu",
|
|
semantic_ratio=0.8,
|
|
)
|
|
service._embedder = FakeEmbedder() # type: ignore[assignment]
|
|
service.meili = Mock()
|
|
service.meili.search_vector.return_value = {
|
|
"hits": [hit(1), hit(2), hit(3)],
|
|
}
|
|
service.meili.search_caption_text.return_value = {
|
|
"hits": [
|
|
hit(3, caption="A red umbrella in the rain"),
|
|
hit(2, caption="A red umbrella by the street"),
|
|
hit(1, caption="A red umbrella on a table"),
|
|
],
|
|
}
|
|
|
|
result = service.search(
|
|
"red umbrella", scene_limit=3, item_limit=3,
|
|
)
|
|
|
|
self.assertEqual(result["caption_text_item_count"], 2)
|
|
self.assertEqual(result["item_count"], 3)
|
|
self.assertEqual(
|
|
[item["jellyfin_item_id"] for item in result["items"]],
|
|
["jf-3", "jf-2", "jf-1"],
|
|
)
|
|
self.assertEqual(
|
|
[item["caption_text_match"] for item in result["items"]],
|
|
[True, True, False],
|
|
)
|
|
self.assertEqual(result["candidate_scene_count"], 3)
|
|
service.meili.search_caption_text.assert_called_once_with(
|
|
"red umbrella",
|
|
20,
|
|
filter_expression='source_kind = "jellyfin"',
|
|
)
|
|
|
|
def test_health_does_not_load_embedding_model(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
db_path = Path(tmp) / "scenes.sqlite3"
|
|
service = SearchService(
|
|
db_path=db_path,
|
|
meili_url="http://meili.test",
|
|
meili_master_key="key",
|
|
meili_index="scenes",
|
|
model_id="model",
|
|
device="cpu",
|
|
semantic_ratio=0.8,
|
|
)
|
|
service.meili = Mock()
|
|
service.meili.stats.return_value = {"numberOfDocuments": 0}
|
|
|
|
result = service.health()
|
|
|
|
self.assertFalse(result["model_loaded"])
|
|
self.assertEqual(result["status"], "ok")
|
|
self.assertEqual(result["meili_documents"], 0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|