Files
jellyfin-scene-search/scene_search/search_service.py
T

331 lines
12 KiB
Python

from __future__ import annotations
import threading
from pathlib import Path
from typing import Any
import numpy as np
from .database import connect, init_db, unpack_vector
from .embedder import SiglipEmbedder
from .meili import Meili
class SearchService:
"""Shared hybrid search implementation for CLI and HTTP consumers."""
MAX_SEARCH_CANDIDATES = 500
SCENES_PER_ITEM = 5
PRIORITY_CAPTION_ITEMS = 2
def __init__(
self,
*,
db_path: Path,
meili_url: str,
meili_master_key: str,
meili_index: str,
model_id: str,
device: str,
semantic_ratio: float,
) -> None:
self.db_path = db_path
self.meili = Meili(meili_url, meili_master_key, meili_index)
self.model_id = model_id
self.device = device
self.semantic_ratio = semantic_ratio
self._embedder: SiglipEmbedder | None = None
self._model_lock = threading.Lock()
@property
def model_loaded(self) -> bool:
return self._embedder is not None
def get_embedder(self) -> SiglipEmbedder:
"""Get the shared embedding model, loading it lazily."""
if self._embedder is None:
self._embedder = SiglipEmbedder(self.model_id, self.device)
return self._embedder
@property
def model_lock(self) -> threading.Lock:
"""Lock guarding exclusive access to the shared GPU embedding model."""
return self._model_lock
def health(self) -> dict[str, Any]:
init_db(self.db_path)
with connect(self.db_path) as con:
media_count = int(con.execute("SELECT COUNT(*) FROM media").fetchone()[0])
frame_count = int(con.execute("SELECT COUNT(*) FROM frames").fetchone()[0])
jellyfin_media_count = int(
con.execute(
"SELECT COUNT(*) FROM media WHERE source_kind = 'jellyfin'"
).fetchone()[0]
)
meili_error: str | None = None
meili_documents: int | None = None
try:
stats = self.meili.stats()
raw_count = stats.get("numberOfDocuments")
if isinstance(raw_count, int):
meili_documents = raw_count
except Exception as exc:
meili_error = str(exc)
return {
"status": "ok" if meili_error is None else "degraded",
"model_loaded": self.model_loaded,
"media_count": media_count,
"jellyfin_media_count": jellyfin_media_count,
"frame_count": frame_count,
"meili_documents": meili_documents,
"meili_error": meili_error,
}
def _vectors_for_hits(self, frame_ids: list[str]) -> dict[str, np.ndarray]:
if not frame_ids:
return {}
placeholders = ",".join("?" for _ in frame_ids)
with connect(self.db_path) as con:
rows = con.execute(
f"""
SELECT f.id, f.embedding_f16, m.embedding_dim
FROM frames f
JOIN media m ON m.id = f.media_id
WHERE f.id IN ({placeholders})
""",
frame_ids,
).fetchall()
return {
str(row["id"]): unpack_vector(
row["embedding_f16"],
int(row["embedding_dim"]),
)
for row in rows
}
@staticmethod
def _scene_from_hit(
hit: dict[str, Any],
siglip_score: float | None,
) -> dict[str, Any]:
ranking_score = hit.get("_rankingScore")
return {
"frame_id": hit.get("id"),
"media_id": hit.get("media_id"),
"jellyfin_item_id": hit.get("jellyfin_item_id"),
"jellyfin_media_source_id": hit.get("jellyfin_media_source_id"),
"jellyfin_library_id": hit.get("jellyfin_library_id"),
"jellyfin_item_type": hit.get("jellyfin_item_type"),
"title": hit.get("title"),
"timestamp_ms": int(hit.get("timestamp_ms") or 0),
"caption": hit.get("caption"),
"caption_text_match": bool(hit.get("_caption_text_match")),
"caption_model": hit.get("caption_model"),
"caption_route": hit.get("caption_route"),
"caption_window_id": hit.get("caption_window_id"),
"caption_window_start_ms": hit.get("caption_window_start_ms"),
"caption_window_end_ms": hit.get("caption_window_end_ms"),
"route_score": hit.get("route_score"),
"tags": hit.get("jellyfin_tags") or [],
"ranking_score": (
float(ranking_score)
if isinstance(ranking_score, (float, int))
else None
),
"siglip_score": siglip_score,
}
@staticmethod
def _group_items(
scenes: list[dict[str, Any]],
item_limit: int,
) -> list[dict[str, Any]]:
grouped: dict[str, dict[str, Any]] = {}
for scene in scenes:
item_id = scene.get("jellyfin_item_id")
if not item_id:
continue
existing = grouped.get(str(item_id))
if existing is None:
grouped[str(item_id)] = {
"jellyfin_item_id": item_id,
"jellyfin_library_id": scene.get("jellyfin_library_id"),
"jellyfin_item_type": scene.get("jellyfin_item_type"),
"title": scene.get("title"),
"tags": scene.get("tags") or [],
"best_timestamp_ms": scene.get("timestamp_ms"),
"best_caption": scene.get("caption"),
"caption_text_match": scene.get("caption_text_match", False),
"ranking_score": scene.get("ranking_score"),
"siglip_score": scene.get("siglip_score"),
"scene_matches": 1,
"matching_scenes": [scene],
}
else:
existing["scene_matches"] = int(existing["scene_matches"]) + 1
matching_scenes = existing["matching_scenes"]
if len(matching_scenes) < SearchService.SCENES_PER_ITEM:
matching_scenes.append(scene)
return list(grouped.values())[:item_limit]
def search(
self,
query: str,
*,
scene_limit: int = 50,
item_limit: int = 20,
jellyfin_only: bool = True,
) -> dict[str, Any]:
query = query.strip()
if not query:
raise ValueError("Search query cannot be empty")
init_db(self.db_path)
# SigLIP / PyTorch model execution is kept behind one lock so concurrent
# API requests do not fight over a single GPU model instance.
with self._model_lock:
embedder = self.get_embedder()
text_vector = embedder.embed_text(query)
filter_expression = (
'source_kind = "jellyfin"'
if jellyfin_only
else None
)
# Jellyfin consumes item-level results, while Meilisearch ranks scene
# frames. A single film can occupy many of the best scene hits, so a
# fixed scene limit can collapse to only a handful of unique items.
# Expand the candidate pool until we have enough unique Jellyfin
# items, Meilisearch is exhausted, or a bounded safety ceiling is hit.
candidate_limit = min(
max(scene_limit, item_limit * 2),
self.MAX_SEARCH_CANDIDATES,
)
hits: list[dict[str, Any]] = []
while True:
result = self.meili.search_vector(
text_vector,
query,
candidate_limit,
self.semantic_ratio,
filter_expression=filter_expression,
)
hits = list(result.get("hits") or [])
unique_item_ids = {
str(hit["jellyfin_item_id"])
for hit in hits
if hit.get("jellyfin_item_id")
}
if (
len(unique_item_ids) >= item_limit
or len(hits) < candidate_limit
or candidate_limit >= self.MAX_SEARCH_CANDIDATES
):
break
candidate_limit = min(
candidate_limit * 2,
self.MAX_SEARCH_CANDIDATES,
)
# Caption windows are indexed on their representative frame.
# With millions of mostly visual-only frames, hybrid vector
# retrieval can bury a literal caption match below the candidate
# cutoff. Fetch exact text matches in the caption field separately
# and reserve just two *distinct items* for them, preserving most
# of the normal semantic/topical search order.
caption_hits: list[dict[str, Any]] = []
try:
caption_results = self.meili.search_caption_text(
query,
min(max(item_limit * 4, 20), 100),
filter_expression=filter_expression,
)
caption_hits = list(caption_results.get("hits") or [])
except Exception as exc:
# Old/incompatible Meili instances should still serve the
# existing semantic path instead of failing every search.
print(f"WARN caption-only search unavailable: {exc}")
priority_hits: list[dict[str, Any]] = []
priority_items: set[str] = set()
for hit in caption_hits:
item_id = hit.get("jellyfin_item_id")
if (
not item_id
or not hit.get("id")
or not str(hit.get("caption") or "").strip()
or str(item_id) in priority_items
):
continue
priority_items.add(str(item_id))
priority_hits.append({**hit, "_caption_text_match": True})
if len(priority_hits) >= min(
self.PRIORITY_CAPTION_ITEMS, item_limit
):
break
priority_frame_ids = {
str(hit["id"]) for hit in priority_hits
}
hits = (
priority_hits
+ [
hit for hit in hits
if str(hit.get("id")) not in priority_frame_ids
]
)[:self.MAX_SEARCH_CANDIDATES]
frame_ids = [
str(hit["id"])
for hit in hits
if hit.get("id") is not None
]
vectors = self._vectors_for_hits(frame_ids)
scored_vectors = [vectors.get(str(hit.get("id"))) for hit in hits]
if hits and all(vector is not None for vector in scored_vectors):
siglip_scores = embedder.match_probabilities(
text_vector,
np.stack(scored_vectors), # type: ignore[arg-type]
)
scores: list[float | None] = [
float(score) for score in siglip_scores
]
else:
scores = [None] * len(hits)
scenes = [
self._scene_from_hit(hit, score)
for hit, score in zip(hits, scores, strict=True)
]
items = self._group_items(scenes, item_limit)
display_scenes = scenes[:scene_limit]
return {
"query": query,
"semantic_ratio": self.semantic_ratio,
"caption_text_item_count": len(priority_hits),
"scene_count": len(display_scenes),
"candidate_scene_count": len(scenes),
"item_count": len(items),
"items": items,
# Kept for backward compatibility with scene-level consumers. The
# dashboard now treats items as canonical and exposes matching scenes
# underneath each item instead of repeating the same title.
"scenes": display_scenes,
}