mirror of
https://github.com/MrSleeps/jellyfin-scene-search.git
synced 2026-11-17 23:41:28 +00:00
Merge pull request #2 from MrSleeps/feature/search-confidence-diagnostics
Add confidence diagnostics for semantic scene search
This commit is contained in:
+92
-4
@@ -165,6 +165,48 @@ def cmd_sync_meili(_: argparse.Namespace) -> None:
|
|||||||
print("Meilisearch rebuild complete (tasks may continue briefly inside Meili).")
|
print("Meilisearch rebuild complete (tasks may continue briefly inside Meili).")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_status(_: argparse.Namespace) -> None:
|
||||||
|
init_db(settings.db_path)
|
||||||
|
|
||||||
|
print("Canonical SQLite")
|
||||||
|
print("----------------")
|
||||||
|
with connect(settings.db_path) as con:
|
||||||
|
media_rows = con.execute(
|
||||||
|
"""
|
||||||
|
SELECT m.id, m.title, m.interval_ms, m.model_id, COUNT(f.id) AS frames
|
||||||
|
FROM media m
|
||||||
|
LEFT JOIN frames f ON f.media_id = m.id
|
||||||
|
GROUP BY m.id, m.title, m.interval_ms, m.model_id
|
||||||
|
ORDER BY m.title, m.id
|
||||||
|
"""
|
||||||
|
).fetchall()
|
||||||
|
total_frames = sum(int(row["frames"]) for row in media_rows)
|
||||||
|
|
||||||
|
print(f"Media items: {len(media_rows)}")
|
||||||
|
print(f"Frames: {total_frames}")
|
||||||
|
if media_rows:
|
||||||
|
print()
|
||||||
|
for row in media_rows:
|
||||||
|
print(
|
||||||
|
f"- {row['title']}: {row['frames']} frames "
|
||||||
|
f"({row['interval_ms'] / 1000:g}s interval)"
|
||||||
|
)
|
||||||
|
|
||||||
|
print()
|
||||||
|
print("Meilisearch")
|
||||||
|
print("-----------")
|
||||||
|
meili = Meili(settings.meili_url, settings.meili_master_key, settings.meili_index)
|
||||||
|
try:
|
||||||
|
stats = meili.stats()
|
||||||
|
except Exception as exc:
|
||||||
|
print(f"Status: unavailable ({exc})")
|
||||||
|
return
|
||||||
|
|
||||||
|
print(f"Index: {settings.meili_index}")
|
||||||
|
print(f"Documents: {stats.get('numberOfDocuments', 'unknown')}")
|
||||||
|
print(f"Indexing: {stats.get('isIndexing', 'unknown')}")
|
||||||
|
|
||||||
def cmd_search(args: argparse.Namespace) -> None:
|
def cmd_search(args: argparse.Namespace) -> None:
|
||||||
init_db(settings.db_path)
|
init_db(settings.db_path)
|
||||||
embedder = build_embedder()
|
embedder = build_embedder()
|
||||||
@@ -177,11 +219,54 @@ def cmd_search(args: argparse.Namespace) -> None:
|
|||||||
print("No hits")
|
print("No hits")
|
||||||
return
|
return
|
||||||
|
|
||||||
|
frame_ids = [hit.get("id") for hit in hits if hit.get("id")]
|
||||||
|
frame_vectors: dict[str, object] = {}
|
||||||
|
if frame_ids:
|
||||||
|
placeholders = ",".join("?" for _ in frame_ids)
|
||||||
|
with connect(settings.db_path) as con:
|
||||||
|
rows = con.execute(
|
||||||
|
f"""
|
||||||
|
SELECT f.id, f.embedding_f16, m.embedding_dim
|
||||||
|
FROM frames f
|
||||||
|
JOIN media m ON m.id = f.media_id
|
||||||
|
WHERE f.id IN ({placeholders})
|
||||||
|
""",
|
||||||
|
frame_ids,
|
||||||
|
).fetchall()
|
||||||
|
frame_vectors = {
|
||||||
|
row["id"]: unpack_vector(row["embedding_f16"], row["embedding_dim"])
|
||||||
|
for row in rows
|
||||||
|
}
|
||||||
|
|
||||||
|
scored_vectors = [frame_vectors.get(hit.get("id")) for hit in hits]
|
||||||
|
if all(item is not None for item in scored_vectors):
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
siglip_scores = embedder.match_probabilities(
|
||||||
|
vector,
|
||||||
|
np.stack(scored_vectors), # type: ignore[arg-type]
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
siglip_scores = [None] * len(hits)
|
||||||
|
|
||||||
print(f"\nQuery: {args.query!r}\n")
|
print(f"\nQuery: {args.query!r}\n")
|
||||||
for i, hit in enumerate(hits, 1):
|
for i, (hit, siglip_score) in enumerate(zip(hits, siglip_scores, strict=True), 1):
|
||||||
score = hit.get("_rankingScore")
|
meili_score = hit.get("_rankingScore")
|
||||||
score_text = f" score={score:.4f}" if isinstance(score, (float, int)) else ""
|
meili_text = (
|
||||||
print(f"{i:2d}. {hit.get('title')} @ {format_ts(int(hit.get('timestamp_ms', 0)))}{score_text}")
|
f" meili={meili_score:.4f}"
|
||||||
|
if isinstance(meili_score, (float, int))
|
||||||
|
else ""
|
||||||
|
)
|
||||||
|
siglip_text = (
|
||||||
|
f" siglip={float(siglip_score):.4f}"
|
||||||
|
if siglip_score is not None
|
||||||
|
else ""
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
f"{i:2d}. {hit.get('title')} @ "
|
||||||
|
f"{format_ts(int(hit.get('timestamp_ms', 0)))}"
|
||||||
|
f"{meili_text}{siglip_text}"
|
||||||
|
)
|
||||||
print(f" {hit.get('thumb_path')}")
|
print(f" {hit.get('thumb_path')}")
|
||||||
|
|
||||||
|
|
||||||
@@ -202,6 +287,9 @@ def make_parser() -> argparse.ArgumentParser:
|
|||||||
sync = sub.add_parser("sync-meili", help="Rebuild/feed Meilisearch from canonical SQLite vectors")
|
sync = sub.add_parser("sync-meili", help="Rebuild/feed Meilisearch from canonical SQLite vectors")
|
||||||
sync.set_defaults(func=cmd_sync_meili)
|
sync.set_defaults(func=cmd_sync_meili)
|
||||||
|
|
||||||
|
status = sub.add_parser("status", help="Show canonical SQLite and Meilisearch index status")
|
||||||
|
status.set_defaults(func=cmd_status)
|
||||||
|
|
||||||
search = sub.add_parser("search", help="Semantic scene search")
|
search = sub.add_parser("search", help="Semantic scene search")
|
||||||
search.add_argument("query")
|
search.add_argument("query")
|
||||||
search.add_argument("--limit", type=int, default=10)
|
search.add_argument("--limit", type=int, default=10)
|
||||||
|
|||||||
@@ -65,6 +65,33 @@ class SiglipEmbedder:
|
|||||||
self._dim = int(arr.shape[-1])
|
self._dim = int(arr.shape[-1])
|
||||||
return arr
|
return arr
|
||||||
|
|
||||||
|
def match_probabilities(
|
||||||
|
self,
|
||||||
|
text_vector: np.ndarray,
|
||||||
|
image_vectors: np.ndarray,
|
||||||
|
) -> np.ndarray:
|
||||||
|
"""Apply SigLIP2's learned logit scale/bias to stored cosine embeddings."""
|
||||||
|
text = torch.as_tensor(text_vector, dtype=torch.float32, device=self.device)
|
||||||
|
images = torch.as_tensor(image_vectors, dtype=torch.float32, device=self.device)
|
||||||
|
|
||||||
|
text = F.normalize(text, p=2, dim=-1)
|
||||||
|
images = F.normalize(images, p=2, dim=-1)
|
||||||
|
cosine = images @ text
|
||||||
|
|
||||||
|
logit_scale = getattr(self.model, "logit_scale", None)
|
||||||
|
logit_bias = getattr(self.model, "logit_bias", None)
|
||||||
|
if logit_scale is None or logit_bias is None:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Loaded model does not expose SigLIP logit_scale/logit_bias; "
|
||||||
|
"cannot calculate model-native match scores."
|
||||||
|
)
|
||||||
|
|
||||||
|
with torch.inference_mode():
|
||||||
|
logits = cosine * logit_scale.float().exp() + logit_bias.float()
|
||||||
|
probabilities = torch.sigmoid(logits)
|
||||||
|
|
||||||
|
return probabilities.detach().cpu().numpy().astype(np.float32)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def dimension(self) -> int | None:
|
def dimension(self) -> int | None:
|
||||||
return self._dim
|
return self._dim
|
||||||
|
|||||||
@@ -54,6 +54,12 @@ class Meili:
|
|||||||
json=documents,
|
json=documents,
|
||||||
))
|
))
|
||||||
|
|
||||||
|
def stats(self) -> dict:
|
||||||
|
self.ensure_index()
|
||||||
|
return self._check(
|
||||||
|
self.session.get(f"{self.url}/indexes/{self.index}/stats")
|
||||||
|
).json()
|
||||||
|
|
||||||
def search_vector(self, vector: Iterable[float], limit: int = 10) -> dict:
|
def search_vector(self, vector: Iterable[float], limit: int = 10) -> dict:
|
||||||
payload = {
|
payload = {
|
||||||
"vector": self._json_vector(vector),
|
"vector": self._json_vector(vector),
|
||||||
|
|||||||
@@ -0,0 +1,39 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import math
|
||||||
|
from types import SimpleNamespace
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import torch
|
||||||
|
|
||||||
|
from scene_search.embedder import SiglipEmbedder
|
||||||
|
|
||||||
|
|
||||||
|
class SiglipMatchScoringTests(unittest.TestCase):
|
||||||
|
def test_match_probabilities_use_model_scale_and_bias(self) -> None:
|
||||||
|
embedder = SiglipEmbedder.__new__(SiglipEmbedder)
|
||||||
|
embedder.device = torch.device("cpu")
|
||||||
|
embedder.model = SimpleNamespace(
|
||||||
|
logit_scale=torch.tensor(math.log(2.0)),
|
||||||
|
logit_bias=torch.tensor(-1.0),
|
||||||
|
)
|
||||||
|
|
||||||
|
text = np.asarray([1.0, 0.0], dtype=np.float32)
|
||||||
|
images = np.asarray(
|
||||||
|
[
|
||||||
|
[1.0, 0.0],
|
||||||
|
[0.0, 1.0],
|
||||||
|
],
|
||||||
|
dtype=np.float32,
|
||||||
|
)
|
||||||
|
|
||||||
|
scores = embedder.match_probabilities(text, images)
|
||||||
|
|
||||||
|
self.assertAlmostEqual(float(scores[0]), 0.7310586, places=6)
|
||||||
|
self.assertAlmostEqual(float(scores[1]), 0.2689414, places=6)
|
||||||
|
self.assertGreater(float(scores[0]), float(scores[1]))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Reference in New Issue
Block a user