mirror of
https://github.com/MrSleeps/jellyfin-scene-search.git
synced 2026-11-16 23:11:27 +00:00
Merge pull request #2 from MrSleeps/feature/search-confidence-diagnostics
Add confidence diagnostics for semantic scene search
This commit is contained in:
+92
-4
@@ -165,6 +165,48 @@ def cmd_sync_meili(_: argparse.Namespace) -> None:
|
||||
print("Meilisearch rebuild complete (tasks may continue briefly inside Meili).")
|
||||
|
||||
|
||||
|
||||
def cmd_status(_: argparse.Namespace) -> None:
|
||||
init_db(settings.db_path)
|
||||
|
||||
print("Canonical SQLite")
|
||||
print("----------------")
|
||||
with connect(settings.db_path) as con:
|
||||
media_rows = con.execute(
|
||||
"""
|
||||
SELECT m.id, m.title, m.interval_ms, m.model_id, COUNT(f.id) AS frames
|
||||
FROM media m
|
||||
LEFT JOIN frames f ON f.media_id = m.id
|
||||
GROUP BY m.id, m.title, m.interval_ms, m.model_id
|
||||
ORDER BY m.title, m.id
|
||||
"""
|
||||
).fetchall()
|
||||
total_frames = sum(int(row["frames"]) for row in media_rows)
|
||||
|
||||
print(f"Media items: {len(media_rows)}")
|
||||
print(f"Frames: {total_frames}")
|
||||
if media_rows:
|
||||
print()
|
||||
for row in media_rows:
|
||||
print(
|
||||
f"- {row['title']}: {row['frames']} frames "
|
||||
f"({row['interval_ms'] / 1000:g}s interval)"
|
||||
)
|
||||
|
||||
print()
|
||||
print("Meilisearch")
|
||||
print("-----------")
|
||||
meili = Meili(settings.meili_url, settings.meili_master_key, settings.meili_index)
|
||||
try:
|
||||
stats = meili.stats()
|
||||
except Exception as exc:
|
||||
print(f"Status: unavailable ({exc})")
|
||||
return
|
||||
|
||||
print(f"Index: {settings.meili_index}")
|
||||
print(f"Documents: {stats.get('numberOfDocuments', 'unknown')}")
|
||||
print(f"Indexing: {stats.get('isIndexing', 'unknown')}")
|
||||
|
||||
def cmd_search(args: argparse.Namespace) -> None:
|
||||
init_db(settings.db_path)
|
||||
embedder = build_embedder()
|
||||
@@ -177,11 +219,54 @@ def cmd_search(args: argparse.Namespace) -> None:
|
||||
print("No hits")
|
||||
return
|
||||
|
||||
frame_ids = [hit.get("id") for hit in hits if hit.get("id")]
|
||||
frame_vectors: dict[str, object] = {}
|
||||
if frame_ids:
|
||||
placeholders = ",".join("?" for _ in frame_ids)
|
||||
with connect(settings.db_path) as con:
|
||||
rows = con.execute(
|
||||
f"""
|
||||
SELECT f.id, f.embedding_f16, m.embedding_dim
|
||||
FROM frames f
|
||||
JOIN media m ON m.id = f.media_id
|
||||
WHERE f.id IN ({placeholders})
|
||||
""",
|
||||
frame_ids,
|
||||
).fetchall()
|
||||
frame_vectors = {
|
||||
row["id"]: unpack_vector(row["embedding_f16"], row["embedding_dim"])
|
||||
for row in rows
|
||||
}
|
||||
|
||||
scored_vectors = [frame_vectors.get(hit.get("id")) for hit in hits]
|
||||
if all(item is not None for item in scored_vectors):
|
||||
import numpy as np
|
||||
|
||||
siglip_scores = embedder.match_probabilities(
|
||||
vector,
|
||||
np.stack(scored_vectors), # type: ignore[arg-type]
|
||||
)
|
||||
else:
|
||||
siglip_scores = [None] * len(hits)
|
||||
|
||||
print(f"\nQuery: {args.query!r}\n")
|
||||
for i, hit in enumerate(hits, 1):
|
||||
score = hit.get("_rankingScore")
|
||||
score_text = f" score={score:.4f}" if isinstance(score, (float, int)) else ""
|
||||
print(f"{i:2d}. {hit.get('title')} @ {format_ts(int(hit.get('timestamp_ms', 0)))}{score_text}")
|
||||
for i, (hit, siglip_score) in enumerate(zip(hits, siglip_scores, strict=True), 1):
|
||||
meili_score = hit.get("_rankingScore")
|
||||
meili_text = (
|
||||
f" meili={meili_score:.4f}"
|
||||
if isinstance(meili_score, (float, int))
|
||||
else ""
|
||||
)
|
||||
siglip_text = (
|
||||
f" siglip={float(siglip_score):.4f}"
|
||||
if siglip_score is not None
|
||||
else ""
|
||||
)
|
||||
print(
|
||||
f"{i:2d}. {hit.get('title')} @ "
|
||||
f"{format_ts(int(hit.get('timestamp_ms', 0)))}"
|
||||
f"{meili_text}{siglip_text}"
|
||||
)
|
||||
print(f" {hit.get('thumb_path')}")
|
||||
|
||||
|
||||
@@ -202,6 +287,9 @@ def make_parser() -> argparse.ArgumentParser:
|
||||
sync = sub.add_parser("sync-meili", help="Rebuild/feed Meilisearch from canonical SQLite vectors")
|
||||
sync.set_defaults(func=cmd_sync_meili)
|
||||
|
||||
status = sub.add_parser("status", help="Show canonical SQLite and Meilisearch index status")
|
||||
status.set_defaults(func=cmd_status)
|
||||
|
||||
search = sub.add_parser("search", help="Semantic scene search")
|
||||
search.add_argument("query")
|
||||
search.add_argument("--limit", type=int, default=10)
|
||||
|
||||
@@ -65,6 +65,33 @@ class SiglipEmbedder:
|
||||
self._dim = int(arr.shape[-1])
|
||||
return arr
|
||||
|
||||
def match_probabilities(
|
||||
self,
|
||||
text_vector: np.ndarray,
|
||||
image_vectors: np.ndarray,
|
||||
) -> np.ndarray:
|
||||
"""Apply SigLIP2's learned logit scale/bias to stored cosine embeddings."""
|
||||
text = torch.as_tensor(text_vector, dtype=torch.float32, device=self.device)
|
||||
images = torch.as_tensor(image_vectors, dtype=torch.float32, device=self.device)
|
||||
|
||||
text = F.normalize(text, p=2, dim=-1)
|
||||
images = F.normalize(images, p=2, dim=-1)
|
||||
cosine = images @ text
|
||||
|
||||
logit_scale = getattr(self.model, "logit_scale", None)
|
||||
logit_bias = getattr(self.model, "logit_bias", None)
|
||||
if logit_scale is None or logit_bias is None:
|
||||
raise RuntimeError(
|
||||
"Loaded model does not expose SigLIP logit_scale/logit_bias; "
|
||||
"cannot calculate model-native match scores."
|
||||
)
|
||||
|
||||
with torch.inference_mode():
|
||||
logits = cosine * logit_scale.float().exp() + logit_bias.float()
|
||||
probabilities = torch.sigmoid(logits)
|
||||
|
||||
return probabilities.detach().cpu().numpy().astype(np.float32)
|
||||
|
||||
@property
|
||||
def dimension(self) -> int | None:
|
||||
return self._dim
|
||||
|
||||
@@ -54,6 +54,12 @@ class Meili:
|
||||
json=documents,
|
||||
))
|
||||
|
||||
def stats(self) -> dict:
|
||||
self.ensure_index()
|
||||
return self._check(
|
||||
self.session.get(f"{self.url}/indexes/{self.index}/stats")
|
||||
).json()
|
||||
|
||||
def search_vector(self, vector: Iterable[float], limit: int = 10) -> dict:
|
||||
payload = {
|
||||
"vector": self._json_vector(vector),
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from scene_search.embedder import SiglipEmbedder
|
||||
|
||||
|
||||
class SiglipMatchScoringTests(unittest.TestCase):
|
||||
def test_match_probabilities_use_model_scale_and_bias(self) -> None:
|
||||
embedder = SiglipEmbedder.__new__(SiglipEmbedder)
|
||||
embedder.device = torch.device("cpu")
|
||||
embedder.model = SimpleNamespace(
|
||||
logit_scale=torch.tensor(math.log(2.0)),
|
||||
logit_bias=torch.tensor(-1.0),
|
||||
)
|
||||
|
||||
text = np.asarray([1.0, 0.0], dtype=np.float32)
|
||||
images = np.asarray(
|
||||
[
|
||||
[1.0, 0.0],
|
||||
[0.0, 1.0],
|
||||
],
|
||||
dtype=np.float32,
|
||||
)
|
||||
|
||||
scores = embedder.match_probabilities(text, images)
|
||||
|
||||
self.assertAlmostEqual(float(scores[0]), 0.7310586, places=6)
|
||||
self.assertAlmostEqual(float(scores[1]), 0.2689414, places=6)
|
||||
self.assertGreater(float(scores[0]), float(scores[1]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user