Merge pull request #2 from MrSleeps/feature/search-confidence-diagnostics

Add confidence diagnostics for semantic scene search
This commit is contained in:
Sleeps
2026-10-06 13:51:17 +01:00
committed by GitHub
4 changed files with 164 additions and 4 deletions
+92 -4
View File
@@ -165,6 +165,48 @@ def cmd_sync_meili(_: argparse.Namespace) -> None:
print("Meilisearch rebuild complete (tasks may continue briefly inside Meili).")
def cmd_status(_: argparse.Namespace) -> None:
init_db(settings.db_path)
print("Canonical SQLite")
print("----------------")
with connect(settings.db_path) as con:
media_rows = con.execute(
"""
SELECT m.id, m.title, m.interval_ms, m.model_id, COUNT(f.id) AS frames
FROM media m
LEFT JOIN frames f ON f.media_id = m.id
GROUP BY m.id, m.title, m.interval_ms, m.model_id
ORDER BY m.title, m.id
"""
).fetchall()
total_frames = sum(int(row["frames"]) for row in media_rows)
print(f"Media items: {len(media_rows)}")
print(f"Frames: {total_frames}")
if media_rows:
print()
for row in media_rows:
print(
f"- {row['title']}: {row['frames']} frames "
f"({row['interval_ms'] / 1000:g}s interval)"
)
print()
print("Meilisearch")
print("-----------")
meili = Meili(settings.meili_url, settings.meili_master_key, settings.meili_index)
try:
stats = meili.stats()
except Exception as exc:
print(f"Status: unavailable ({exc})")
return
print(f"Index: {settings.meili_index}")
print(f"Documents: {stats.get('numberOfDocuments', 'unknown')}")
print(f"Indexing: {stats.get('isIndexing', 'unknown')}")
def cmd_search(args: argparse.Namespace) -> None:
init_db(settings.db_path)
embedder = build_embedder()
@@ -177,11 +219,54 @@ def cmd_search(args: argparse.Namespace) -> None:
print("No hits")
return
frame_ids = [hit.get("id") for hit in hits if hit.get("id")]
frame_vectors: dict[str, object] = {}
if frame_ids:
placeholders = ",".join("?" for _ in frame_ids)
with connect(settings.db_path) as con:
rows = con.execute(
f"""
SELECT f.id, f.embedding_f16, m.embedding_dim
FROM frames f
JOIN media m ON m.id = f.media_id
WHERE f.id IN ({placeholders})
""",
frame_ids,
).fetchall()
frame_vectors = {
row["id"]: unpack_vector(row["embedding_f16"], row["embedding_dim"])
for row in rows
}
scored_vectors = [frame_vectors.get(hit.get("id")) for hit in hits]
if all(item is not None for item in scored_vectors):
import numpy as np
siglip_scores = embedder.match_probabilities(
vector,
np.stack(scored_vectors), # type: ignore[arg-type]
)
else:
siglip_scores = [None] * len(hits)
print(f"\nQuery: {args.query!r}\n")
for i, hit in enumerate(hits, 1):
score = hit.get("_rankingScore")
score_text = f" score={score:.4f}" if isinstance(score, (float, int)) else ""
print(f"{i:2d}. {hit.get('title')} @ {format_ts(int(hit.get('timestamp_ms', 0)))}{score_text}")
for i, (hit, siglip_score) in enumerate(zip(hits, siglip_scores, strict=True), 1):
meili_score = hit.get("_rankingScore")
meili_text = (
f" meili={meili_score:.4f}"
if isinstance(meili_score, (float, int))
else ""
)
siglip_text = (
f" siglip={float(siglip_score):.4f}"
if siglip_score is not None
else ""
)
print(
f"{i:2d}. {hit.get('title')} @ "
f"{format_ts(int(hit.get('timestamp_ms', 0)))}"
f"{meili_text}{siglip_text}"
)
print(f" {hit.get('thumb_path')}")
@@ -202,6 +287,9 @@ def make_parser() -> argparse.ArgumentParser:
sync = sub.add_parser("sync-meili", help="Rebuild/feed Meilisearch from canonical SQLite vectors")
sync.set_defaults(func=cmd_sync_meili)
status = sub.add_parser("status", help="Show canonical SQLite and Meilisearch index status")
status.set_defaults(func=cmd_status)
search = sub.add_parser("search", help="Semantic scene search")
search.add_argument("query")
search.add_argument("--limit", type=int, default=10)
+27
View File
@@ -65,6 +65,33 @@ class SiglipEmbedder:
self._dim = int(arr.shape[-1])
return arr
def match_probabilities(
self,
text_vector: np.ndarray,
image_vectors: np.ndarray,
) -> np.ndarray:
"""Apply SigLIP2's learned logit scale/bias to stored cosine embeddings."""
text = torch.as_tensor(text_vector, dtype=torch.float32, device=self.device)
images = torch.as_tensor(image_vectors, dtype=torch.float32, device=self.device)
text = F.normalize(text, p=2, dim=-1)
images = F.normalize(images, p=2, dim=-1)
cosine = images @ text
logit_scale = getattr(self.model, "logit_scale", None)
logit_bias = getattr(self.model, "logit_bias", None)
if logit_scale is None or logit_bias is None:
raise RuntimeError(
"Loaded model does not expose SigLIP logit_scale/logit_bias; "
"cannot calculate model-native match scores."
)
with torch.inference_mode():
logits = cosine * logit_scale.float().exp() + logit_bias.float()
probabilities = torch.sigmoid(logits)
return probabilities.detach().cpu().numpy().astype(np.float32)
@property
def dimension(self) -> int | None:
return self._dim
+6
View File
@@ -54,6 +54,12 @@ class Meili:
json=documents,
))
def stats(self) -> dict:
self.ensure_index()
return self._check(
self.session.get(f"{self.url}/indexes/{self.index}/stats")
).json()
def search_vector(self, vector: Iterable[float], limit: int = 10) -> dict:
payload = {
"vector": self._json_vector(vector),
+39
View File
@@ -0,0 +1,39 @@
from __future__ import annotations
import math
from types import SimpleNamespace
import unittest
import numpy as np
import torch
from scene_search.embedder import SiglipEmbedder
class SiglipMatchScoringTests(unittest.TestCase):
def test_match_probabilities_use_model_scale_and_bias(self) -> None:
embedder = SiglipEmbedder.__new__(SiglipEmbedder)
embedder.device = torch.device("cpu")
embedder.model = SimpleNamespace(
logit_scale=torch.tensor(math.log(2.0)),
logit_bias=torch.tensor(-1.0),
)
text = np.asarray([1.0, 0.0], dtype=np.float32)
images = np.asarray(
[
[1.0, 0.0],
[0.0, 1.0],
],
dtype=np.float32,
)
scores = embedder.match_probabilities(text, images)
self.assertAlmostEqual(float(scores[0]), 0.7310586, places=6)
self.assertAlmostEqual(float(scores[1]), 0.2689414, places=6)
self.assertGreater(float(scores[0]), float(scores[1]))
if __name__ == "__main__":
unittest.main()