diff --git a/scene_search/cli.py b/scene_search/cli.py index cabee64..2316f9f 100644 --- a/scene_search/cli.py +++ b/scene_search/cli.py @@ -165,6 +165,48 @@ def cmd_sync_meili(_: argparse.Namespace) -> None: print("Meilisearch rebuild complete (tasks may continue briefly inside Meili).") + +def cmd_status(_: argparse.Namespace) -> None: + init_db(settings.db_path) + + print("Canonical SQLite") + print("----------------") + with connect(settings.db_path) as con: + media_rows = con.execute( + """ + SELECT m.id, m.title, m.interval_ms, m.model_id, COUNT(f.id) AS frames + FROM media m + LEFT JOIN frames f ON f.media_id = m.id + GROUP BY m.id, m.title, m.interval_ms, m.model_id + ORDER BY m.title, m.id + """ + ).fetchall() + total_frames = sum(int(row["frames"]) for row in media_rows) + + print(f"Media items: {len(media_rows)}") + print(f"Frames: {total_frames}") + if media_rows: + print() + for row in media_rows: + print( + f"- {row['title']}: {row['frames']} frames " + f"({row['interval_ms'] / 1000:g}s interval)" + ) + + print() + print("Meilisearch") + print("-----------") + meili = Meili(settings.meili_url, settings.meili_master_key, settings.meili_index) + try: + stats = meili.stats() + except Exception as exc: + print(f"Status: unavailable ({exc})") + return + + print(f"Index: {settings.meili_index}") + print(f"Documents: {stats.get('numberOfDocuments', 'unknown')}") + print(f"Indexing: {stats.get('isIndexing', 'unknown')}") + def cmd_search(args: argparse.Namespace) -> None: init_db(settings.db_path) embedder = build_embedder() @@ -177,11 +219,54 @@ def cmd_search(args: argparse.Namespace) -> None: print("No hits") return + frame_ids = [hit.get("id") for hit in hits if hit.get("id")] + frame_vectors: dict[str, object] = {} + if frame_ids: + placeholders = ",".join("?" for _ in frame_ids) + with connect(settings.db_path) as con: + rows = con.execute( + f""" + SELECT f.id, f.embedding_f16, m.embedding_dim + FROM frames f + JOIN media m ON m.id = f.media_id + WHERE f.id IN ({placeholders}) + """, + frame_ids, + ).fetchall() + frame_vectors = { + row["id"]: unpack_vector(row["embedding_f16"], row["embedding_dim"]) + for row in rows + } + + scored_vectors = [frame_vectors.get(hit.get("id")) for hit in hits] + if all(item is not None for item in scored_vectors): + import numpy as np + + siglip_scores = embedder.match_probabilities( + vector, + np.stack(scored_vectors), # type: ignore[arg-type] + ) + else: + siglip_scores = [None] * len(hits) + print(f"\nQuery: {args.query!r}\n") - for i, hit in enumerate(hits, 1): - score = hit.get("_rankingScore") - score_text = f" score={score:.4f}" if isinstance(score, (float, int)) else "" - print(f"{i:2d}. {hit.get('title')} @ {format_ts(int(hit.get('timestamp_ms', 0)))}{score_text}") + for i, (hit, siglip_score) in enumerate(zip(hits, siglip_scores, strict=True), 1): + meili_score = hit.get("_rankingScore") + meili_text = ( + f" meili={meili_score:.4f}" + if isinstance(meili_score, (float, int)) + else "" + ) + siglip_text = ( + f" siglip={float(siglip_score):.4f}" + if siglip_score is not None + else "" + ) + print( + f"{i:2d}. {hit.get('title')} @ " + f"{format_ts(int(hit.get('timestamp_ms', 0)))}" + f"{meili_text}{siglip_text}" + ) print(f" {hit.get('thumb_path')}") @@ -202,6 +287,9 @@ def make_parser() -> argparse.ArgumentParser: sync = sub.add_parser("sync-meili", help="Rebuild/feed Meilisearch from canonical SQLite vectors") sync.set_defaults(func=cmd_sync_meili) + status = sub.add_parser("status", help="Show canonical SQLite and Meilisearch index status") + status.set_defaults(func=cmd_status) + search = sub.add_parser("search", help="Semantic scene search") search.add_argument("query") search.add_argument("--limit", type=int, default=10) diff --git a/scene_search/embedder.py b/scene_search/embedder.py index c7ffeeb..e365e96 100644 --- a/scene_search/embedder.py +++ b/scene_search/embedder.py @@ -65,6 +65,33 @@ class SiglipEmbedder: self._dim = int(arr.shape[-1]) return arr + def match_probabilities( + self, + text_vector: np.ndarray, + image_vectors: np.ndarray, + ) -> np.ndarray: + """Apply SigLIP2's learned logit scale/bias to stored cosine embeddings.""" + text = torch.as_tensor(text_vector, dtype=torch.float32, device=self.device) + images = torch.as_tensor(image_vectors, dtype=torch.float32, device=self.device) + + text = F.normalize(text, p=2, dim=-1) + images = F.normalize(images, p=2, dim=-1) + cosine = images @ text + + logit_scale = getattr(self.model, "logit_scale", None) + logit_bias = getattr(self.model, "logit_bias", None) + if logit_scale is None or logit_bias is None: + raise RuntimeError( + "Loaded model does not expose SigLIP logit_scale/logit_bias; " + "cannot calculate model-native match scores." + ) + + with torch.inference_mode(): + logits = cosine * logit_scale.float().exp() + logit_bias.float() + probabilities = torch.sigmoid(logits) + + return probabilities.detach().cpu().numpy().astype(np.float32) + @property def dimension(self) -> int | None: return self._dim diff --git a/scene_search/meili.py b/scene_search/meili.py index f4628d3..35b9806 100644 --- a/scene_search/meili.py +++ b/scene_search/meili.py @@ -54,6 +54,12 @@ class Meili: json=documents, )) + def stats(self) -> dict: + self.ensure_index() + return self._check( + self.session.get(f"{self.url}/indexes/{self.index}/stats") + ).json() + def search_vector(self, vector: Iterable[float], limit: int = 10) -> dict: payload = { "vector": self._json_vector(vector), diff --git a/tests/test_embedder.py b/tests/test_embedder.py new file mode 100644 index 0000000..88d7acb --- /dev/null +++ b/tests/test_embedder.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import math +from types import SimpleNamespace +import unittest + +import numpy as np +import torch + +from scene_search.embedder import SiglipEmbedder + + +class SiglipMatchScoringTests(unittest.TestCase): + def test_match_probabilities_use_model_scale_and_bias(self) -> None: + embedder = SiglipEmbedder.__new__(SiglipEmbedder) + embedder.device = torch.device("cpu") + embedder.model = SimpleNamespace( + logit_scale=torch.tensor(math.log(2.0)), + logit_bias=torch.tensor(-1.0), + ) + + text = np.asarray([1.0, 0.0], dtype=np.float32) + images = np.asarray( + [ + [1.0, 0.0], + [0.0, 1.0], + ], + dtype=np.float32, + ) + + scores = embedder.match_probabilities(text, images) + + self.assertAlmostEqual(float(scores[0]), 0.7310586, places=6) + self.assertAlmostEqual(float(scores[1]), 0.2689414, places=6) + self.assertGreater(float(scores[0]), float(scores[1])) + + +if __name__ == "__main__": + unittest.main()