mirror of
https://github.com/MrSleeps/jellyfin-scene-search.git
synced 2026-11-16 23:11:27 +00:00
308 lines
10 KiB
Python
308 lines
10 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
from pathlib import Path
|
|
|
|
from PIL import Image
|
|
|
|
from .config import settings
|
|
from .database import connect, init_db, pack_vector, unpack_vector
|
|
from .embedder import SiglipEmbedder
|
|
from .frames import extract_frames, media_id_for, sha256_file
|
|
from .meili import Meili
|
|
|
|
|
|
def format_ts(ms: int) -> str:
|
|
total = ms // 1000
|
|
h, rem = divmod(total, 3600)
|
|
m, s = divmod(rem, 60)
|
|
return f"{h:02d}:{m:02d}:{s:02d}"
|
|
|
|
|
|
def build_embedder() -> SiglipEmbedder:
|
|
print(f"Loading {settings.model_id} on {settings.device}...")
|
|
return SiglipEmbedder(settings.model_id, settings.device)
|
|
|
|
|
|
def cmd_init(_: argparse.Namespace) -> None:
|
|
init_db(settings.db_path)
|
|
print(f"SQLite ready: {settings.db_path}")
|
|
|
|
|
|
def cmd_index_file(args: argparse.Namespace) -> None:
|
|
init_db(settings.db_path)
|
|
path = Path(args.path).expanduser().resolve()
|
|
if not path.is_file():
|
|
raise SystemExit(f"Not a file: {path}")
|
|
|
|
interval_ms = int(round(args.interval * 1000))
|
|
stat = path.stat()
|
|
digest = sha256_file(path)
|
|
media_id = media_id_for(path, digest)
|
|
title = args.title or path.stem
|
|
|
|
with connect(settings.db_path) as con:
|
|
row = con.execute("SELECT * FROM media WHERE id = ?", (media_id,)).fetchone()
|
|
if (
|
|
row
|
|
and row["content_sha256"] == digest
|
|
and row["interval_ms"] == interval_ms
|
|
and row["model_id"] == settings.model_id
|
|
and not args.force
|
|
):
|
|
count = con.execute("SELECT COUNT(*) FROM frames WHERE media_id = ?", (media_id,)).fetchone()[0]
|
|
print(f"Already indexed: {title} ({count} frames). Use --force to rebuild.")
|
|
return
|
|
|
|
out_dir = settings.thumbs_dir / media_id
|
|
print(f"Extracting one frame every {args.interval:g}s from {title!r}...")
|
|
frame_paths = extract_frames(path, out_dir, args.interval)
|
|
if not frame_paths:
|
|
raise SystemExit("ffmpeg produced no frames")
|
|
|
|
embedder = build_embedder()
|
|
rows: list[tuple] = []
|
|
dim = None
|
|
|
|
for start in range(0, len(frame_paths), settings.batch_size):
|
|
batch_paths = frame_paths[start:start + settings.batch_size]
|
|
images = []
|
|
try:
|
|
for frame_path in batch_paths:
|
|
images.append(Image.open(frame_path).convert("RGB"))
|
|
vectors = embedder.embed_images(images)
|
|
finally:
|
|
for image in images:
|
|
image.close()
|
|
|
|
dim = int(vectors.shape[1])
|
|
for offset, (frame_path, vector) in enumerate(zip(batch_paths, vectors, strict=True)):
|
|
frame_number = start + offset
|
|
timestamp_ms = frame_number * interval_ms
|
|
frame_id = hashlib.sha256(f"{media_id}:{timestamp_ms}:{settings.model_id}".encode()).hexdigest()[:40]
|
|
rows.append((
|
|
frame_id,
|
|
media_id,
|
|
frame_number,
|
|
timestamp_ms,
|
|
str(frame_path),
|
|
pack_vector(vector),
|
|
))
|
|
print(f"Embedded {min(start + len(batch_paths), len(frame_paths))}/{len(frame_paths)} frames")
|
|
|
|
assert dim is not None
|
|
with connect(settings.db_path) as con:
|
|
con.execute("DELETE FROM media WHERE id = ?", (media_id,))
|
|
con.execute(
|
|
"""
|
|
INSERT INTO media
|
|
(id, source_path, title, content_sha256, size_bytes, mtime_ns,
|
|
interval_ms, model_id, embedding_dim)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
(media_id, str(path), title, digest, stat.st_size, stat.st_mtime_ns,
|
|
interval_ms, settings.model_id, dim),
|
|
)
|
|
con.executemany(
|
|
"""
|
|
INSERT INTO frames
|
|
(id, media_id, frame_number, timestamp_ms, thumb_path, embedding_f16)
|
|
VALUES (?, ?, ?, ?, ?, ?)
|
|
""",
|
|
rows,
|
|
)
|
|
|
|
print(f"Stored {len(rows)} canonical frame embeddings in {settings.db_path}")
|
|
print("Next: python -m scene_search.cli sync-meili")
|
|
|
|
|
|
def cmd_sync_meili(_: argparse.Namespace) -> None:
|
|
init_db(settings.db_path)
|
|
meili = Meili(settings.meili_url, settings.meili_master_key, settings.meili_index)
|
|
|
|
with connect(settings.db_path) as con:
|
|
dims = [r[0] for r in con.execute("SELECT DISTINCT embedding_dim FROM media")]
|
|
if not dims:
|
|
raise SystemExit("Nothing indexed yet")
|
|
if len(dims) != 1:
|
|
raise SystemExit(f"Multiple embedding dimensions found: {dims}. Use separate indexes per model.")
|
|
dim = int(dims[0])
|
|
meili.configure_user_vectors(dim)
|
|
|
|
cursor = con.execute(
|
|
"""
|
|
SELECT f.id, f.media_id, f.timestamp_ms, f.thumb_path, f.embedding_f16,
|
|
m.title, m.source_path, m.embedding_dim, m.model_id
|
|
FROM frames f
|
|
JOIN media m ON m.id = f.media_id
|
|
ORDER BY f.id
|
|
"""
|
|
)
|
|
|
|
total = 0
|
|
while True:
|
|
rows = cursor.fetchmany(100)
|
|
if not rows:
|
|
break
|
|
docs = []
|
|
for row in rows:
|
|
vec = unpack_vector(row["embedding_f16"], row["embedding_dim"])
|
|
docs.append({
|
|
"id": row["id"],
|
|
"media_id": row["media_id"],
|
|
"title": row["title"],
|
|
"source_path": row["source_path"],
|
|
"timestamp_ms": row["timestamp_ms"],
|
|
"thumb_path": row["thumb_path"],
|
|
"model_id": row["model_id"],
|
|
"_vectors": {"siglip2": vec.tolist()},
|
|
})
|
|
meili.add_documents(docs)
|
|
total += len(docs)
|
|
print(f"Queued {total} documents into Meilisearch")
|
|
|
|
print("Meilisearch rebuild complete (tasks may continue briefly inside Meili).")
|
|
|
|
|
|
|
|
def cmd_status(_: argparse.Namespace) -> None:
|
|
init_db(settings.db_path)
|
|
|
|
print("Canonical SQLite")
|
|
print("----------------")
|
|
with connect(settings.db_path) as con:
|
|
media_rows = con.execute(
|
|
"""
|
|
SELECT m.id, m.title, m.interval_ms, m.model_id, COUNT(f.id) AS frames
|
|
FROM media m
|
|
LEFT JOIN frames f ON f.media_id = m.id
|
|
GROUP BY m.id, m.title, m.interval_ms, m.model_id
|
|
ORDER BY m.title, m.id
|
|
"""
|
|
).fetchall()
|
|
total_frames = sum(int(row["frames"]) for row in media_rows)
|
|
|
|
print(f"Media items: {len(media_rows)}")
|
|
print(f"Frames: {total_frames}")
|
|
if media_rows:
|
|
print()
|
|
for row in media_rows:
|
|
print(
|
|
f"- {row['title']}: {row['frames']} frames "
|
|
f"({row['interval_ms'] / 1000:g}s interval)"
|
|
)
|
|
|
|
print()
|
|
print("Meilisearch")
|
|
print("-----------")
|
|
meili = Meili(settings.meili_url, settings.meili_master_key, settings.meili_index)
|
|
try:
|
|
stats = meili.stats()
|
|
except Exception as exc:
|
|
print(f"Status: unavailable ({exc})")
|
|
return
|
|
|
|
print(f"Index: {settings.meili_index}")
|
|
print(f"Documents: {stats.get('numberOfDocuments', 'unknown')}")
|
|
print(f"Indexing: {stats.get('isIndexing', 'unknown')}")
|
|
|
|
def cmd_search(args: argparse.Namespace) -> None:
|
|
init_db(settings.db_path)
|
|
embedder = build_embedder()
|
|
vector = embedder.embed_text(args.query)
|
|
meili = Meili(settings.meili_url, settings.meili_master_key, settings.meili_index)
|
|
result = meili.search_vector(vector, args.limit)
|
|
|
|
hits = result.get("hits", [])
|
|
if not hits:
|
|
print("No hits")
|
|
return
|
|
|
|
frame_ids = [hit.get("id") for hit in hits if hit.get("id")]
|
|
frame_vectors: dict[str, object] = {}
|
|
if frame_ids:
|
|
placeholders = ",".join("?" for _ in frame_ids)
|
|
with connect(settings.db_path) as con:
|
|
rows = con.execute(
|
|
f"""
|
|
SELECT f.id, f.embedding_f16, m.embedding_dim
|
|
FROM frames f
|
|
JOIN media m ON m.id = f.media_id
|
|
WHERE f.id IN ({placeholders})
|
|
""",
|
|
frame_ids,
|
|
).fetchall()
|
|
frame_vectors = {
|
|
row["id"]: unpack_vector(row["embedding_f16"], row["embedding_dim"])
|
|
for row in rows
|
|
}
|
|
|
|
scored_vectors = [frame_vectors.get(hit.get("id")) for hit in hits]
|
|
if all(item is not None for item in scored_vectors):
|
|
import numpy as np
|
|
|
|
siglip_scores = embedder.match_probabilities(
|
|
vector,
|
|
np.stack(scored_vectors), # type: ignore[arg-type]
|
|
)
|
|
else:
|
|
siglip_scores = [None] * len(hits)
|
|
|
|
print(f"\nQuery: {args.query!r}\n")
|
|
for i, (hit, siglip_score) in enumerate(zip(hits, siglip_scores, strict=True), 1):
|
|
meili_score = hit.get("_rankingScore")
|
|
meili_text = (
|
|
f" meili={meili_score:.4f}"
|
|
if isinstance(meili_score, (float, int))
|
|
else ""
|
|
)
|
|
siglip_text = (
|
|
f" siglip={float(siglip_score):.4f}"
|
|
if siglip_score is not None
|
|
else ""
|
|
)
|
|
print(
|
|
f"{i:2d}. {hit.get('title')} @ "
|
|
f"{format_ts(int(hit.get('timestamp_ms', 0)))}"
|
|
f"{meili_text}{siglip_text}"
|
|
)
|
|
print(f" {hit.get('thumb_path')}")
|
|
|
|
|
|
def make_parser() -> argparse.ArgumentParser:
|
|
p = argparse.ArgumentParser(prog="jellyfin-scene-search")
|
|
sub = p.add_subparsers(dest="command", required=True)
|
|
|
|
init = sub.add_parser("init", help="Create the persistent SQLite database")
|
|
init.set_defaults(func=cmd_init)
|
|
|
|
idx = sub.add_parser("index-file", help="Index a local test video")
|
|
idx.add_argument("path")
|
|
idx.add_argument("--title")
|
|
idx.add_argument("--interval", type=float, default=2.0, help="Seconds between test frames (default: 2)")
|
|
idx.add_argument("--force", action="store_true")
|
|
idx.set_defaults(func=cmd_index_file)
|
|
|
|
sync = sub.add_parser("sync-meili", help="Rebuild/feed Meilisearch from canonical SQLite vectors")
|
|
sync.set_defaults(func=cmd_sync_meili)
|
|
|
|
status = sub.add_parser("status", help="Show canonical SQLite and Meilisearch index status")
|
|
status.set_defaults(func=cmd_status)
|
|
|
|
search = sub.add_parser("search", help="Semantic scene search")
|
|
search.add_argument("query")
|
|
search.add_argument("--limit", type=int, default=10)
|
|
search.set_defaults(func=cmd_search)
|
|
|
|
return p
|
|
|
|
|
|
def main() -> None:
|
|
args = make_parser().parse_args()
|
|
args.func(args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|