Files
jellyfin-scene-search/scene_search/cli.py
T

665 lines
23 KiB
Python

from __future__ import annotations
import argparse
import hashlib
from pathlib import Path
from PIL import Image
from .captioner import (
OllamaCaptioner,
ensure_caption_prompt_file,
ensure_specialist_prompt_file,
)
from .config import settings
from .database import connect, init_db, pack_vector, unpack_vector
from .embedder import SiglipEmbedder
from .frames import extract_frames, media_id_for, sha256_file
from .meili import Meili
from .routing import ExplicitContentRouter, apply_context_routing
def format_ts(ms: int) -> str:
total = ms // 1000
h, rem = divmod(total, 3600)
m, s = divmod(rem, 60)
return f"{h:02d}:{m:02d}:{s:02d}"
def build_embedder() -> SiglipEmbedder:
print(f"Loading {settings.model_id} on {settings.device}...")
return SiglipEmbedder(settings.model_id, settings.device)
def cmd_init(_: argparse.Namespace) -> None:
init_db(settings.db_path)
ensure_caption_prompt_file(settings.caption_prompt_file)
ensure_specialist_prompt_file(settings.caption_specialist_prompt_file)
print(f"SQLite ready: {settings.db_path}")
print(f"General caption prompt: {settings.caption_prompt_file}")
print(f"Specialist caption prompt: {settings.caption_specialist_prompt_file}")
def cmd_index_file(args: argparse.Namespace) -> None:
init_db(settings.db_path)
path = Path(args.path).expanduser().resolve()
if not path.is_file():
raise SystemExit(f"Not a file: {path}")
interval_ms = int(round(args.interval * 1000))
stat = path.stat()
digest = sha256_file(path)
media_id = media_id_for(path, digest)
title = args.title or path.stem
with connect(settings.db_path) as con:
row = con.execute("SELECT * FROM media WHERE id = ?", (media_id,)).fetchone()
if (
row
and row["content_sha256"] == digest
and row["interval_ms"] == interval_ms
and row["model_id"] == settings.model_id
and not args.force
):
count = con.execute("SELECT COUNT(*) FROM frames WHERE media_id = ?", (media_id,)).fetchone()[0]
print(f"Already indexed: {title} ({count} frames). Use --force to rebuild.")
return
out_dir = settings.thumbs_dir / media_id
print(f"Extracting one frame every {args.interval:g}s from {title!r}...")
frame_paths = extract_frames(path, out_dir, args.interval)
if not frame_paths:
raise SystemExit("ffmpeg produced no frames")
embedder = build_embedder()
rows: list[tuple] = []
dim = None
for start in range(0, len(frame_paths), settings.batch_size):
batch_paths = frame_paths[start:start + settings.batch_size]
images = []
try:
for frame_path in batch_paths:
images.append(Image.open(frame_path).convert("RGB"))
vectors = embedder.embed_images(images)
finally:
for image in images:
image.close()
dim = int(vectors.shape[1])
for offset, (frame_path, vector) in enumerate(zip(batch_paths, vectors, strict=True)):
frame_number = start + offset
timestamp_ms = frame_number * interval_ms
frame_id = hashlib.sha256(f"{media_id}:{timestamp_ms}:{settings.model_id}".encode()).hexdigest()[:40]
rows.append((
frame_id,
media_id,
frame_number,
timestamp_ms,
str(frame_path),
pack_vector(vector),
))
print(f"Embedded {min(start + len(batch_paths), len(frame_paths))}/{len(frame_paths)} frames")
assert dim is not None
with connect(settings.db_path) as con:
con.execute("DELETE FROM media WHERE id = ?", (media_id,))
con.execute(
"""
INSERT INTO media
(id, source_path, title, content_sha256, size_bytes, mtime_ns,
interval_ms, model_id, embedding_dim)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(media_id, str(path), title, digest, stat.st_size, stat.st_mtime_ns,
interval_ms, settings.model_id, dim),
)
con.executemany(
"""
INSERT INTO frames
(id, media_id, frame_number, timestamp_ms, thumb_path, embedding_f16)
VALUES (?, ?, ?, ?, ?, ?)
""",
rows,
)
print(f"Stored {len(rows)} canonical frame embeddings in {settings.db_path}")
print("Next: python -m scene_search.cli sync-meili")
def cmd_sync_meili(_: argparse.Namespace) -> None:
init_db(settings.db_path)
meili = Meili(settings.meili_url, settings.meili_master_key, settings.meili_index)
with connect(settings.db_path) as con:
dims = [r[0] for r in con.execute("SELECT DISTINCT embedding_dim FROM media")]
if not dims:
raise SystemExit("Nothing indexed yet")
if len(dims) != 1:
raise SystemExit(f"Multiple embedding dimensions found: {dims}. Use separate indexes per model.")
dim = int(dims[0])
meili.configure_user_vectors(dim)
cursor = con.execute(
"""
SELECT f.id, f.media_id, f.timestamp_ms, f.thumb_path, f.embedding_f16,
f.caption, f.caption_model, f.captioned_at,
f.caption_route, f.caption_route_reason, f.route_score,
f.route_model, f.route_policy, f.routed_at,
m.title, m.source_path, m.embedding_dim, m.model_id
FROM frames f
JOIN media m ON m.id = f.media_id
ORDER BY f.id
"""
)
total = 0
while True:
rows = cursor.fetchmany(100)
if not rows:
break
docs = []
for row in rows:
vec = unpack_vector(row["embedding_f16"], row["embedding_dim"])
docs.append({
"id": row["id"],
"media_id": row["media_id"],
"title": row["title"],
"source_path": row["source_path"],
"timestamp_ms": row["timestamp_ms"],
"thumb_path": row["thumb_path"],
"model_id": row["model_id"],
"caption": row["caption"],
"caption_model": row["caption_model"],
"captioned_at": row["captioned_at"],
"caption_route": row["caption_route"],
"caption_route_reason": row["caption_route_reason"],
"route_score": row["route_score"],
"route_model": row["route_model"],
"route_policy": row["route_policy"],
"routed_at": row["routed_at"],
"_vectors": {"siglip2": vec.tolist()},
})
meili.add_documents(docs)
total += len(docs)
print(f"Queued {total} documents into Meilisearch")
print("Meilisearch rebuild complete (tasks may continue briefly inside Meili).")
def _route_policy() -> str:
if not settings.caption_router_enabled:
return f"disabled|general={settings.caption_general_model}"
return (
f"model={settings.caption_router_model}|"
f"threshold={settings.caption_router_threshold:.6f}|"
f"context={settings.caption_router_context_seconds:.3f}"
)
def _select_caption_rows(
args: argparse.Namespace,
*,
only_needing_caption: bool,
) -> list:
conditions = []
params: list[object] = []
if only_needing_caption and not getattr(args, "force", False):
expected_models = [settings.caption_general_model]
if settings.caption_router_enabled:
expected_models.append(settings.caption_specialist_model)
placeholders = ",".join("?" for _ in expected_models)
conditions.append(
f"(f.caption IS NULL OR f.caption_model IS NULL "
f"OR f.caption_model NOT IN ({placeholders}) "
f"OR f.route_policy IS NULL OR f.route_policy != ?)"
)
params.extend(expected_models)
params.append(_route_policy())
if getattr(args, "media_id", None):
conditions.append("f.media_id = ?")
params.append(args.media_id)
if getattr(args, "title", None):
conditions.append("m.title = ?")
params.append(args.title)
where = f"WHERE {' AND '.join(conditions)}" if conditions else ""
limit = ""
if getattr(args, "limit", None) is not None:
limit = " LIMIT ?"
params.append(args.limit)
with connect(settings.db_path) as con:
return con.execute(
f"""
SELECT f.id, f.media_id, f.timestamp_ms, f.thumb_path, f.caption,
f.caption_model, f.caption_route, f.route_score, f.route_policy,
m.title
FROM frames f
JOIN media m ON m.id = f.media_id
{where}
ORDER BY m.title, f.timestamp_ms
{limit}
""",
params,
).fetchall()
def _route_rows(rows: list, *, verbose: bool = False) -> tuple[list[tuple], int]:
valid_rows = []
missing = 0
for row in rows:
image_path = Path(row["thumb_path"])
if not image_path.is_file():
missing += 1
print(
f"SKIP {row['title']} @ {format_ts(int(row['timestamp_ms']))}: "
f"missing thumbnail {image_path}"
)
continue
valid_rows.append(row)
if not valid_rows:
return [], missing
policy = _route_policy()
if settings.caption_router_enabled:
print(
f"Routing {len(valid_rows)} frame(s) with "
f"{settings.caption_router_model} on {settings.caption_router_device}..."
)
router = ExplicitContentRouter(
settings.caption_router_model,
settings.caption_router_device,
settings.caption_router_batch_size,
)
score_values = router.score_paths(
[Path(row["thumb_path"]) for row in valid_rows]
)
scores = {
str(row["id"]): score
for row, score in zip(valid_rows, score_values, strict=True)
}
route_input = [
{
"id": str(row["id"]),
"media_id": str(row["media_id"]),
"timestamp_ms": int(row["timestamp_ms"]),
}
for row in valid_rows
]
decisions = apply_context_routing(
route_input,
scores,
settings.caption_router_threshold,
int(round(settings.caption_router_context_seconds * 1000)),
)
route_model = settings.caption_router_model
else:
scores = {str(row["id"]): 0.0 for row in valid_rows}
decisions = {
str(row["id"]): ("general", "disabled")
for row in valid_rows
}
route_model = "disabled"
routed: list[tuple] = []
with connect(settings.db_path) as con:
for row in valid_rows:
frame_id = str(row["id"])
route, reason = decisions[frame_id]
score = scores[frame_id]
con.execute(
"""
UPDATE frames
SET caption_route = ?, caption_route_reason = ?, route_score = ?,
route_model = ?, route_policy = ?, routed_at = CURRENT_TIMESTAMP
WHERE id = ?
""",
(route, reason, score, route_model, policy, frame_id),
)
routed.append((row, route, score, reason))
if verbose:
print(
f"{row['title']} @ {format_ts(int(row['timestamp_ms']))}: "
f"score={score:.4f} -> {route} ({reason})"
)
general_count = sum(1 for _, route, _, _ in routed if route == "general")
specialist_count = len(routed) - general_count
if settings.caption_router_enabled:
print(
f"Routing complete: {general_count} general, "
f"{specialist_count} specialist "
f"(threshold {settings.caption_router_threshold:.2f}, "
f"context ±{settings.caption_router_context_seconds:g}s)"
)
else:
print(f"Routing disabled: {general_count} frame(s) -> general model")
return routed, missing
def _caption_queue(
label: str,
routed_rows: list[tuple],
model: str,
prompt: str,
) -> tuple[int, int]:
if not routed_rows:
return 0, 0
print(f"\n{label}: {len(routed_rows)} frame(s) with {model}")
captioner = OllamaCaptioner(
settings.ollama_url,
model,
settings.caption_timeout_seconds,
prompt,
settings.caption_max_tokens,
)
completed = 0
failed = 0
try:
for row, route, score, reason in routed_rows:
image_path = Path(row["thumb_path"])
try:
caption = captioner.caption(image_path)
except Exception as exc:
failed += 1
print(
f"FAIL {row['title']} @ "
f"{format_ts(int(row['timestamp_ms']))}: {exc}"
)
continue
with connect(settings.db_path) as con:
con.execute(
"""
UPDATE frames
SET caption = ?, caption_model = ?,
captioned_at = CURRENT_TIMESTAMP
WHERE id = ?
""",
(caption, model, row["id"]),
)
completed += 1
print(
f"{completed:4d}/{len(routed_rows)} {row['title']} @ "
f"{format_ts(int(row['timestamp_ms']))} "
f"[{route} {score:.3f}/{reason}]: {caption}"
)
finally:
try:
captioner.unload()
print(f"Unloaded {model} from Ollama.")
except Exception as exc:
print(f"WARN could not unload {model}: {exc}")
return completed, failed
def cmd_route(args: argparse.Namespace) -> None:
init_db(settings.db_path)
rows = _select_caption_rows(args, only_needing_caption=False)
if not rows:
print("No matching frames.")
return
_, missing = _route_rows(rows, verbose=True)
if missing:
print(f"Routing skipped {missing} frame(s) with missing thumbnails.")
def cmd_caption(args: argparse.Namespace) -> None:
init_db(settings.db_path)
rows = _select_caption_rows(args, only_needing_caption=True)
if not rows:
print("No frames need captioning.")
return
routed, missing = _route_rows(rows)
if not routed:
print(f"No captionable frames ({missing} missing thumbnails).")
return
general_prompt = ensure_caption_prompt_file(settings.caption_prompt_file)
specialist_prompt = ensure_specialist_prompt_file(
settings.caption_specialist_prompt_file
)
general_rows = [
item
for item in routed
if item[1] == "general"
and (
args.force
or item[0]["caption"] is None
or item[0]["caption_model"] != settings.caption_general_model
)
]
specialist_rows = [
item
for item in routed
if item[1] == "specialist"
and (
args.force
or item[0]["caption"] is None
or item[0]["caption_model"] != settings.caption_specialist_model
)
]
retained = len(routed) - len(general_rows) - len(specialist_rows)
if retained:
print(
f"Keeping {retained} existing caption(s) because they already use "
f"the model selected by the router."
)
print(f"General prompt: {settings.caption_prompt_file}")
print(f"Specialist prompt: {settings.caption_specialist_prompt_file}")
print(f"Ollama: {settings.ollama_url}")
general_done, general_failed = _caption_queue(
"General queue",
general_rows,
settings.caption_general_model,
general_prompt,
)
specialist_done, specialist_failed = _caption_queue(
"Specialist queue",
specialist_rows,
settings.caption_specialist_model,
specialist_prompt,
)
completed = general_done + specialist_done
failed = general_failed + specialist_failed + missing
print(f"\nCaptioning complete: {completed} saved, {failed} failed/skipped.")
if args.sync:
cmd_sync_meili(args)
elif completed:
print("Next: python -m scene_search.cli sync-meili")
def cmd_status(_: argparse.Namespace) -> None:
init_db(settings.db_path)
print("Canonical SQLite")
print("----------------")
with connect(settings.db_path) as con:
media_rows = con.execute(
"""
SELECT m.id, m.title, m.interval_ms, m.model_id,
COUNT(f.id) AS frames,
SUM(CASE WHEN f.caption IS NOT NULL THEN 1 ELSE 0 END) AS captioned,
SUM(CASE WHEN f.caption_route = 'specialist' THEN 1 ELSE 0 END)
AS specialist_routed
FROM media m
LEFT JOIN frames f ON f.media_id = m.id
GROUP BY m.id, m.title, m.interval_ms, m.model_id
ORDER BY m.title, m.id
"""
).fetchall()
total_frames = sum(int(row["frames"]) for row in media_rows)
print(f"Media items: {len(media_rows)}")
print(f"Frames: {total_frames}")
if media_rows:
print()
for row in media_rows:
print(
f"- {row['title']}: {row['frames']} frames, "
f"{row['captioned']} captioned, "
f"{row['specialist_routed']} specialist-routed "
f"({row['interval_ms'] / 1000:g}s interval)"
)
print()
print("Meilisearch")
print("-----------")
meili = Meili(settings.meili_url, settings.meili_master_key, settings.meili_index)
try:
stats = meili.stats()
except Exception as exc:
print(f"Status: unavailable ({exc})")
return
print(f"Index: {settings.meili_index}")
print(f"Documents: {stats.get('numberOfDocuments', 'unknown')}")
print(f"Indexing: {stats.get('isIndexing', 'unknown')}")
def cmd_search(args: argparse.Namespace) -> None:
init_db(settings.db_path)
embedder = build_embedder()
vector = embedder.embed_text(args.query)
meili = Meili(settings.meili_url, settings.meili_master_key, settings.meili_index)
result = meili.search_vector(
vector,
args.query,
args.limit,
settings.hybrid_semantic_ratio,
)
hits = result.get("hits", [])
if not hits:
print("No hits")
return
frame_ids = [hit.get("id") for hit in hits if hit.get("id")]
frame_vectors: dict[str, object] = {}
if frame_ids:
placeholders = ",".join("?" for _ in frame_ids)
with connect(settings.db_path) as con:
rows = con.execute(
f"""
SELECT f.id, f.embedding_f16, m.embedding_dim
FROM frames f
JOIN media m ON m.id = f.media_id
WHERE f.id IN ({placeholders})
""",
frame_ids,
).fetchall()
frame_vectors = {
row["id"]: unpack_vector(row["embedding_f16"], row["embedding_dim"])
for row in rows
}
scored_vectors = [frame_vectors.get(hit.get("id")) for hit in hits]
if all(item is not None for item in scored_vectors):
import numpy as np
siglip_scores = embedder.match_probabilities(
vector,
np.stack(scored_vectors), # type: ignore[arg-type]
)
else:
siglip_scores = [None] * len(hits)
print(f"\nQuery: {args.query!r}\n")
for i, (hit, siglip_score) in enumerate(zip(hits, siglip_scores, strict=True), 1):
meili_score = hit.get("_rankingScore")
meili_text = (
f" meili={meili_score:.4f}"
if isinstance(meili_score, (float, int))
else ""
)
siglip_text = (
f" siglip={float(siglip_score):.4f}"
if siglip_score is not None
else ""
)
print(
f"{i:2d}. {hit.get('title')} @ "
f"{format_ts(int(hit.get('timestamp_ms', 0)))}"
f"{meili_text}{siglip_text}"
)
caption = hit.get("caption")
if caption:
print(f" {caption}")
route = hit.get("caption_route")
route_score = hit.get("route_score")
if route:
score_text = (
f" score={float(route_score):.3f}"
if isinstance(route_score, (float, int))
else ""
)
print(f" route={route}{score_text}")
print(f" {hit.get('thumb_path')}")
def make_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(prog="jellyfin-scene-search")
sub = p.add_subparsers(dest="command", required=True)
init = sub.add_parser("init", help="Create the persistent SQLite database")
init.set_defaults(func=cmd_init)
idx = sub.add_parser("index-file", help="Index a local test video")
idx.add_argument("path")
idx.add_argument("--title")
idx.add_argument("--interval", type=float, default=2.0, help="Seconds between test frames (default: 2)")
idx.add_argument("--force", action="store_true")
idx.set_defaults(func=cmd_index_file)
sync = sub.add_parser("sync-meili", help="Rebuild/feed Meilisearch from canonical SQLite vectors")
sync.set_defaults(func=cmd_sync_meili)
route = sub.add_parser("route", help="Classify frames and show which caption model they will use")
route.add_argument("--media-id", help="Only route frames from one media ID")
route.add_argument("--title", help="Only route frames from one exact media title")
route.add_argument("--limit", type=int, help="Maximum number of frames to route")
route.set_defaults(func=cmd_route)
caption = sub.add_parser("caption", help="Route frames and generate persistent captions with Ollama")
caption.add_argument("--media-id", help="Only caption frames from one media ID")
caption.add_argument("--title", help="Only caption frames from one exact media title")
caption.add_argument("--limit", type=int, help="Maximum number of frames to caption")
caption.add_argument("--force", action="store_true", help="Regenerate captions even if already present")
caption.add_argument("--sync", action="store_true", help="Sync captions to Meilisearch when finished")
caption.set_defaults(func=cmd_caption)
status = sub.add_parser("status", help="Show canonical SQLite and Meilisearch index status")
status.set_defaults(func=cmd_status)
search = sub.add_parser("search", help="Hybrid caption + visual scene search")
search.add_argument("query")
search.add_argument("--limit", type=int, default=10)
search.set_defaults(func=cmd_search)
return p
def main() -> None:
args = make_parser().parse_args()
args.func(args)
if __name__ == "__main__":
main()