Files
jellyfin-scene-search/scene_search/cli.py
T
2026-10-06 17:55:12 +01:00

767 lines
26 KiB
Python

from __future__ import annotations
import argparse
import hashlib
from pathlib import Path
from PIL import Image
from .captioner import (
OllamaCaptioner,
ensure_caption_prompt_file,
ensure_specialist_prompt_file,
)
from .config import settings
from .database import connect, init_db, pack_vector, unpack_vector
from .embedder import SiglipEmbedder
from .frames import extract_frames, media_id_for, sha256_file
from .jellyfin import (
caption_context_from_tags,
route_override_from_tags,
tags_from_json,
)
from .jellyfin_index import (
cmd_jellyfin_index,
cmd_jellyfin_items,
cmd_jellyfin_libraries,
cmd_jellyfin_test,
)
from .maintenance import list_media_by_source, remove_media_by_source
from .meili import Meili
from .routing import ExplicitContentRouter, apply_context_routing
from .search_service import SearchService
from .sync import rebuild_meili
def format_ts(ms: int) -> str:
total = ms // 1000
h, rem = divmod(total, 3600)
m, s = divmod(rem, 60)
return f"{h:02d}:{m:02d}:{s:02d}"
def build_embedder() -> SiglipEmbedder:
print(f"Loading {settings.model_id} on {settings.device}...")
return SiglipEmbedder(settings.model_id, settings.device)
def cmd_init(_: argparse.Namespace) -> None:
init_db(settings.db_path)
ensure_caption_prompt_file(settings.caption_prompt_file)
ensure_specialist_prompt_file(settings.caption_specialist_prompt_file)
print(f"SQLite ready: {settings.db_path}")
print(f"General caption prompt: {settings.caption_prompt_file}")
print(f"Specialist caption prompt: {settings.caption_specialist_prompt_file}")
def cmd_index_file(args: argparse.Namespace) -> None:
init_db(settings.db_path)
path = Path(args.path).expanduser().resolve()
if not path.is_file():
raise SystemExit(f"Not a file: {path}")
interval_ms = int(round(args.interval * 1000))
stat = path.stat()
digest = sha256_file(path)
media_id = media_id_for(path, digest)
title = args.title or path.stem
with connect(settings.db_path) as con:
row = con.execute("SELECT * FROM media WHERE id = ?", (media_id,)).fetchone()
if (
row
and row["content_sha256"] == digest
and row["interval_ms"] == interval_ms
and row["model_id"] == settings.model_id
and not args.force
):
count = con.execute("SELECT COUNT(*) FROM frames WHERE media_id = ?", (media_id,)).fetchone()[0]
print(f"Already indexed: {title} ({count} frames). Use --force to rebuild.")
return
out_dir = settings.thumbs_dir / media_id
print(f"Extracting one frame every {args.interval:g}s from {title!r}...")
frame_paths = extract_frames(path, out_dir, args.interval)
if not frame_paths:
raise SystemExit("ffmpeg produced no frames")
embedder = build_embedder()
rows: list[tuple] = []
dim = None
for start in range(0, len(frame_paths), settings.batch_size):
batch_paths = frame_paths[start:start + settings.batch_size]
images = []
try:
for frame_path in batch_paths:
images.append(Image.open(frame_path).convert("RGB"))
vectors = embedder.embed_images(images)
finally:
for image in images:
image.close()
dim = int(vectors.shape[1])
for offset, (frame_path, vector) in enumerate(zip(batch_paths, vectors, strict=True)):
frame_number = start + offset
timestamp_ms = frame_number * interval_ms
frame_id = hashlib.sha256(f"{media_id}:{timestamp_ms}:{settings.model_id}".encode()).hexdigest()[:40]
rows.append((
frame_id,
media_id,
frame_number,
timestamp_ms,
str(frame_path),
pack_vector(vector),
))
print(f"Embedded {min(start + len(batch_paths), len(frame_paths))}/{len(frame_paths)} frames")
assert dim is not None
with connect(settings.db_path) as con:
con.execute("DELETE FROM media WHERE id = ?", (media_id,))
con.execute(
"""
INSERT INTO media
(id, source_path, title, content_sha256, size_bytes, mtime_ns,
interval_ms, model_id, embedding_dim)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(media_id, str(path), title, digest, stat.st_size, stat.st_mtime_ns,
interval_ms, settings.model_id, dim),
)
con.executemany(
"""
INSERT INTO frames
(id, media_id, frame_number, timestamp_ms, thumb_path, embedding_f16)
VALUES (?, ?, ?, ?, ?, ?)
""",
rows,
)
print(f"Stored {len(rows)} canonical frame embeddings in {settings.db_path}")
print("Next: python -m scene_search.cli sync-meili")
def cmd_sync_meili(_: argparse.Namespace) -> None:
def report(update: dict) -> None:
if update.get("stage") == "sync":
print(
f"Queued {int(update.get('documents_processed') or 0)} "
"documents into Meilisearch"
)
total = rebuild_meili(progress=report)
if total:
print("Meilisearch rebuild complete (tasks may continue briefly inside Meili).")
else:
print("Meilisearch cleared; no canonical media remain.")
def _route_policy() -> str:
tag_policy = (
f"force-specialist={settings.jellyfin_force_specialist_tag}|"
f"force-general={settings.jellyfin_force_general_tag}|"
f"specialist-tags={','.join(settings.jellyfin_specialist_tags)}"
)
if not settings.caption_router_enabled:
return (
f"disabled|general={settings.caption_general_model}|{tag_policy}"
)
return (
f"model={settings.caption_router_model}|"
f"threshold={settings.caption_router_threshold:.6f}|"
f"context={settings.caption_router_context_seconds:.3f}|"
f"{tag_policy}"
)
def _select_caption_rows(
args: argparse.Namespace,
*,
only_needing_caption: bool,
) -> list:
conditions = []
params: list[object] = []
if only_needing_caption and not getattr(args, "force", False):
expected_models = [
settings.caption_general_model,
settings.caption_specialist_model,
]
placeholders = ",".join("?" for _ in expected_models)
conditions.append(
f"(f.caption IS NULL OR f.caption_model IS NULL "
f"OR f.caption_model NOT IN ({placeholders}) "
f"OR f.route_policy IS NULL OR f.route_policy != ?)"
)
params.extend(expected_models)
params.append(_route_policy())
if getattr(args, "media_id", None):
conditions.append("f.media_id = ?")
params.append(args.media_id)
if getattr(args, "title", None):
conditions.append("m.title = ?")
params.append(args.title)
where = f"WHERE {' AND '.join(conditions)}" if conditions else ""
limit = ""
if getattr(args, "limit", None) is not None:
limit = " LIMIT ?"
params.append(args.limit)
with connect(settings.db_path) as con:
return con.execute(
f"""
SELECT f.id, f.media_id, f.timestamp_ms, f.thumb_path, f.caption,
f.caption_model, f.caption_route, f.route_score, f.route_policy,
m.title, m.jellyfin_tags
FROM frames f
JOIN media m ON m.id = f.media_id
{where}
ORDER BY m.title, f.timestamp_ms
{limit}
""",
params,
).fetchall()
def _route_rows(rows: list, *, verbose: bool = False) -> tuple[list[tuple], int]:
valid_rows = []
missing = 0
for row in rows:
image_path = Path(row["thumb_path"])
if not image_path.is_file():
missing += 1
print(
f"SKIP {row['title']} @ {format_ts(int(row['timestamp_ms']))}: "
f"missing thumbnail {image_path}"
)
continue
valid_rows.append(row)
if not valid_rows:
return [], missing
policy = _route_policy()
if settings.caption_router_enabled:
print(
f"Routing {len(valid_rows)} frame(s) with "
f"{settings.caption_router_model} on {settings.caption_router_device}..."
)
router = ExplicitContentRouter(
settings.caption_router_model,
settings.caption_router_device,
settings.caption_router_batch_size,
)
score_values = router.score_paths(
[Path(row["thumb_path"]) for row in valid_rows]
)
scores = {
str(row["id"]): score
for row, score in zip(valid_rows, score_values, strict=True)
}
route_input = [
{
"id": str(row["id"]),
"media_id": str(row["media_id"]),
"timestamp_ms": int(row["timestamp_ms"]),
}
for row in valid_rows
]
decisions = apply_context_routing(
route_input,
scores,
settings.caption_router_threshold,
int(round(settings.caption_router_context_seconds * 1000)),
)
route_model = settings.caption_router_model
else:
scores = {str(row["id"]): 0.0 for row in valid_rows}
decisions = {
str(row["id"]): ("general", "disabled")
for row in valid_rows
}
route_model = "disabled"
for row in valid_rows:
tags = tags_from_json(row["jellyfin_tags"])
override = route_override_from_tags(
tags,
force_specialist_tag=settings.jellyfin_force_specialist_tag,
force_general_tag=settings.jellyfin_force_general_tag,
specialist_tags=settings.jellyfin_specialist_tags,
)
if override is not None:
decisions[str(row["id"])] = override
routed: list[tuple] = []
with connect(settings.db_path) as con:
for row in valid_rows:
frame_id = str(row["id"])
route, reason = decisions[frame_id]
score = scores[frame_id]
con.execute(
"""
UPDATE frames
SET caption_route = ?, caption_route_reason = ?, route_score = ?,
route_model = ?, route_policy = ?, routed_at = CURRENT_TIMESTAMP
WHERE id = ?
""",
(route, reason, score, route_model, policy, frame_id),
)
routed.append((row, route, score, reason))
if verbose:
print(
f"{row['title']} @ {format_ts(int(row['timestamp_ms']))}: "
f"score={score:.4f} -> {route} ({reason})"
)
general_count = sum(1 for _, route, _, _ in routed if route == "general")
specialist_count = len(routed) - general_count
if settings.caption_router_enabled:
print(
f"Routing complete: {general_count} general, "
f"{specialist_count} specialist "
f"(threshold {settings.caption_router_threshold:.2f}, "
f"context ±{settings.caption_router_context_seconds:g}s)"
)
else:
print(f"Routing disabled: {general_count} frame(s) -> general model")
return routed, missing
def _unload_ollama_model(model: str) -> None:
captioner = OllamaCaptioner(
settings.ollama_url,
model,
settings.caption_timeout_seconds,
max_tokens=settings.caption_max_tokens,
)
try:
captioner.unload()
except Exception as exc:
print(f"WARN could not unload {model}: {exc}")
def _caption_queue(
label: str,
routed_rows: list[tuple],
model: str,
prompt: str,
) -> tuple[int, int]:
if not routed_rows:
return 0, 0
print(f"\n{label}: {len(routed_rows)} frame(s) with {model}")
captioner = OllamaCaptioner(
settings.ollama_url,
model,
settings.caption_timeout_seconds,
prompt,
settings.caption_max_tokens,
)
completed = 0
failed = 0
try:
for row, route, score, reason in routed_rows:
image_path = Path(row["thumb_path"])
try:
context_hint = None
if settings.jellyfin_tag_context:
context_hint = caption_context_from_tags(
tags_from_json(row["jellyfin_tags"]),
control_tags=(
settings.jellyfin_force_specialist_tag,
settings.jellyfin_force_general_tag,
settings.jellyfin_skip_tag,
),
)
caption = captioner.caption(image_path, context_hint=context_hint)
except Exception as exc:
failed += 1
print(
f"FAIL {row['title']} @ "
f"{format_ts(int(row['timestamp_ms']))}: {exc}"
)
continue
with connect(settings.db_path) as con:
con.execute(
"""
UPDATE frames
SET caption = ?, caption_model = ?,
captioned_at = CURRENT_TIMESTAMP
WHERE id = ?
""",
(caption, model, row["id"]),
)
completed += 1
print(
f"{completed:4d}/{len(routed_rows)} {row['title']} @ "
f"{format_ts(int(row['timestamp_ms']))} "
f"[{route} {score:.3f}/{reason}]: {caption}"
)
finally:
try:
captioner.unload()
print(f"Unloaded {model} from Ollama.")
except Exception as exc:
print(f"WARN could not unload {model}: {exc}")
return completed, failed
def cmd_route(args: argparse.Namespace) -> None:
init_db(settings.db_path)
rows = _select_caption_rows(args, only_needing_caption=False)
if not rows:
print("No matching frames.")
return
_, missing = _route_rows(rows, verbose=True)
if missing:
print(f"Routing skipped {missing} frame(s) with missing thumbnails.")
def cmd_caption(args: argparse.Namespace) -> None:
init_db(settings.db_path)
rows = _select_caption_rows(args, only_needing_caption=True)
if not rows:
print("No frames need captioning.")
return
routed, missing = _route_rows(rows)
if not routed:
print(f"No captionable frames ({missing} missing thumbnails).")
return
general_prompt = ensure_caption_prompt_file(settings.caption_prompt_file)
specialist_prompt = ensure_specialist_prompt_file(
settings.caption_specialist_prompt_file
)
general_rows = [
item
for item in routed
if item[1] == "general"
and (
args.force
or item[0]["caption"] is None
or item[0]["caption_model"] != settings.caption_general_model
)
]
specialist_rows = [
item
for item in routed
if item[1] == "specialist"
and (
args.force
or item[0]["caption"] is None
or item[0]["caption_model"] != settings.caption_specialist_model
)
]
retained = len(routed) - len(general_rows) - len(specialist_rows)
if retained:
print(
f"Keeping {retained} existing caption(s) because they already use "
f"the model selected by the router."
)
print(f"General prompt: {settings.caption_prompt_file}")
print(f"Specialist prompt: {settings.caption_specialist_prompt_file}")
print(f"Ollama: {settings.ollama_url}")
if general_rows:
_unload_ollama_model(settings.caption_specialist_model)
general_done, general_failed = _caption_queue(
"General queue",
general_rows,
settings.caption_general_model,
general_prompt,
)
if specialist_rows:
_unload_ollama_model(settings.caption_general_model)
specialist_done, specialist_failed = _caption_queue(
"Specialist queue",
specialist_rows,
settings.caption_specialist_model,
specialist_prompt,
)
completed = general_done + specialist_done
failed = general_failed + specialist_failed + missing
print(f"\nCaptioning complete: {completed} saved, {failed} failed/skipped.")
if args.sync:
cmd_sync_meili(args)
elif completed:
print("Next: python -m scene_search.cli sync-meili")
def cmd_remove(args: argparse.Namespace) -> None:
source_kind = args.source.strip().lower()
matches = list_media_by_source(settings.db_path, source_kind)
if not matches:
print(f"No media found with source_kind={source_kind!r}.")
return
print(f"Media matching source_kind={source_kind!r}:")
for item in matches:
print(f"- {item.title} ({item.media_id})")
if not args.yes:
answer = input(
f"Remove {len(matches)} media item(s), their frames/captions/vectors, "
"and cached thumbnails? [y/N] "
).strip().lower()
if answer not in {"y", "yes"}:
print("Cancelled.")
return
removed = remove_media_by_source(
settings.db_path,
settings.thumbs_dir,
source_kind,
)
print(f"Removed {len(removed)} media item(s) from the canonical index.")
with connect(settings.db_path) as con:
remaining = int(con.execute("SELECT COUNT(*) FROM media").fetchone()[0])
if remaining:
print("Rebuilding Meilisearch from remaining canonical media...")
cmd_sync_meili(args)
else:
print("No canonical media remain; clearing Meilisearch.")
meili = Meili(
settings.meili_url,
settings.meili_master_key,
settings.meili_index,
)
meili.clear_documents()
print("Meilisearch cleared.")
def cmd_status(_: argparse.Namespace) -> None:
init_db(settings.db_path)
print("Canonical SQLite")
print("----------------")
with connect(settings.db_path) as con:
media_rows = con.execute(
"""
SELECT m.id, m.title, m.interval_ms, m.model_id,
COUNT(f.id) AS frames,
SUM(CASE WHEN f.caption IS NOT NULL THEN 1 ELSE 0 END) AS captioned,
SUM(CASE WHEN f.caption_route = 'specialist' THEN 1 ELSE 0 END)
AS specialist_routed
FROM media m
LEFT JOIN frames f ON f.media_id = m.id
GROUP BY m.id, m.title, m.interval_ms, m.model_id
ORDER BY m.title, m.id
"""
).fetchall()
total_frames = sum(int(row["frames"]) for row in media_rows)
print(f"Media items: {len(media_rows)}")
print(f"Frames: {total_frames}")
if media_rows:
print()
for row in media_rows:
print(
f"- {row['title']}: {row['frames']} frames, "
f"{row['captioned']} captioned, "
f"{row['specialist_routed']} specialist-routed "
f"({row['interval_ms'] / 1000:g}s interval)"
)
print()
print("Meilisearch")
print("-----------")
meili = Meili(settings.meili_url, settings.meili_master_key, settings.meili_index)
try:
stats = meili.stats()
except Exception as exc:
print(f"Status: unavailable ({exc})")
return
print(f"Index: {settings.meili_index}")
print(f"Documents: {stats.get('numberOfDocuments', 'unknown')}")
print(f"Indexing: {stats.get('isIndexing', 'unknown')}")
def cmd_search(args: argparse.Namespace) -> None:
service = SearchService(
db_path=settings.db_path,
meili_url=settings.meili_url,
meili_master_key=settings.meili_master_key,
meili_index=settings.meili_index,
model_id=settings.model_id,
device=settings.device,
semantic_ratio=settings.hybrid_semantic_ratio,
)
result = service.search(
args.query,
scene_limit=args.limit,
item_limit=args.limit,
jellyfin_only=False,
)
scenes = result["scenes"]
if not scenes:
print("No hits")
return
print(f"\nQuery: {args.query!r}\n")
for i, scene in enumerate(scenes, 1):
meili_score = scene.get("ranking_score")
siglip_score = scene.get("siglip_score")
meili_text = f" meili={meili_score:.4f}" if meili_score is not None else ""
siglip_text = f" siglip={siglip_score:.4f}" if siglip_score is not None else ""
print(
f"{i:2d}. {scene.get('title')} @ "
f"{format_ts(int(scene.get('timestamp_ms') or 0))}"
f"{meili_text}{siglip_text}"
)
if scene.get("caption"):
print(f" {scene['caption']}")
if scene.get("caption_route"):
route_score = scene.get("route_score")
score_text = f" score={float(route_score):.3f}" if route_score is not None else ""
print(f" route={scene['caption_route']}{score_text}")
if scene.get("jellyfin_item_id"):
tags = scene.get("tags") or []
tags_text = f" tags={', '.join(tags)}" if tags else ""
print(f" jellyfin={scene['jellyfin_item_id']}{tags_text}")
def cmd_serve(_: argparse.Namespace) -> None:
from .api import main as api_main
api_main()
def make_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(prog="jellyfin-scene-search")
sub = p.add_subparsers(dest="command", required=True)
init = sub.add_parser("init", help="Create the persistent SQLite database")
init.set_defaults(func=cmd_init)
idx = sub.add_parser("index-file", help="Index a local test video")
idx.add_argument("path")
idx.add_argument("--title")
idx.add_argument("--interval", type=float, default=2.0, help="Seconds between test frames (default: 2)")
idx.add_argument("--force", action="store_true")
idx.set_defaults(func=cmd_index_file)
sync = sub.add_parser("sync-meili", help="Rebuild/feed Meilisearch from canonical SQLite vectors")
sync.set_defaults(func=cmd_sync_meili)
jf_test = sub.add_parser("jellyfin-test", help="Verify Jellyfin API connectivity")
jf_test.set_defaults(func=cmd_jellyfin_test)
jf_libs = sub.add_parser("jellyfin-libraries", help="List Jellyfin media libraries")
jf_libs.set_defaults(func=cmd_jellyfin_libraries)
jf_items = sub.add_parser("jellyfin-items", help="Inspect video items, tags and Trickplay")
jf_items.add_argument(
"--library",
action="append",
help="Jellyfin library name; repeat to inspect multiple libraries",
)
jf_items.add_argument(
"--all-libraries",
action="store_true",
help="Inspect every Jellyfin library visible to the API key",
)
jf_items.add_argument(
"--exclude-library",
action="append",
help="Library name to exclude; repeat as needed",
)
jf_items.add_argument(
"--limit",
type=int,
help="Maximum number of items to inspect per selected library",
)
jf_items.set_defaults(func=cmd_jellyfin_items)
jf_index = sub.add_parser("jellyfin-index", help="Index one or more Jellyfin libraries via Trickplay")
jf_index.add_argument(
"--library",
action="append",
help="Jellyfin library name; repeat to index multiple libraries",
)
jf_index.add_argument(
"--all-libraries",
action="store_true",
help="Index every Jellyfin library visible to the API key",
)
jf_index.add_argument(
"--exclude-library",
action="append",
help="Library name to exclude; repeat as needed",
)
jf_index.add_argument(
"--limit",
type=int,
help="Maximum number of items to index per selected library; disables pruning",
)
jf_index.add_argument("--force", action="store_true", help="Rebuild matching Jellyfin items")
jf_index.set_defaults(func=cmd_jellyfin_index)
route = sub.add_parser("route", help="Classify frames and show which caption model they will use")
route.add_argument("--media-id", help="Only route frames from one media ID")
route.add_argument("--title", help="Only route frames from one exact media title")
route.add_argument("--limit", type=int, help="Maximum number of frames to route")
route.set_defaults(func=cmd_route)
caption = sub.add_parser("caption", help="Route frames and generate persistent captions with Ollama")
caption.add_argument("--media-id", help="Only caption frames from one media ID")
caption.add_argument("--title", help="Only caption frames from one exact media title")
caption.add_argument("--limit", type=int, help="Maximum number of frames to caption")
caption.add_argument("--force", action="store_true", help="Regenerate captions even if already present")
caption.add_argument("--sync", action="store_true", help="Sync captions to Meilisearch when finished")
caption.set_defaults(func=cmd_caption)
remove = sub.add_parser("remove", help="Remove indexed media by source kind")
remove.add_argument(
"--source",
required=True,
choices=["file", "jellyfin"],
help="Source kind to remove",
)
remove.add_argument(
"--yes",
action="store_true",
help="Skip the interactive confirmation prompt",
)
remove.set_defaults(func=cmd_remove)
status = sub.add_parser("status", help="Show canonical SQLite and Meilisearch index status")
status.set_defaults(func=cmd_status)
serve = sub.add_parser("serve", help="Run the local HTTP search API")
serve.set_defaults(func=cmd_serve)
search = sub.add_parser("search", help="Hybrid caption + visual scene search")
search.add_argument("query")
search.add_argument("--limit", type=int, default=10)
search.set_defaults(func=cmd_search)
return p
def main() -> None:
args = make_parser().parse_args()
args.func(args)
if __name__ == "__main__":
main()