from __future__ import annotations import hashlib import io import math import shutil from collections.abc import Callable from pathlib import Path from PIL import Image from .caption_windows import ( load_media_caption_windows, restore_media_caption_windows, ) from .config import settings from .database import connect, init_db, pack_vector from .embedder import SiglipEmbedder from .frames import extract_frames from .jellyfin import ( JellyfinClient, item_tags, jellyfin_media_id, primary_media_source, tags_to_json, ) def build_jellyfin_client() -> JellyfinClient: return JellyfinClient( settings.jellyfin_url, settings.jellyfin_api_key, settings.jellyfin_timeout_seconds, ) def _library_name(library: dict) -> str: return str(library.get("Name") or library.get("Id") or "(unnamed)") ProgressCallback = Callable[[dict], None] def resolve_library_ids( client: JellyfinClient, library_ids: list[str], ) -> list[dict]: requested = {str(value).strip() for value in library_ids if str(value).strip()} if not requested: raise RuntimeError("Select at least one Jellyfin library.") available = client.media_folders() by_id = { str(library.get("Id") or ""): library for library in available if str(library.get("Id") or "") } missing = sorted(requested.difference(by_id)) if missing: raise RuntimeError( "Jellyfin library id(s) not found: " + ", ".join(missing) ) return sorted( [by_id[library_id] for library_id in requested], key=lambda library: _library_name(library).casefold(), ) def resolve_libraries( client: JellyfinClient, requested: list[str] | None, *, all_libraries: bool = False, excluded: list[str] | None = None, ) -> list[dict]: if all_libraries and requested: raise RuntimeError("Use either --all-libraries or --library, not both.") available = client.media_folders() by_name = { _library_name(library).casefold(): library for library in available } excluded_names = { name.casefold() for name in (*settings.jellyfin_exclude_libraries, *(excluded or [])) if name.strip() } if all_libraries: selected = [ library for library in available if _library_name(library).casefold() not in excluded_names ] else: configured = list(settings.jellyfin_libraries) if not configured and settings.jellyfin_library.strip(): configured = [settings.jellyfin_library.strip()] names = requested or configured if not names: raise RuntimeError( "No Jellyfin libraries selected. Repeat --library, use " "--all-libraries, set JELLYFIN_LIBRARIES, or keep using " "the legacy JELLYFIN_LIBRARY setting." ) selected = [] missing = [] seen_ids: set[str] = set() for name in names: key = name.strip().casefold() library = by_name.get(key) if library is None: missing.append(name) continue if key in excluded_names: continue library_id = str(library.get("Id") or "") if library_id and library_id in seen_ids: continue if library_id: seen_ids.add(library_id) selected.append(library) if missing: available_names = ", ".join( sorted(_library_name(library) for library in available) ) raise RuntimeError( f"Jellyfin librar{'y' if len(missing) == 1 else 'ies'} not found: " f"{', '.join(repr(name) for name in missing)}. " f"Available: {available_names or 'none'}" ) if not selected: raise RuntimeError("No Jellyfin libraries remain after exclusions.") return sorted(selected, key=lambda library: _library_name(library).casefold()) def _display_tags(tags: list[str]) -> str: return ", ".join(tags) if tags else "-" def cmd_jellyfin_test(_: object) -> None: client = build_jellyfin_client() info = client.system_info() libraries = client.media_folders() print( f"Connected to Jellyfin {info.get('Version', '?')} " f"({info.get('ServerName') or info.get('ProductName') or 'server'})" ) print(f"Libraries visible to API key: {len(libraries)}") def cmd_jellyfin_libraries(_: object) -> None: client = build_jellyfin_client() libraries = client.media_folders() if not libraries: print("No Jellyfin libraries returned.") return configured = { name.casefold() for name in settings.jellyfin_libraries } if settings.jellyfin_library.strip(): configured.add(settings.jellyfin_library.strip().casefold()) excluded = { name.casefold() for name in settings.jellyfin_exclude_libraries } for item in sorted(libraries, key=lambda row: _library_name(row).casefold()): name = _library_name(item) flags = [] if name.casefold() in configured: flags.append("configured") if name.casefold() in excluded: flags.append("excluded") suffix = f"\t[{', '.join(flags)}]" if flags else "" print(f"{name}\t{item.get('Id')}{suffix}") def cmd_jellyfin_items(args: object) -> None: client = build_jellyfin_client() libraries = resolve_libraries( client, getattr(args, "library", None), all_libraries=bool(getattr(args, "all_libraries", False)), excluded=getattr(args, "exclude_library", None), ) limit = getattr(args, "limit", None) for library in libraries: library_name = _library_name(library) summaries = client.items(str(library["Id"]), limit=limit) print(f"\n=== {library_name} ({len(summaries)} video item(s)) ===") if not summaries: print("No video items found.") continue for summary in summaries: detail = summary tags = item_tags(detail) trickplay = client.select_trickplay( detail, settings.jellyfin_trickplay_width, ) tp = ( f"{trickplay.width}px/{trickplay.interval_ms}ms/" f"{trickplay.thumbnail_count} thumbs" if trickplay else "none" ) print( f"{detail.get('Name') or summary.get('Name') or '(unnamed)'}\n" f" id: {detail.get('Id')}\n" f" type: {detail.get('Type')}\n" f" tags: {_display_tags(tags)}\n" f" trickplay: {tp}\n" f" path: {detail.get('Path') or '-'}" ) def _embed_frames( embedder: SiglipEmbedder, frame_paths: list[Path], media_id: str, interval_ms: int, *, progress: ProgressCallback | None = None, progress_context: dict | None = None, ) -> tuple[list[tuple], int]: rows: list[tuple] = [] dim: int | None = None for start in range(0, len(frame_paths), settings.batch_size): batch_paths = frame_paths[start:start + settings.batch_size] images: list[Image.Image] = [] try: for frame_path in batch_paths: images.append(Image.open(frame_path).convert("RGB")) vectors = embedder.embed_images(images) finally: for image in images: image.close() dim = int(vectors.shape[1]) for offset, (frame_path, vector) in enumerate( zip(batch_paths, vectors, strict=True) ): frame_number = start + offset timestamp_ms = frame_number * interval_ms frame_id = hashlib.sha256( f"{media_id}:{timestamp_ms}:{settings.model_id}".encode() ).hexdigest()[:40] rows.append( ( frame_id, media_id, frame_number, timestamp_ms, str(frame_path), pack_vector(vector), ) ) processed = min(start + len(batch_paths), len(frame_paths)) print(f" embedded {processed}/{len(frame_paths)}") if progress is not None: progress( { **(progress_context or {}), "stage": "embedding", "frames_processed": processed, "frames_total": len(frame_paths), } ) if dim is None: raise RuntimeError("No frames were embedded") return rows, dim def _embed_jellyfin_trickplay( client: JellyfinClient, embedder: SiglipEmbedder, item_id: str, selection, media_id: str, *, progress: ProgressCallback | None = None, progress_context: dict | None = None, ) -> tuple[list[tuple], int]: """Embed Jellyfin Trickplay tiles in memory without persisting duplicate JPEGs.""" rows: list[tuple] = [] dim: int | None = None pending_images: list[Image.Image] = [] pending_numbers: list[int] = [] def flush() -> None: nonlocal dim if not pending_images: return try: vectors = embedder.embed_images(pending_images) dim = int(vectors.shape[1]) for frame_number, vector in zip( pending_numbers, vectors, strict=True, ): timestamp_ms = frame_number * selection.interval_ms frame_id = hashlib.sha256( f"{media_id}:{timestamp_ms}:{settings.model_id}".encode() ).hexdigest()[:40] rows.append( ( frame_id, media_id, frame_number, timestamp_ms, ( f"jellyfin://{item_id}/trickplay/" f"{selection.width}/{frame_number}" ), pack_vector(vector), ) ) finally: for image in pending_images: image.close() pending_images.clear() pending_numbers.clear() if progress is not None: progress( { **(progress_context or {}), "stage": "embedding", "frames_processed": len(rows), "frames_total": selection.thumbnail_count, } ) print(f" embedded {len(rows)}/{selection.thumbnail_count}") capacity = selection.thumbnails_per_sheet sheet_count = math.ceil(selection.thumbnail_count / capacity) for sheet_index in range(sheet_count): raw = client.trickplay_sheet_bytes( item_id, selection, sheet_index, ) with Image.open(io.BytesIO(raw)) as opened: sheet = opened.convert("RGB") try: for local_index in range(capacity): frame_number = sheet_index * capacity + local_index if frame_number >= selection.thumbnail_count: break column = local_index % selection.tile_columns tile_row = local_index // selection.tile_columns left = column * selection.width top = tile_row * selection.height pending_images.append( sheet.crop( ( left, top, left + selection.width, top + selection.height, ) ) ) pending_numbers.append(frame_number) if len(pending_images) >= settings.batch_size: flush() finally: sheet.close() flush() if dim is None: raise RuntimeError("No Jellyfin Trickplay frames were embedded") return rows, dim FRAME_ANNOTATION_COLUMNS = ( "caption", "caption_model", "captioned_at", "caption_route", "caption_route_reason", "route_score", "route_model", "route_policy", "routed_at", ) def _load_frame_annotations(media_id: str) -> dict[str, tuple]: with connect(settings.db_path) as con: rows = con.execute( """ SELECT id, caption, caption_model, captioned_at, caption_route, caption_route_reason, route_score, route_model, route_policy, routed_at FROM frames WHERE media_id = ? """, (media_id,), ).fetchall() return { str(row["id"]): tuple(row[column] for column in FRAME_ANNOTATION_COLUMNS) for row in rows } def _restore_frame_annotations( con, annotations: dict[str, tuple], frame_ids: set[str], ) -> int: matches = [ (*values, frame_id) for frame_id, values in annotations.items() if frame_id in frame_ids ] if not matches: return 0 con.executemany( """ UPDATE frames SET caption = ?, caption_model = ?, captioned_at = ?, caption_route = ?, caption_route_reason = ?, route_score = ?, route_model = ?, route_policy = ?, routed_at = ? WHERE id = ? """, matches, ) return len(matches) def reconcile_selected_libraries( selected_library_ids: set[str], ) -> dict[str, object]: """Make the plugin/dashboard library selection authoritative locally.""" init_db(settings.db_path) selected = { str(value).strip() for value in selected_library_ids if str(value).strip() } removed_library_ids: list[str] = [] removed_media_ids: list[str] = [] removed_items: list[dict[str, object]] = [] with connect(settings.db_path) as con: rows = con.execute( """ SELECT id, title, jellyfin_item_id, jellyfin_library_id FROM media WHERE source_kind = 'jellyfin' AND jellyfin_library_id IS NOT NULL AND jellyfin_library_id != '' """ ).fetchall() stale = [ row for row in rows if str(row["jellyfin_library_id"]) not in selected ] removed_library_ids = sorted( {str(row["jellyfin_library_id"]) for row in stale} ) removed_media_ids = [str(row["id"]) for row in stale] removed_items = [ { "status": "removed", "library_id": str(row["jellyfin_library_id"] or ""), "library": str(row["jellyfin_library_id"] or ""), "item_id": str(row["jellyfin_item_id"] or ""), "title": str(row["title"] or row["jellyfin_item_id"] or row["id"]), "reason": "Library deselected from Scene Search", } for row in stale ] if removed_library_ids: placeholders = ",".join("?" for _ in removed_library_ids) con.execute( f""" DELETE FROM media WHERE source_kind = 'jellyfin' AND jellyfin_library_id IN ({placeholders}) """, removed_library_ids, ) for media_id in removed_media_ids: shutil.rmtree(settings.thumbs_dir / media_id, ignore_errors=True) return { "removed_library_ids": removed_library_ids, "removed_libraries": len(removed_library_ids), "removed_media": len(removed_media_ids), "removed_items": removed_items, } def _reconcile_library( library_id: str, current_item_ids: set[str], *, library_name: str | None = None, report_items: list[dict[str, object]] | None = None, ) -> int: stale_rows = [] with connect(settings.db_path) as con: rows = con.execute( """ SELECT id, title, jellyfin_item_id FROM media WHERE source_kind = 'jellyfin' AND jellyfin_library_id = ? """, (library_id,), ).fetchall() stale_rows = [ row for row in rows if str(row["jellyfin_item_id"] or "") not in current_item_ids ] for row in stale_rows: con.execute("DELETE FROM media WHERE id = ?", (row["id"],)) for row in stale_rows: shutil.rmtree(settings.thumbs_dir / str(row["id"]), ignore_errors=True) if report_items is not None: report_items.append( { "status": "removed", "library_id": library_id, "library": library_name or library_id, "item_id": str(row["jellyfin_item_id"] or ""), "title": str(row["title"] or row["jellyfin_item_id"] or row["id"]), "reason": "Item no longer present in Jellyfin library", } ) return len(stale_rows) def _index_library( client: JellyfinClient, library: dict, *, embedder: SiglipEmbedder | None, limit: int | None, force: bool, progress: ProgressCallback | None = None, report_items: list[dict[str, object]] | None = None, ) -> tuple[SiglipEmbedder | None, int, int, int, int]: library_id = str(library["Id"]) library_name = _library_name(library) summaries = client.items(library_id, limit=limit) print( f"\n=== Jellyfin library {library_name!r}: " f"{len(summaries)} video item(s) selected ===" ) indexed = 0 skipped = 0 failed = 0 pruned = 0 current_item_ids: set[str] = set() if progress is not None: progress( { "stage": "library", "library_id": library_id, "library": library_name, "item_number": 0, "item_total": len(summaries), } ) for number, summary in enumerate(summaries, 1): try: detail = summary item_id = str(detail.get("Id") or summary["Id"]) current_item_ids.add(item_id) title = str(detail.get("Name") or summary.get("Name") or item_id) if progress is not None: progress( { "stage": "item", "library_id": library_id, "library": library_name, "item_id": item_id, "item": title, "item_number": number, "item_total": len(summaries), "frames_processed": 0, "frames_total": 0, } ) tags = item_tags(detail) normalized_tags = {tag.casefold() for tag in tags} if settings.jellyfin_skip_tag and settings.jellyfin_skip_tag in normalized_tags: with connect(settings.db_path) as con: existing = con.execute( "SELECT id FROM media WHERE jellyfin_item_id = ?", (item_id,), ).fetchone() if existing: con.execute("DELETE FROM media WHERE id = ?", (existing["id"],)) pruned += 1 if report_items is not None: report_items.append( { "status": "removed", "library_id": library_id, "library": library_name, "item_id": item_id, "title": title, "reason": "Jellyfin skip tag", } ) shutil.rmtree( settings.thumbs_dir / jellyfin_media_id(item_id), ignore_errors=True, ) print( f"[{number}/{len(summaries)}] SKIP {title}: " "Jellyfin skip tag (removed from canonical index)" ) skipped += 1 continue selection = client.select_trickplay( detail, settings.jellyfin_trickplay_width, ) source = primary_media_source(detail) source_path_text = str( detail.get("Path") or source.get("Path") or f"jellyfin://{item_id}" ) media_id = jellyfin_media_id(item_id) etag = str(detail.get("Etag") or "") content_marker = hashlib.sha256( f"jellyfin:{item_id}:{etag}".encode() ).hexdigest() expected_interval_ms = ( selection.interval_ms if selection else int(round(settings.jellyfin_fallback_interval_seconds * 1000)) ) with connect(settings.db_path) as con: existing = con.execute( "SELECT * FROM media WHERE jellyfin_item_id = ?", (item_id,), ).fetchone() tags_json = tags_to_json(tags) if ( existing and existing["jellyfin_etag"] == etag and existing["jellyfin_tags"] == tags_json and existing["model_id"] == settings.model_id and int(existing["interval_ms"]) == expected_interval_ms and str(existing["jellyfin_library_id"] or "") == library_id and not force ): with connect(settings.db_path) as con: frame_count = int( con.execute( "SELECT COUNT(*) FROM frames WHERE media_id = ?", (existing["id"],), ).fetchone()[0] ) print( f"[{number}/{len(summaries)}] {title}: already indexed " f"({frame_count} frames)" ) skipped += 1 continue out_dir = settings.thumbs_dir / media_id if selection: print( f"[{number}/{len(summaries)}] {title}: Trickplay " f"{selection.width}px every {selection.interval_ms / 1000:g}s " "(Jellyfin-backed, no duplicate thumbnails)" ) interval_ms = selection.interval_ms media_source_id = selection.media_source_id trickplay_width = selection.width trickplay_tile_width = selection.tile_columns trickplay_tile_height = selection.tile_rows else: local_path = Path(source_path_text) if not local_path.is_file(): reason = "No Trickplay and media path is not locally accessible" print( f"[{number}/{len(summaries)}] SKIP {title}: " "no Trickplay and media path is not locally accessible" ) failed += 1 if report_items is not None: report_items.append( { "status": "failed", "library_id": library_id, "library": library_name, "item_id": item_id, "title": title, "reason": reason, } ) continue print( f"[{number}/{len(summaries)}] {title}: no Trickplay; " f"ffmpeg fallback every " f"{settings.jellyfin_fallback_interval_seconds:g}s" ) frame_paths = extract_frames( local_path, out_dir, settings.jellyfin_fallback_interval_seconds, ) interval_ms = int( round(settings.jellyfin_fallback_interval_seconds * 1000) ) media_source_id = str(source.get("Id") or "") or None trickplay_width = None trickplay_tile_width = None trickplay_tile_height = None if selection is None and not frame_paths: reason = "No frames were extracted" print(f"[{number}/{len(summaries)}] SKIP {title}: no frames") failed += 1 if report_items is not None: report_items.append( { "status": "failed", "library_id": library_id, "library": library_name, "item_id": item_id, "title": title, "reason": reason, } ) continue if embedder is None: print(f"Loading {settings.model_id} on {settings.device}...") embedder = SiglipEmbedder(settings.model_id, settings.device) progress_context = { "library_id": library_id, "library": library_name, "item_id": item_id, "item": title, "item_number": number, "item_total": len(summaries), } if selection: frame_rows, embedding_dim = _embed_jellyfin_trickplay( client, embedder, item_id, selection, media_id, progress=progress, progress_context=progress_context, ) else: frame_rows, embedding_dim = _embed_frames( embedder, frame_paths, media_id, interval_ms, progress=progress, progress_context=progress_context, ) size_bytes = int(source.get("Size") or 0) # Preserve AI work when the underlying media still appears to be the # same source. Frame ids are deterministic by media/timestamp/model, # so only exact surviving frames receive the old annotations. annotations: dict[str, tuple] = {} caption_window_snapshot: list[dict] = [] if ( existing and str(existing["source_path"] or "") == source_path_text and int(existing["size_bytes"] or 0) == size_bytes and str(existing["model_id"] or "") == settings.model_id and str(existing["jellyfin_media_source_id"] or "") == str(media_source_id or "") and str(existing["jellyfin_tags"] or "") == tags_json ): annotations = _load_frame_annotations(str(existing["id"])) caption_window_snapshot = load_media_caption_windows( settings.db_path, str(existing["id"]), ) with connect(settings.db_path) as con: con.execute("DELETE FROM media WHERE id = ?", (media_id,)) con.execute( """ INSERT INTO media ( id, source_path, title, content_sha256, size_bytes, mtime_ns, interval_ms, model_id, embedding_dim, source_kind, jellyfin_item_id, jellyfin_media_source_id, jellyfin_library_id, jellyfin_item_type, jellyfin_tags, jellyfin_etag, trickplay_width, trickplay_tile_width, trickplay_tile_height ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( media_id, source_path_text, title, content_marker, size_bytes, 0, interval_ms, settings.model_id, embedding_dim, "jellyfin", item_id, media_source_id, library_id, str(detail.get("Type") or ""), tags_json, etag, trickplay_width, trickplay_tile_width, trickplay_tile_height, ), ) con.executemany( """ INSERT INTO frames ( id, media_id, frame_number, timestamp_ms, thumb_path, embedding_f16 ) VALUES (?, ?, ?, ?, ?, ?) """, frame_rows, ) surviving_frame_ids = {str(row[0]) for row in frame_rows} preserved = _restore_frame_annotations( con, annotations, surviving_frame_ids, ) preserved_windows = restore_media_caption_windows( con, caption_window_snapshot, surviving_frame_ids, ) preserved_note = "" if preserved or preserved_windows: preserved_note = ( f"; preserved {preserved} frame annotation(s), " f"{preserved_windows} caption window(s)" ) print( f" stored {len(frame_rows)} frames; tags={_display_tags(tags)}" f"{preserved_note}" ) indexed += 1 if report_items is not None: report_items.append( { "status": "indexed", "library_id": library_id, "library": library_name, "item_id": item_id, "title": title, "frames": len(frame_rows), "preserved_annotations": preserved, "preserved_caption_windows": preserved_windows, "reason": "Indexed successfully", } ) except Exception as exc: failed += 1 failed_item_id = str(summary.get("Id") or "") failed_title = str(summary.get("Name") or failed_item_id or "(unnamed)") print( f"[{number}/{len(summaries)}] FAIL " f"{failed_title}: {exc}" ) if report_items is not None: report_items.append( { "status": "failed", "library_id": library_id, "library": library_name, "item_id": failed_item_id, "title": failed_title, "reason": str(exc), } ) if limit is None: reconciled = _reconcile_library( library_id, current_item_ids, library_name=library_name, report_items=report_items, ) pruned += reconciled if reconciled: print(f" pruned {reconciled} stale item(s) no longer present in Jellyfin") else: print(" limited run: stale-item reconciliation skipped") return embedder, indexed, skipped, failed, pruned def index_libraries( client: JellyfinClient, libraries: list[dict], *, embedder: SiglipEmbedder | None = None, limit: int | None = None, force: bool = False, progress: ProgressCallback | None = None, report_items: list[dict[str, object]] | None = None, ) -> tuple[SiglipEmbedder | None, dict]: total_indexed = 0 total_skipped = 0 total_failed = 0 total_pruned = 0 for library_number, library in enumerate(libraries, 1): if progress is not None: progress( { "stage": "library", "library_id": str(library.get("Id") or ""), "library": _library_name(library), "library_number": library_number, "library_total": len(libraries), } ) embedder, indexed, skipped, failed, pruned = _index_library( client, library, embedder=embedder, limit=limit, force=force, progress=progress, report_items=report_items, ) total_indexed += indexed total_skipped += skipped total_failed += failed total_pruned += pruned summary = { "library_total": len(libraries), "indexed": total_indexed, "skipped": total_skipped, "failed": total_failed, "pruned": total_pruned, } if progress is not None: progress({"stage": "indexed", **summary}) return embedder, summary def cmd_jellyfin_index(args: object) -> None: init_db(settings.db_path) client = build_jellyfin_client() libraries = resolve_libraries( client, getattr(args, "library", None), all_libraries=bool(getattr(args, "all_libraries", False)), excluded=getattr(args, "exclude_library", None), ) limit = getattr(args, "limit", None) force = bool(getattr(args, "force", False)) print( f"Selected {len(libraries)} Jellyfin librar" f"{'y' if len(libraries) == 1 else 'ies'}: " + ", ".join(_library_name(library) for library in libraries) ) _, summary = index_libraries( client, libraries, limit=limit, force=force, ) print( f"\nJellyfin indexing complete across {summary['library_total']} librar" f"{'y' if summary['library_total'] == 1 else 'ies'}: " f"{summary['indexed']} indexed, {summary['skipped']} skipped, " f"{summary['failed']} failed, {summary['pruned']} pruned." ) if summary["indexed"] or summary["pruned"]: print("Next: python -m scene_search.cli caption --sync")