from __future__ import annotations import hashlib import io import math import shutil from collections.abc import Callable from pathlib import Path from PIL import Image from .config import settings from .database import connect, init_db, pack_vector from .embedder import SiglipEmbedder from .frames import extract_frames from .jellyfin import ( JellyfinClient, item_tags, jellyfin_media_id, primary_media_source, tags_to_json, ) def build_jellyfin_client() -> JellyfinClient: return JellyfinClient( settings.jellyfin_url, settings.jellyfin_api_key, settings.jellyfin_timeout_seconds, ) def _library_name(library: dict) -> str: return str(library.get("Name") or library.get("Id") or "(unnamed)") ProgressCallback = Callable[[dict], None] def resolve_library_ids( client: JellyfinClient, library_ids: list[str], ) -> list[dict]: requested = {str(value).strip() for value in library_ids if str(value).strip()} if not requested: raise RuntimeError("Select at least one Jellyfin library.") available = client.media_folders() by_id = { str(library.get("Id") or ""): library for library in available if str(library.get("Id") or "") } missing = sorted(requested.difference(by_id)) if missing: raise RuntimeError( "Jellyfin library id(s) not found: " + ", ".join(missing) ) return sorted( [by_id[library_id] for library_id in requested], key=lambda library: _library_name(library).casefold(), ) def resolve_libraries( client: JellyfinClient, requested: list[str] | None, *, all_libraries: bool = False, excluded: list[str] | None = None, ) -> list[dict]: if all_libraries and requested: raise RuntimeError("Use either --all-libraries or --library, not both.") available = client.media_folders() by_name = { _library_name(library).casefold(): library for library in available } excluded_names = { name.casefold() for name in (*settings.jellyfin_exclude_libraries, *(excluded or [])) if name.strip() } if all_libraries: selected = [ library for library in available if _library_name(library).casefold() not in excluded_names ] else: configured = list(settings.jellyfin_libraries) if not configured and settings.jellyfin_library.strip(): configured = [settings.jellyfin_library.strip()] names = requested or configured if not names: raise RuntimeError( "No Jellyfin libraries selected. Repeat --library, use " "--all-libraries, set JELLYFIN_LIBRARIES, or keep using " "the legacy JELLYFIN_LIBRARY setting." ) selected = [] missing = [] seen_ids: set[str] = set() for name in names: key = name.strip().casefold() library = by_name.get(key) if library is None: missing.append(name) continue if key in excluded_names: continue library_id = str(library.get("Id") or "") if library_id and library_id in seen_ids: continue if library_id: seen_ids.add(library_id) selected.append(library) if missing: available_names = ", ".join( sorted(_library_name(library) for library in available) ) raise RuntimeError( f"Jellyfin librar{'y' if len(missing) == 1 else 'ies'} not found: " f"{', '.join(repr(name) for name in missing)}. " f"Available: {available_names or 'none'}" ) if not selected: raise RuntimeError("No Jellyfin libraries remain after exclusions.") return sorted(selected, key=lambda library: _library_name(library).casefold()) def _display_tags(tags: list[str]) -> str: return ", ".join(tags) if tags else "-" def cmd_jellyfin_test(_: object) -> None: client = build_jellyfin_client() info = client.system_info() libraries = client.media_folders() print( f"Connected to Jellyfin {info.get('Version', '?')} " f"({info.get('ServerName') or info.get('ProductName') or 'server'})" ) print(f"Libraries visible to API key: {len(libraries)}") def cmd_jellyfin_libraries(_: object) -> None: client = build_jellyfin_client() libraries = client.media_folders() if not libraries: print("No Jellyfin libraries returned.") return configured = { name.casefold() for name in settings.jellyfin_libraries } if settings.jellyfin_library.strip(): configured.add(settings.jellyfin_library.strip().casefold()) excluded = { name.casefold() for name in settings.jellyfin_exclude_libraries } for item in sorted(libraries, key=lambda row: _library_name(row).casefold()): name = _library_name(item) flags = [] if name.casefold() in configured: flags.append("configured") if name.casefold() in excluded: flags.append("excluded") suffix = f"\t[{', '.join(flags)}]" if flags else "" print(f"{name}\t{item.get('Id')}{suffix}") def cmd_jellyfin_items(args: object) -> None: client = build_jellyfin_client() libraries = resolve_libraries( client, getattr(args, "library", None), all_libraries=bool(getattr(args, "all_libraries", False)), excluded=getattr(args, "exclude_library", None), ) limit = getattr(args, "limit", None) for library in libraries: library_name = _library_name(library) summaries = client.items(str(library["Id"]), limit=limit) print(f"\n=== {library_name} ({len(summaries)} video item(s)) ===") if not summaries: print("No video items found.") continue for summary in summaries: detail = summary tags = item_tags(detail) trickplay = client.select_trickplay( detail, settings.jellyfin_trickplay_width, ) tp = ( f"{trickplay.width}px/{trickplay.interval_ms}ms/" f"{trickplay.thumbnail_count} thumbs" if trickplay else "none" ) print( f"{detail.get('Name') or summary.get('Name') or '(unnamed)'}\n" f" id: {detail.get('Id')}\n" f" type: {detail.get('Type')}\n" f" tags: {_display_tags(tags)}\n" f" trickplay: {tp}\n" f" path: {detail.get('Path') or '-'}" ) def _embed_frames( embedder: SiglipEmbedder, frame_paths: list[Path], media_id: str, interval_ms: int, *, progress: ProgressCallback | None = None, progress_context: dict | None = None, ) -> tuple[list[tuple], int]: rows: list[tuple] = [] dim: int | None = None for start in range(0, len(frame_paths), settings.batch_size): batch_paths = frame_paths[start:start + settings.batch_size] images: list[Image.Image] = [] try: for frame_path in batch_paths: images.append(Image.open(frame_path).convert("RGB")) vectors = embedder.embed_images(images) finally: for image in images: image.close() dim = int(vectors.shape[1]) for offset, (frame_path, vector) in enumerate( zip(batch_paths, vectors, strict=True) ): frame_number = start + offset timestamp_ms = frame_number * interval_ms frame_id = hashlib.sha256( f"{media_id}:{timestamp_ms}:{settings.model_id}".encode() ).hexdigest()[:40] rows.append( ( frame_id, media_id, frame_number, timestamp_ms, str(frame_path), pack_vector(vector), ) ) processed = min(start + len(batch_paths), len(frame_paths)) print(f" embedded {processed}/{len(frame_paths)}") if progress is not None: progress( { **(progress_context or {}), "stage": "embedding", "frames_processed": processed, "frames_total": len(frame_paths), } ) if dim is None: raise RuntimeError("No frames were embedded") return rows, dim def _embed_jellyfin_trickplay( client: JellyfinClient, embedder: SiglipEmbedder, item_id: str, selection, media_id: str, *, progress: ProgressCallback | None = None, progress_context: dict | None = None, ) -> tuple[list[tuple], int]: """Embed Jellyfin Trickplay tiles in memory without persisting duplicate JPEGs.""" rows: list[tuple] = [] dim: int | None = None pending_images: list[Image.Image] = [] pending_numbers: list[int] = [] def flush() -> None: nonlocal dim if not pending_images: return try: vectors = embedder.embed_images(pending_images) dim = int(vectors.shape[1]) for frame_number, vector in zip( pending_numbers, vectors, strict=True, ): timestamp_ms = frame_number * selection.interval_ms frame_id = hashlib.sha256( f"{media_id}:{timestamp_ms}:{settings.model_id}".encode() ).hexdigest()[:40] rows.append( ( frame_id, media_id, frame_number, timestamp_ms, ( f"jellyfin://{item_id}/trickplay/" f"{selection.width}/{frame_number}" ), pack_vector(vector), ) ) finally: for image in pending_images: image.close() pending_images.clear() pending_numbers.clear() if progress is not None: progress( { **(progress_context or {}), "stage": "embedding", "frames_processed": len(rows), "frames_total": selection.thumbnail_count, } ) print(f" embedded {len(rows)}/{selection.thumbnail_count}") capacity = selection.thumbnails_per_sheet sheet_count = math.ceil(selection.thumbnail_count / capacity) for sheet_index in range(sheet_count): raw = client.trickplay_sheet_bytes( item_id, selection, sheet_index, ) with Image.open(io.BytesIO(raw)) as opened: sheet = opened.convert("RGB") try: for local_index in range(capacity): frame_number = sheet_index * capacity + local_index if frame_number >= selection.thumbnail_count: break column = local_index % selection.tile_columns tile_row = local_index // selection.tile_columns left = column * selection.width top = tile_row * selection.height pending_images.append( sheet.crop( ( left, top, left + selection.width, top + selection.height, ) ) ) pending_numbers.append(frame_number) if len(pending_images) >= settings.batch_size: flush() finally: sheet.close() flush() if dim is None: raise RuntimeError("No Jellyfin Trickplay frames were embedded") return rows, dim FRAME_ANNOTATION_COLUMNS = ( "caption", "caption_model", "captioned_at", "caption_route", "caption_route_reason", "route_score", "route_model", "route_policy", "routed_at", ) def _load_frame_annotations(media_id: str) -> dict[str, tuple]: with connect(settings.db_path) as con: rows = con.execute( """ SELECT id, caption, caption_model, captioned_at, caption_route, caption_route_reason, route_score, route_model, route_policy, routed_at FROM frames WHERE media_id = ? """, (media_id,), ).fetchall() return { str(row["id"]): tuple(row[column] for column in FRAME_ANNOTATION_COLUMNS) for row in rows } def _restore_frame_annotations( con, annotations: dict[str, tuple], frame_ids: set[str], ) -> int: matches = [ (*values, frame_id) for frame_id, values in annotations.items() if frame_id in frame_ids ] if not matches: return 0 con.executemany( """ UPDATE frames SET caption = ?, caption_model = ?, captioned_at = ?, caption_route = ?, caption_route_reason = ?, route_score = ?, route_model = ?, route_policy = ?, routed_at = ? WHERE id = ? """, matches, ) return len(matches) def reconcile_selected_libraries( selected_library_ids: set[str], ) -> dict[str, object]: """Make the plugin/dashboard library selection authoritative locally.""" init_db(settings.db_path) selected = { str(value).strip() for value in selected_library_ids if str(value).strip() } removed_library_ids: list[str] = [] removed_media_ids: list[str] = [] removed_items: list[dict[str, object]] = [] with connect(settings.db_path) as con: rows = con.execute( """ SELECT id, title, jellyfin_item_id, jellyfin_library_id FROM media WHERE source_kind = 'jellyfin' AND jellyfin_library_id IS NOT NULL AND jellyfin_library_id != '' """ ).fetchall() stale = [ row for row in rows if str(row["jellyfin_library_id"]) not in selected ] removed_library_ids = sorted( {str(row["jellyfin_library_id"]) for row in stale} ) removed_media_ids = [str(row["id"]) for row in stale] removed_items = [ { "status": "removed", "library_id": str(row["jellyfin_library_id"] or ""), "library": str(row["jellyfin_library_id"] or ""), "item_id": str(row["jellyfin_item_id"] or ""), "title": str(row["title"] or row["jellyfin_item_id"] or row["id"]), "reason": "Library deselected from Scene Search", } for row in stale ] if removed_library_ids: placeholders = ",".join("?" for _ in removed_library_ids) con.execute( f""" DELETE FROM media WHERE source_kind = 'jellyfin' AND jellyfin_library_id IN ({placeholders}) """, removed_library_ids, ) for media_id in removed_media_ids: shutil.rmtree(settings.thumbs_dir / media_id, ignore_errors=True) return { "removed_library_ids": removed_library_ids, "removed_libraries": len(removed_library_ids), "removed_media": len(removed_media_ids), "removed_items": removed_items, } def _reconcile_library( library_id: str, current_item_ids: set[str], *, library_name: str | None = None, report_items: list[dict[str, object]] | None = None, ) -> int: stale_rows = [] with connect(settings.db_path) as con: rows = con.execute( """ SELECT id, title, jellyfin_item_id FROM media WHERE source_kind = 'jellyfin' AND jellyfin_library_id = ? """, (library_id,), ).fetchall() stale_rows = [ row for row in rows if str(row["jellyfin_item_id"] or "") not in current_item_ids ] for row in stale_rows: con.execute("DELETE FROM media WHERE id = ?", (row["id"],)) for row in stale_rows: shutil.rmtree(settings.thumbs_dir / str(row["id"]), ignore_errors=True) if report_items is not None: report_items.append( { "status": "removed", "library_id": library_id, "library": library_name or library_id, "item_id": str(row["jellyfin_item_id"] or ""), "title": str(row["title"] or row["jellyfin_item_id"] or row["id"]), "reason": "Item no longer present in Jellyfin library", } ) return len(stale_rows) def _index_library( client: JellyfinClient, library: dict, *, embedder: SiglipEmbedder | None, limit: int | None, force: bool, progress: ProgressCallback | None = None, report_items: list[dict[str, object]] | None = None, ) -> tuple[SiglipEmbedder | None, int, int, int, int]: library_id = str(library["Id"]) library_name = _library_name(library) summaries = client.items(library_id, limit=limit) print( f"\n=== Jellyfin library {library_name!r}: " f"{len(summaries)} video item(s) selected ===" ) indexed = 0 skipped = 0 failed = 0 pruned = 0 current_item_ids: set[str] = set() if progress is not None: progress( { "stage": "library", "library_id": library_id, "library": library_name, "item_number": 0, "item_total": len(summaries), } ) for number, summary in enumerate(summaries, 1): try: detail = summary item_id = str(detail.get("Id") or summary["Id"]) current_item_ids.add(item_id) title = str(detail.get("Name") or summary.get("Name") or item_id) if progress is not None: progress( { "stage": "item", "library_id": library_id, "library": library_name, "item_id": item_id, "item": title, "item_number": number, "item_total": len(summaries), "frames_processed": 0, "frames_total": 0, } ) tags = item_tags(detail) normalized_tags = {tag.casefold() for tag in tags} if settings.jellyfin_skip_tag and settings.jellyfin_skip_tag in normalized_tags: with connect(settings.db_path) as con: existing = con.execute( "SELECT id FROM media WHERE jellyfin_item_id = ?", (item_id,), ).fetchone() if existing: con.execute("DELETE FROM media WHERE id = ?", (existing["id"],)) pruned += 1 if report_items is not None: report_items.append( { "status": "removed", "library_id": library_id, "library": library_name, "item_id": item_id, "title": title, "reason": "Jellyfin skip tag", } ) shutil.rmtree( settings.thumbs_dir / jellyfin_media_id(item_id), ignore_errors=True, ) print( f"[{number}/{len(summaries)}] SKIP {title}: " "Jellyfin skip tag (removed from canonical index)" ) skipped += 1 continue selection = client.select_trickplay( detail, settings.jellyfin_trickplay_width, ) source = primary_media_source(detail) source_path_text = str( detail.get("Path") or source.get("Path") or f"jellyfin://{item_id}" ) media_id = jellyfin_media_id(item_id) etag = str(detail.get("Etag") or "") content_marker = hashlib.sha256( f"jellyfin:{item_id}:{etag}".encode() ).hexdigest() expected_interval_ms = ( selection.interval_ms if selection else int(round(settings.jellyfin_fallback_interval_seconds * 1000)) ) with connect(settings.db_path) as con: existing = con.execute( "SELECT * FROM media WHERE jellyfin_item_id = ?", (item_id,), ).fetchone() tags_json = tags_to_json(tags) if ( existing and existing["jellyfin_etag"] == etag and existing["jellyfin_tags"] == tags_json and existing["model_id"] == settings.model_id and int(existing["interval_ms"]) == expected_interval_ms and str(existing["jellyfin_library_id"] or "") == library_id and not force ): with connect(settings.db_path) as con: frame_count = int( con.execute( "SELECT COUNT(*) FROM frames WHERE media_id = ?", (existing["id"],), ).fetchone()[0] ) print( f"[{number}/{len(summaries)}] {title}: already indexed " f"({frame_count} frames)" ) skipped += 1 continue out_dir = settings.thumbs_dir / media_id if selection: print( f"[{number}/{len(summaries)}] {title}: Trickplay " f"{selection.width}px every {selection.interval_ms / 1000:g}s " "(Jellyfin-backed, no duplicate thumbnails)" ) interval_ms = selection.interval_ms media_source_id = selection.media_source_id trickplay_width = selection.width trickplay_tile_width = selection.tile_columns trickplay_tile_height = selection.tile_rows else: local_path = Path(source_path_text) if not local_path.is_file(): reason = "No Trickplay and media path is not locally accessible" print( f"[{number}/{len(summaries)}] SKIP {title}: " "no Trickplay and media path is not locally accessible" ) failed += 1 if report_items is not None: report_items.append( { "status": "failed", "library_id": library_id, "library": library_name, "item_id": item_id, "title": title, "reason": reason, } ) continue print( f"[{number}/{len(summaries)}] {title}: no Trickplay; " f"ffmpeg fallback every " f"{settings.jellyfin_fallback_interval_seconds:g}s" ) frame_paths = extract_frames( local_path, out_dir, settings.jellyfin_fallback_interval_seconds, ) interval_ms = int( round(settings.jellyfin_fallback_interval_seconds * 1000) ) media_source_id = str(source.get("Id") or "") or None trickplay_width = None trickplay_tile_width = None trickplay_tile_height = None if selection is None and not frame_paths: reason = "No frames were extracted" print(f"[{number}/{len(summaries)}] SKIP {title}: no frames") failed += 1 if report_items is not None: report_items.append( { "status": "failed", "library_id": library_id, "library": library_name, "item_id": item_id, "title": title, "reason": reason, } ) continue if embedder is None: print(f"Loading {settings.model_id} on {settings.device}...") embedder = SiglipEmbedder(settings.model_id, settings.device) progress_context = { "library_id": library_id, "library": library_name, "item_id": item_id, "item": title, "item_number": number, "item_total": len(summaries), } if selection: frame_rows, embedding_dim = _embed_jellyfin_trickplay( client, embedder, item_id, selection, media_id, progress=progress, progress_context=progress_context, ) else: frame_rows, embedding_dim = _embed_frames( embedder, frame_paths, media_id, interval_ms, progress=progress, progress_context=progress_context, ) size_bytes = int(source.get("Size") or 0) # Preserve AI work when the underlying media still appears to be the # same source. Frame ids are deterministic by media/timestamp/model, # so only exact surviving frames receive the old annotations. annotations: dict[str, tuple] = {} if ( existing and str(existing["source_path"] or "") == source_path_text and int(existing["size_bytes"] or 0) == size_bytes and str(existing["model_id"] or "") == settings.model_id and str(existing["jellyfin_media_source_id"] or "") == str(media_source_id or "") and str(existing["jellyfin_tags"] or "") == tags_json ): annotations = _load_frame_annotations(str(existing["id"])) with connect(settings.db_path) as con: con.execute("DELETE FROM media WHERE id = ?", (media_id,)) con.execute( """ INSERT INTO media ( id, source_path, title, content_sha256, size_bytes, mtime_ns, interval_ms, model_id, embedding_dim, source_kind, jellyfin_item_id, jellyfin_media_source_id, jellyfin_library_id, jellyfin_item_type, jellyfin_tags, jellyfin_etag, trickplay_width, trickplay_tile_width, trickplay_tile_height ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( media_id, source_path_text, title, content_marker, size_bytes, 0, interval_ms, settings.model_id, embedding_dim, "jellyfin", item_id, media_source_id, library_id, str(detail.get("Type") or ""), tags_json, etag, trickplay_width, trickplay_tile_width, trickplay_tile_height, ), ) con.executemany( """ INSERT INTO frames ( id, media_id, frame_number, timestamp_ms, thumb_path, embedding_f16 ) VALUES (?, ?, ?, ?, ?, ?) """, frame_rows, ) preserved = _restore_frame_annotations( con, annotations, {str(row[0]) for row in frame_rows}, ) preserved_note = ( f"; preserved {preserved} caption/route annotation(s)" if preserved else "" ) print( f" stored {len(frame_rows)} frames; tags={_display_tags(tags)}" f"{preserved_note}" ) indexed += 1 if report_items is not None: report_items.append( { "status": "indexed", "library_id": library_id, "library": library_name, "item_id": item_id, "title": title, "frames": len(frame_rows), "preserved_annotations": preserved, "reason": "Indexed successfully", } ) except Exception as exc: failed += 1 failed_item_id = str(summary.get("Id") or "") failed_title = str(summary.get("Name") or failed_item_id or "(unnamed)") print( f"[{number}/{len(summaries)}] FAIL " f"{failed_title}: {exc}" ) if report_items is not None: report_items.append( { "status": "failed", "library_id": library_id, "library": library_name, "item_id": failed_item_id, "title": failed_title, "reason": str(exc), } ) if limit is None: reconciled = _reconcile_library( library_id, current_item_ids, library_name=library_name, report_items=report_items, ) pruned += reconciled if reconciled: print(f" pruned {reconciled} stale item(s) no longer present in Jellyfin") else: print(" limited run: stale-item reconciliation skipped") return embedder, indexed, skipped, failed, pruned def index_libraries( client: JellyfinClient, libraries: list[dict], *, embedder: SiglipEmbedder | None = None, limit: int | None = None, force: bool = False, progress: ProgressCallback | None = None, report_items: list[dict[str, object]] | None = None, ) -> tuple[SiglipEmbedder | None, dict]: total_indexed = 0 total_skipped = 0 total_failed = 0 total_pruned = 0 for library_number, library in enumerate(libraries, 1): if progress is not None: progress( { "stage": "library", "library_id": str(library.get("Id") or ""), "library": _library_name(library), "library_number": library_number, "library_total": len(libraries), } ) embedder, indexed, skipped, failed, pruned = _index_library( client, library, embedder=embedder, limit=limit, force=force, progress=progress, report_items=report_items, ) total_indexed += indexed total_skipped += skipped total_failed += failed total_pruned += pruned summary = { "library_total": len(libraries), "indexed": total_indexed, "skipped": total_skipped, "failed": total_failed, "pruned": total_pruned, } if progress is not None: progress({"stage": "indexed", **summary}) return embedder, summary def cmd_jellyfin_index(args: object) -> None: init_db(settings.db_path) client = build_jellyfin_client() libraries = resolve_libraries( client, getattr(args, "library", None), all_libraries=bool(getattr(args, "all_libraries", False)), excluded=getattr(args, "exclude_library", None), ) limit = getattr(args, "limit", None) force = bool(getattr(args, "force", False)) print( f"Selected {len(libraries)} Jellyfin librar" f"{'y' if len(libraries) == 1 else 'ies'}: " + ", ".join(_library_name(library) for library in libraries) ) _, summary = index_libraries( client, libraries, limit=limit, force=force, ) print( f"\nJellyfin indexing complete across {summary['library_total']} librar" f"{'y' if summary['library_total'] == 1 else 'ies'}: " f"{summary['indexed']} indexed, {summary['skipped']} skipped, " f"{summary['failed']} failed, {summary['pruned']} pruned." ) if summary["indexed"] or summary["pruned"]: print("Next: python -m scene_search.cli caption --sync")