From 7bd405d1b73cd1ffbb83ce5311fcf47b01ecc172 Mon Sep 17 00:00:00 2001 From: Sleeps Date: Tue, 6 Oct 2026 15:34:36 +0100 Subject: [PATCH 01/20] Add Jellyfin media identity and trickplay metadata --- scene_search/database.py | 41 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/scene_search/database.py b/scene_search/database.py index 6faecb5..df4c673 100644 --- a/scene_search/database.py +++ b/scene_search/database.py @@ -21,6 +21,16 @@ CREATE TABLE IF NOT EXISTS media ( interval_ms INTEGER NOT NULL, model_id TEXT NOT NULL, embedding_dim INTEGER NOT NULL, + source_kind TEXT NOT NULL DEFAULT 'file', + jellyfin_item_id TEXT, + jellyfin_media_source_id TEXT, + jellyfin_library_id TEXT, + jellyfin_item_type TEXT, + jellyfin_tags TEXT, + jellyfin_etag TEXT, + trickplay_width INTEGER, + trickplay_tile_width INTEGER, + trickplay_tile_height INTEGER, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP ); @@ -46,6 +56,10 @@ CREATE TABLE IF NOT EXISTS frames ( CREATE INDEX IF NOT EXISTS idx_frames_media_timestamp ON frames(media_id, timestamp_ms); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_media_jellyfin_item + ON media(jellyfin_item_id) + WHERE jellyfin_item_id IS NOT NULL; """ @@ -61,6 +75,32 @@ def connect(path: Path) -> Iterator[sqlite3.Connection]: con.close() +def _ensure_media_jellyfin_columns(con: sqlite3.Connection) -> None: + columns = {row["name"] for row in con.execute("PRAGMA table_info(media)")} + additions = { + "source_kind": "TEXT NOT NULL DEFAULT 'file'", + "jellyfin_item_id": "TEXT", + "jellyfin_media_source_id": "TEXT", + "jellyfin_library_id": "TEXT", + "jellyfin_item_type": "TEXT", + "jellyfin_tags": "TEXT", + "jellyfin_etag": "TEXT", + "trickplay_width": "INTEGER", + "trickplay_tile_width": "INTEGER", + "trickplay_tile_height": "INTEGER", + } + for name, sql_type in additions.items(): + if name not in columns: + con.execute(f"ALTER TABLE media ADD COLUMN {name} {sql_type}") + con.execute( + """ + CREATE UNIQUE INDEX IF NOT EXISTS idx_media_jellyfin_item + ON media(jellyfin_item_id) + WHERE jellyfin_item_id IS NOT NULL + """ + ) + + def _ensure_frame_caption_columns(con: sqlite3.Connection) -> None: columns = {row["name"] for row in con.execute("PRAGMA table_info(frames)")} additions = { @@ -82,6 +122,7 @@ def _ensure_frame_caption_columns(con: sqlite3.Connection) -> None: def init_db(path: Path) -> None: with connect(path) as con: con.executescript(SCHEMA) + _ensure_media_jellyfin_columns(con) _ensure_frame_caption_columns(con) From f226f59cb7c6ed3ee0d7d24fcf4e397d77ad6164 Mon Sep 17 00:00:00 2001 From: Sleeps Date: Tue, 6 Oct 2026 15:34:57 +0100 Subject: [PATCH 02/20] Add Jellyfin API and tag routing configuration --- scene_search/config.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/scene_search/config.py b/scene_search/config.py index ee742e0..1e5d228 100644 --- a/scene_search/config.py +++ b/scene_search/config.py @@ -9,6 +9,15 @@ from dotenv import load_dotenv load_dotenv() +def _csv_env(name: str, default: str = "") -> tuple[str, ...]: + value = os.getenv(name, default) + return tuple( + part.strip().lower() + for part in value.split(",") + if part.strip() + ) + + @dataclass(frozen=True) class Settings: meili_url: str = os.getenv("MEILI_URL", "http://127.0.0.1:7700").rstrip("/") @@ -63,6 +72,27 @@ class Settings: os.getenv("CAPTION_ROUTER_CONTEXT_SECONDS", "6") ) + jellyfin_url: str = os.getenv("JELLYFIN_URL", "").rstrip("/") + jellyfin_api_key: str = os.getenv("JELLYFIN_API_KEY", "") + jellyfin_library: str = os.getenv("JELLYFIN_LIBRARY", "") + jellyfin_trickplay_width: int = int(os.getenv("JELLYFIN_TRICKPLAY_WIDTH", "320")) + jellyfin_timeout_seconds: int = int(os.getenv("JELLYFIN_TIMEOUT_SECONDS", "60")) + jellyfin_specialist_tags: tuple[str, ...] = _csv_env( + "JELLYFIN_SPECIALIST_TAGS" + ) + jellyfin_force_specialist_tag: str = os.getenv( + "JELLYFIN_FORCE_SPECIALIST_TAG", "scene-search:specialist" + ).strip().lower() + jellyfin_force_general_tag: str = os.getenv( + "JELLYFIN_FORCE_GENERAL_TAG", "scene-search:general" + ).strip().lower() + jellyfin_skip_tag: str = os.getenv( + "JELLYFIN_SKIP_TAG", "scene-search:skip" + ).strip().lower() + jellyfin_tag_context: bool = os.getenv( + "JELLYFIN_TAG_CONTEXT", "true" + ).strip().lower() not in {"0", "false", "no", "off"} + data_dir: Path = Path(os.getenv("DATA_DIR", "./data")).resolve() @property @@ -73,7 +103,12 @@ class Settings: def thumbs_dir(self) -> Path: return self.data_dir / "thumbs" + @property + def jellyfin_cache_dir(self) -> Path: + return self.data_dir / "jellyfin" + settings = Settings() settings.data_dir.mkdir(parents=True, exist_ok=True) settings.thumbs_dir.mkdir(parents=True, exist_ok=True) +settings.jellyfin_cache_dir.mkdir(parents=True, exist_ok=True) From 9e5dbc7c8ecbb38933e3ce6015338e51ee893967 Mon Sep 17 00:00:00 2001 From: Sleeps Date: Tue, 6 Oct 2026 15:35:55 +0100 Subject: [PATCH 03/20] Add Jellyfin API and Trickplay client --- scene_search/jellyfin.py | 250 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 250 insertions(+) create mode 100644 scene_search/jellyfin.py diff --git a/scene_search/jellyfin.py b/scene_search/jellyfin.py new file mode 100644 index 0000000..a791221 --- /dev/null +++ b/scene_search/jellyfin.py @@ -0,0 +1,250 @@ +from __future__ import annotations + +import hashlib +import io +import math +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import requests +from PIL import Image + + +@dataclass(frozen=True) +class TrickplaySelection: + media_source_id: str | None + width: int + height: int + interval_ms: int + thumbnail_count: int + tile_columns: int + tile_rows: int + + @property + def thumbnails_per_sheet(self) -> int: + return self.tile_columns * self.tile_rows + + +class JellyfinClient: + def __init__(self, url: str, api_key: str, timeout_seconds: int = 60) -> None: + if not url: + raise RuntimeError("JELLYFIN_URL is not configured") + if not api_key: + raise RuntimeError("JELLYFIN_API_KEY is not configured") + + self.url = url.rstrip("/") + self.timeout_seconds = timeout_seconds + self.session = requests.Session() + self.session.headers.update({ + "X-Emby-Token": api_key, + "Authorization": f'MediaBrowser Token="{api_key}"', + "Accept": "application/json", + }) + + def _get(self, path: str, *, params: dict[str, Any] | None = None) -> requests.Response: + response = self.session.get( + f"{self.url}{path}", + params=params, + timeout=self.timeout_seconds, + ) + if not response.ok: + raise RuntimeError( + f"Jellyfin {response.status_code} GET {path}: {response.text[:500]}" + ) + return response + + def system_info(self) -> dict[str, Any]: + return self._get("/System/Info").json() + + def media_folders(self) -> list[dict[str, Any]]: + data = self._get("/Library/MediaFolders").json() + return list(data.get("Items") or []) + + def find_library(self, name: str) -> dict[str, Any]: + libraries = self.media_folders() + matches = [ + item + for item in libraries + if str(item.get("Name") or "").casefold() == name.casefold() + ] + if not matches: + available = ", ".join( + sorted(str(item.get("Name") or item.get("Id")) for item in libraries) + ) + raise RuntimeError( + f"Jellyfin library {name!r} not found. Available: {available or 'none'}" + ) + return matches[0] + + def items( + self, + library_id: str, + *, + limit: int | None = None, + ) -> list[dict[str, Any]]: + results: list[dict[str, Any]] = [] + start = 0 + page_size = 200 + + while limit is None or len(results) < limit: + request_limit = page_size + if limit is not None: + request_limit = min(request_limit, limit - len(results)) + if request_limit <= 0: + break + + data = self._get( + "/Items", + params={ + "ParentId": library_id, + "Recursive": "true", + "IncludeItemTypes": "Movie,Episode,Video", + "Fields": "Path", + "EnableTotalRecordCount": "true", + "StartIndex": start, + "Limit": request_limit, + }, + ).json() + page = list(data.get("Items") or []) + if not page: + break + + results.extend(page) + start += len(page) + total = int(data.get("TotalRecordCount") or len(results)) + if start >= total or len(page) < request_limit: + break + + return results[:limit] if limit is not None else results + + def item(self, item_id: str) -> dict[str, Any]: + return self._get(f"/Items/{item_id}").json() + + @staticmethod + def select_trickplay( + item: dict[str, Any], + preferred_width: int = 320, + ) -> TrickplaySelection | None: + manifest = item.get("Trickplay") or {} + candidates: list[TrickplaySelection] = [] + + if not isinstance(manifest, dict): + return None + + for media_source_id, widths in manifest.items(): + if not isinstance(widths, dict): + continue + for width_key, raw_info in widths.items(): + if not isinstance(raw_info, dict): + continue + try: + width = int(raw_info.get("Width") or width_key) + height = int(raw_info.get("Height") or 0) + interval_ms = int(raw_info.get("Interval") or 0) + thumbnail_count = int(raw_info.get("ThumbnailCount") or 0) + tile_columns = int(raw_info.get("TileWidth") or 0) + tile_rows = int(raw_info.get("TileHeight") or 0) + except (TypeError, ValueError): + continue + + if min( + width, + height, + interval_ms, + thumbnail_count, + tile_columns, + tile_rows, + ) <= 0: + continue + + candidates.append( + TrickplaySelection( + media_source_id=str(media_source_id) if media_source_id else None, + width=width, + height=height, + interval_ms=interval_ms, + thumbnail_count=thumbnail_count, + tile_columns=tile_columns, + tile_rows=tile_rows, + ) + ) + + if not candidates: + return None + + return min( + candidates, + key=lambda candidate: ( + abs(candidate.width - preferred_width), + -candidate.width, + ), + ) + + def download_trickplay_frames( + self, + item_id: str, + selection: TrickplaySelection, + out_dir: Path, + ) -> list[Path]: + out_dir.mkdir(parents=True, exist_ok=True) + for old in out_dir.glob("frame_*.jpg"): + old.unlink() + + capacity = selection.thumbnails_per_sheet + sheet_count = math.ceil(selection.thumbnail_count / capacity) + paths: list[Path] = [] + + for sheet_index in range(sheet_count): + params: dict[str, Any] = {} + if selection.media_source_id: + params["MediaSourceId"] = selection.media_source_id + + response = self._get( + f"/Videos/{item_id}/Trickplay/{selection.width}/{sheet_index}.jpg", + params=params, + ) + + with Image.open(io.BytesIO(response.content)) as sheet: + sheet = sheet.convert("RGB") + for local_index in range(capacity): + global_index = sheet_index * capacity + local_index + if global_index >= selection.thumbnail_count: + break + + column = local_index % selection.tile_columns + row = local_index // selection.tile_columns + left = column * selection.width + top = row * selection.height + frame = sheet.crop(( + left, + top, + left + selection.width, + top + selection.height, + )) + path = out_dir / f"frame_{global_index + 1:06d}.jpg" + frame.save(path, "JPEG", quality=90) + paths.append(path) + + return paths + + +def jellyfin_media_id(item_id: str) -> str: + return hashlib.sha256(f"jellyfin:{item_id}".encode()).hexdigest()[:32] + + +def item_tags(item: dict[str, Any]) -> list[str]: + tags = item.get("Tags") or [] + if not isinstance(tags, list): + return [] + return [str(tag).strip() for tag in tags if str(tag).strip()] + + +def primary_media_source(item: dict[str, Any]) -> dict[str, Any]: + sources = item.get("MediaSources") or [] + if not isinstance(sources, list): + return {} + for source in sources: + if isinstance(source, dict): + return source + return {} From 6c256f6cf96bd995df267217aebb60a68cdc86e7 Mon Sep 17 00:00:00 2001 From: Sleeps Date: Tue, 6 Oct 2026 15:36:11 +0100 Subject: [PATCH 04/20] Allow Jellyfin tag context in captions --- scene_search/captioner.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/scene_search/captioner.py b/scene_search/captioner.py index 976e8d7..99dba2f 100644 --- a/scene_search/captioner.py +++ b/scene_search/captioner.py @@ -56,8 +56,15 @@ class OllamaCaptioner: self.max_tokens = max_tokens self.session = requests.Session() - def caption(self, image_path: Path) -> str: + def caption(self, image_path: Path, context_hint: str | None = None) -> str: image_b64 = base64.b64encode(image_path.read_bytes()).decode("ascii") + prompt = self.prompt + if context_hint: + prompt = ( + f"{prompt}\n\nMetadata context for this video: {context_hint}\n" + "Use this only as a vocabulary/context hint. Do not claim a tagged " + "detail is visible unless it is actually visible in the frame." + ) response = self.session.post( f"{self.url}/api/chat", json={ @@ -66,7 +73,7 @@ class OllamaCaptioner: "messages": [ { "role": "user", - "content": self.prompt, + "content": prompt, "images": [image_b64], } ], From ba744a3ba2fecd0972af7e69f9be6ca3ec7c2c23 Mon Sep 17 00:00:00 2001 From: Sleeps Date: Tue, 6 Oct 2026 15:36:54 +0100 Subject: [PATCH 05/20] Add Jellyfin tag context and routing helpers --- scene_search/jellyfin.py | 46 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/scene_search/jellyfin.py b/scene_search/jellyfin.py index a791221..3d42714 100644 --- a/scene_search/jellyfin.py +++ b/scene_search/jellyfin.py @@ -2,6 +2,7 @@ from __future__ import annotations import hashlib import io +import json import math from dataclasses import dataclass from pathlib import Path @@ -248,3 +249,48 @@ def primary_media_source(item: dict[str, Any]) -> dict[str, Any]: if isinstance(source, dict): return source return {} + + +def tags_to_json(tags: list[str]) -> str: + return json.dumps(tags, ensure_ascii=False, separators=(",", ":")) + + +def tags_from_json(value: str | None) -> list[str]: + if not value: + return [] + try: + parsed = json.loads(value) + except (TypeError, json.JSONDecodeError): + return [] + if not isinstance(parsed, list): + return [] + return [str(tag).strip() for tag in parsed if str(tag).strip()] + + +def route_override_from_tags( + tags: list[str], + *, + force_specialist_tag: str, + force_general_tag: str, + specialist_tags: tuple[str, ...], +) -> tuple[str, str] | None: + normalized = {tag.casefold() for tag in tags} + if force_specialist_tag and force_specialist_tag.casefold() in normalized: + return ("specialist", "jellyfin-force-tag") + if force_general_tag and force_general_tag.casefold() in normalized: + return ("general", "jellyfin-force-tag") + if normalized.intersection(tag.casefold() for tag in specialist_tags): + return ("specialist", "jellyfin-specialist-tag") + return None + + +def caption_context_from_tags( + tags: list[str], + *, + control_tags: tuple[str, ...] = (), +) -> str | None: + controls = {tag.casefold() for tag in control_tags if tag} + visible_tags = [tag for tag in tags if tag.casefold() not in controls] + if not visible_tags: + return None + return "Jellyfin tags: " + ", ".join(visible_tags) From b40ec58aed30650b609b32a1fa59475244a89c1c Mon Sep 17 00:00:00 2001 From: Sleeps Date: Tue, 6 Oct 2026 15:37:27 +0100 Subject: [PATCH 06/20] Configure Jellyfin ffmpeg fallback interval --- scene_search/config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scene_search/config.py b/scene_search/config.py index 1e5d228..4a59fc1 100644 --- a/scene_search/config.py +++ b/scene_search/config.py @@ -76,6 +76,9 @@ class Settings: jellyfin_api_key: str = os.getenv("JELLYFIN_API_KEY", "") jellyfin_library: str = os.getenv("JELLYFIN_LIBRARY", "") jellyfin_trickplay_width: int = int(os.getenv("JELLYFIN_TRICKPLAY_WIDTH", "320")) + jellyfin_fallback_interval_seconds: float = float( + os.getenv("JELLYFIN_FALLBACK_INTERVAL_SECONDS", "10") + ) jellyfin_timeout_seconds: int = int(os.getenv("JELLYFIN_TIMEOUT_SECONDS", "60")) jellyfin_specialist_tags: tuple[str, ...] = _csv_env( "JELLYFIN_SPECIALIST_TAGS" From 6da2de99bf4458ccc9f307a6078fcd64c5c6675a Mon Sep 17 00:00:00 2001 From: Sleeps Date: Tue, 6 Oct 2026 15:38:25 +0100 Subject: [PATCH 07/20] Add Jellyfin library discovery and Trickplay indexing --- scene_search/jellyfin_index.py | 348 +++++++++++++++++++++++++++++++++ 1 file changed, 348 insertions(+) create mode 100644 scene_search/jellyfin_index.py diff --git a/scene_search/jellyfin_index.py b/scene_search/jellyfin_index.py new file mode 100644 index 0000000..5cab165 --- /dev/null +++ b/scene_search/jellyfin_index.py @@ -0,0 +1,348 @@ +from __future__ import annotations + +import hashlib +from pathlib import Path + +from PIL import Image + +from .config import settings +from .database import connect, init_db, pack_vector +from .embedder import SiglipEmbedder +from .frames import extract_frames +from .jellyfin import ( + JellyfinClient, + item_tags, + jellyfin_media_id, + primary_media_source, + tags_to_json, +) + + +def build_jellyfin_client() -> JellyfinClient: + return JellyfinClient( + settings.jellyfin_url, + settings.jellyfin_api_key, + settings.jellyfin_timeout_seconds, + ) + + +def resolve_library_name(requested: str | None) -> str: + name = (requested or settings.jellyfin_library).strip() + if not name: + raise RuntimeError( + "No Jellyfin library selected. Pass --library or set JELLYFIN_LIBRARY." + ) + return name + + +def _display_tags(tags: list[str]) -> str: + return ", ".join(tags) if tags else "-" + + +def cmd_jellyfin_test(_: object) -> None: + client = build_jellyfin_client() + info = client.system_info() + libraries = client.media_folders() + print( + f"Connected to Jellyfin {info.get('Version', '?')} " + f"({info.get('ServerName') or info.get('ProductName') or 'server'})" + ) + print(f"Libraries visible to API key: {len(libraries)}") + + +def cmd_jellyfin_libraries(_: object) -> None: + client = build_jellyfin_client() + libraries = client.media_folders() + if not libraries: + print("No Jellyfin libraries returned.") + return + + for item in sorted(libraries, key=lambda row: str(row.get("Name") or "")): + print(f"{item.get('Name') or '(unnamed)'}\t{item.get('Id')}") + + +def cmd_jellyfin_items(args: object) -> None: + client = build_jellyfin_client() + library_name = resolve_library_name(getattr(args, "library", None)) + library = client.find_library(library_name) + summaries = client.items(str(library["Id"]), limit=getattr(args, "limit", None)) + + if not summaries: + print(f"No video items found in {library_name!r}.") + return + + for summary in summaries: + detail = client.item(str(summary["Id"])) + tags = item_tags(detail) + trickplay = client.select_trickplay(detail, settings.jellyfin_trickplay_width) + tp = ( + f"{trickplay.width}px/{trickplay.interval_ms}ms/" + f"{trickplay.thumbnail_count} thumbs" + if trickplay + else "none" + ) + print( + f"{detail.get('Name') or summary.get('Name') or '(unnamed)'}\n" + f" id: {detail.get('Id')}\n" + f" type: {detail.get('Type')}\n" + f" tags: {_display_tags(tags)}\n" + f" trickplay: {tp}\n" + f" path: {detail.get('Path') or '-'}" + ) + + +def _embed_frames( + embedder: SiglipEmbedder, + frame_paths: list[Path], + media_id: str, + interval_ms: int, +) -> tuple[list[tuple], int]: + rows: list[tuple] = [] + dim: int | None = None + + for start in range(0, len(frame_paths), settings.batch_size): + batch_paths = frame_paths[start:start + settings.batch_size] + images: list[Image.Image] = [] + try: + for frame_path in batch_paths: + images.append(Image.open(frame_path).convert("RGB")) + vectors = embedder.embed_images(images) + finally: + for image in images: + image.close() + + dim = int(vectors.shape[1]) + for offset, (frame_path, vector) in enumerate( + zip(batch_paths, vectors, strict=True) + ): + frame_number = start + offset + timestamp_ms = frame_number * interval_ms + frame_id = hashlib.sha256( + f"{media_id}:{timestamp_ms}:{settings.model_id}".encode() + ).hexdigest()[:40] + rows.append( + ( + frame_id, + media_id, + frame_number, + timestamp_ms, + str(frame_path), + pack_vector(vector), + ) + ) + + print( + f" embedded " + f"{min(start + len(batch_paths), len(frame_paths))}/{len(frame_paths)}" + ) + + if dim is None: + raise RuntimeError("No frames were embedded") + return rows, dim + + +def cmd_jellyfin_index(args: object) -> None: + init_db(settings.db_path) + client = build_jellyfin_client() + library_name = resolve_library_name(getattr(args, "library", None)) + library = client.find_library(library_name) + summaries = client.items(str(library["Id"]), limit=getattr(args, "limit", None)) + force = bool(getattr(args, "force", False)) + + print( + f"Jellyfin library {library_name!r}: {len(summaries)} video item(s) selected" + ) + + embedder: SiglipEmbedder | None = None + indexed = 0 + skipped = 0 + failed = 0 + + for number, summary in enumerate(summaries, 1): + try: + detail = client.item(str(summary["Id"])) + item_id = str(detail.get("Id") or summary["Id"]) + title = str(detail.get("Name") or summary.get("Name") or item_id) + tags = item_tags(detail) + normalized_tags = {tag.casefold() for tag in tags} + + if settings.jellyfin_skip_tag and settings.jellyfin_skip_tag in normalized_tags: + print(f"[{number}/{len(summaries)}] SKIP {title}: Jellyfin skip tag") + skipped += 1 + continue + + selection = client.select_trickplay( + detail, + settings.jellyfin_trickplay_width, + ) + source = primary_media_source(detail) + source_path_text = str( + detail.get("Path") + or source.get("Path") + or f"jellyfin://{item_id}" + ) + media_id = jellyfin_media_id(item_id) + etag = str(detail.get("Etag") or "") + content_marker = hashlib.sha256( + f"jellyfin:{item_id}:{etag}".encode() + ).hexdigest() + + expected_interval_ms = ( + selection.interval_ms + if selection + else int(round(settings.jellyfin_fallback_interval_seconds * 1000)) + ) + + with connect(settings.db_path) as con: + existing = con.execute( + "SELECT * FROM media WHERE jellyfin_item_id = ?", + (item_id,), + ).fetchone() + + if ( + existing + and existing["jellyfin_etag"] == etag + and existing["model_id"] == settings.model_id + and int(existing["interval_ms"]) == expected_interval_ms + and not force + ): + frame_count = 0 + with connect(settings.db_path) as con: + frame_count = int( + con.execute( + "SELECT COUNT(*) FROM frames WHERE media_id = ?", + (existing["id"],), + ).fetchone()[0] + ) + print( + f"[{number}/{len(summaries)}] {title}: already indexed " + f"({frame_count} frames)" + ) + skipped += 1 + continue + + out_dir = settings.thumbs_dir / media_id + if selection: + print( + f"[{number}/{len(summaries)}] {title}: Trickplay " + f"{selection.width}px every {selection.interval_ms / 1000:g}s" + ) + frame_paths = client.download_trickplay_frames( + item_id, + selection, + out_dir, + ) + interval_ms = selection.interval_ms + media_source_id = selection.media_source_id + trickplay_width = selection.width + trickplay_tile_width = selection.tile_columns + trickplay_tile_height = selection.tile_rows + else: + local_path = Path(source_path_text) + if not local_path.is_file(): + print( + f"[{number}/{len(summaries)}] SKIP {title}: " + "no Trickplay and media path is not locally accessible" + ) + failed += 1 + continue + + print( + f"[{number}/{len(summaries)}] {title}: no Trickplay; " + f"ffmpeg fallback every " + f"{settings.jellyfin_fallback_interval_seconds:g}s" + ) + frame_paths = extract_frames( + local_path, + out_dir, + settings.jellyfin_fallback_interval_seconds, + ) + interval_ms = int( + round(settings.jellyfin_fallback_interval_seconds * 1000) + ) + media_source_id = str(source.get("Id") or "") or None + trickplay_width = None + trickplay_tile_width = None + trickplay_tile_height = None + + if not frame_paths: + print(f"[{number}/{len(summaries)}] SKIP {title}: no frames") + failed += 1 + continue + + if embedder is None: + print(f"Loading {settings.model_id} on {settings.device}...") + embedder = SiglipEmbedder(settings.model_id, settings.device) + + frame_rows, embedding_dim = _embed_frames( + embedder, + frame_paths, + media_id, + interval_ms, + ) + + size_bytes = int(source.get("Size") or 0) + with connect(settings.db_path) as con: + con.execute("DELETE FROM media WHERE id = ?", (media_id,)) + con.execute( + """ + INSERT INTO media ( + id, source_path, title, content_sha256, size_bytes, mtime_ns, + interval_ms, model_id, embedding_dim, source_kind, + jellyfin_item_id, jellyfin_media_source_id, jellyfin_library_id, + jellyfin_item_type, jellyfin_tags, jellyfin_etag, + trickplay_width, trickplay_tile_width, trickplay_tile_height + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + media_id, + source_path_text, + title, + content_marker, + size_bytes, + 0, + interval_ms, + settings.model_id, + embedding_dim, + "jellyfin", + item_id, + media_source_id, + str(library["Id"]), + str(detail.get("Type") or ""), + tags_to_json(tags), + etag, + trickplay_width, + trickplay_tile_width, + trickplay_tile_height, + ), + ) + con.executemany( + """ + INSERT INTO frames ( + id, media_id, frame_number, timestamp_ms, + thumb_path, embedding_f16 + ) + VALUES (?, ?, ?, ?, ?, ?) + """, + frame_rows, + ) + + print( + f" stored {len(frame_rows)} frames; tags={_display_tags(tags)}" + ) + indexed += 1 + + except Exception as exc: + failed += 1 + print( + f"[{number}/{len(summaries)}] FAIL " + f"{summary.get('Name') or summary.get('Id')}: {exc}" + ) + + print( + f"Jellyfin indexing complete: {indexed} indexed, " + f"{skipped} skipped, {failed} failed." + ) + if indexed: + print("Next: python -m scene_search.cli caption --sync") From 275cfa2e402b83603d7838362814693f9940d80f Mon Sep 17 00:00:00 2001 From: Sleeps Date: Tue, 6 Oct 2026 15:39:15 +0100 Subject: [PATCH 08/20] Connect Jellyfin ingestion, tags and search metadata --- scene_search/cli.py | 77 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 72 insertions(+), 5 deletions(-) diff --git a/scene_search/cli.py b/scene_search/cli.py index 0a199ca..422f404 100644 --- a/scene_search/cli.py +++ b/scene_search/cli.py @@ -15,6 +15,17 @@ from .config import settings from .database import connect, init_db, pack_vector, unpack_vector from .embedder import SiglipEmbedder from .frames import extract_frames, media_id_for, sha256_file +from .jellyfin import ( + caption_context_from_tags, + route_override_from_tags, + tags_from_json, +) +from .jellyfin_index import ( + cmd_jellyfin_index, + cmd_jellyfin_items, + cmd_jellyfin_libraries, + cmd_jellyfin_test, +) from .meili import Meili from .routing import ExplicitContentRouter, apply_context_routing @@ -146,7 +157,10 @@ def cmd_sync_meili(_: argparse.Namespace) -> None: f.caption, f.caption_model, f.captioned_at, f.caption_route, f.caption_route_reason, f.route_score, f.route_model, f.route_policy, f.routed_at, - m.title, m.source_path, m.embedding_dim, m.model_id + m.title, m.source_path, m.embedding_dim, m.model_id, + m.source_kind, m.jellyfin_item_id, m.jellyfin_media_source_id, + m.jellyfin_library_id, m.jellyfin_item_type, m.jellyfin_tags, + m.trickplay_width FROM frames f JOIN media m ON m.id = f.media_id ORDER BY f.id @@ -178,6 +192,13 @@ def cmd_sync_meili(_: argparse.Namespace) -> None: "route_model": row["route_model"], "route_policy": row["route_policy"], "routed_at": row["routed_at"], + "source_kind": row["source_kind"], + "jellyfin_item_id": row["jellyfin_item_id"], + "jellyfin_media_source_id": row["jellyfin_media_source_id"], + "jellyfin_library_id": row["jellyfin_library_id"], + "jellyfin_item_type": row["jellyfin_item_type"], + "jellyfin_tags": tags_from_json(row["jellyfin_tags"]), + "trickplay_width": row["trickplay_width"], "_vectors": {"siglip2": vec.tolist()}, }) meili.add_documents(docs) @@ -189,12 +210,20 @@ def cmd_sync_meili(_: argparse.Namespace) -> None: def _route_policy() -> str: + tag_policy = ( + f"force-specialist={settings.jellyfin_force_specialist_tag}|" + f"force-general={settings.jellyfin_force_general_tag}|" + f"specialist-tags={','.join(settings.jellyfin_specialist_tags)}" + ) if not settings.caption_router_enabled: - return f"disabled|general={settings.caption_general_model}" + return ( + f"disabled|general={settings.caption_general_model}|{tag_policy}" + ) return ( f"model={settings.caption_router_model}|" f"threshold={settings.caption_router_threshold:.6f}|" - f"context={settings.caption_router_context_seconds:.3f}" + f"context={settings.caption_router_context_seconds:.3f}|" + f"{tag_policy}" ) @@ -237,7 +266,7 @@ def _select_caption_rows( f""" SELECT f.id, f.media_id, f.timestamp_ms, f.thumb_path, f.caption, f.caption_model, f.caption_route, f.route_score, f.route_policy, - m.title + m.title, m.jellyfin_tags FROM frames f JOIN media m ON m.id = f.media_id {where} @@ -306,6 +335,17 @@ def _route_rows(rows: list, *, verbose: bool = False) -> tuple[list[tuple], int] } route_model = "disabled" + for row in valid_rows: + tags = tags_from_json(row["jellyfin_tags"]) + override = route_override_from_tags( + tags, + force_specialist_tag=settings.jellyfin_force_specialist_tag, + force_general_tag=settings.jellyfin_force_general_tag, + specialist_tags=settings.jellyfin_specialist_tags, + ) + if override is not None: + decisions[str(row["id"])] = override + routed: list[tuple] = [] with connect(settings.db_path) as con: for row in valid_rows: @@ -380,7 +420,17 @@ def _caption_queue( for row, route, score, reason in routed_rows: image_path = Path(row["thumb_path"]) try: - caption = captioner.caption(image_path) + context_hint = None + if settings.jellyfin_tag_context: + context_hint = caption_context_from_tags( + tags_from_json(row["jellyfin_tags"]), + control_tags=( + settings.jellyfin_force_specialist_tag, + settings.jellyfin_force_general_tag, + settings.jellyfin_skip_tag, + ), + ) + caption = captioner.caption(image_path, context_hint=context_hint) except Exception as exc: failed += 1 print( @@ -648,6 +698,23 @@ def make_parser() -> argparse.ArgumentParser: sync = sub.add_parser("sync-meili", help="Rebuild/feed Meilisearch from canonical SQLite vectors") sync.set_defaults(func=cmd_sync_meili) + jf_test = sub.add_parser("jellyfin-test", help="Verify Jellyfin API connectivity") + jf_test.set_defaults(func=cmd_jellyfin_test) + + jf_libs = sub.add_parser("jellyfin-libraries", help="List Jellyfin media libraries") + jf_libs.set_defaults(func=cmd_jellyfin_libraries) + + jf_items = sub.add_parser("jellyfin-items", help="Inspect video items, tags and Trickplay") + jf_items.add_argument("--library", help="Jellyfin library name (or use JELLYFIN_LIBRARY)") + jf_items.add_argument("--limit", type=int, help="Maximum number of items to inspect") + jf_items.set_defaults(func=cmd_jellyfin_items) + + jf_index = sub.add_parser("jellyfin-index", help="Index a Jellyfin library via Trickplay") + jf_index.add_argument("--library", help="Jellyfin library name (or use JELLYFIN_LIBRARY)") + jf_index.add_argument("--limit", type=int, help="Maximum number of items to index") + jf_index.add_argument("--force", action="store_true", help="Rebuild matching Jellyfin items") + jf_index.set_defaults(func=cmd_jellyfin_index) + route = sub.add_parser("route", help="Classify frames and show which caption model they will use") route.add_argument("--media-id", help="Only route frames from one media ID") route.add_argument("--title", help="Only route frames from one exact media title") From 6f185fd7adfbfca42a35ec15c81364ea94e86dee Mon Sep 17 00:00:00 2001 From: Sleeps Date: Tue, 6 Oct 2026 15:39:58 +0100 Subject: [PATCH 09/20] Run Jellyfin index creation after column migration --- scene_search/database.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scene_search/database.py b/scene_search/database.py index df4c673..5665e73 100644 --- a/scene_search/database.py +++ b/scene_search/database.py @@ -56,10 +56,6 @@ CREATE TABLE IF NOT EXISTS frames ( CREATE INDEX IF NOT EXISTS idx_frames_media_timestamp ON frames(media_id, timestamp_ms); - -CREATE UNIQUE INDEX IF NOT EXISTS idx_media_jellyfin_item - ON media(jellyfin_item_id) - WHERE jellyfin_item_id IS NOT NULL; """ From f789ebc5b8a176d8584dddc0022c9d4c5e884c43 Mon Sep 17 00:00:00 2001 From: Sleeps Date: Tue, 6 Oct 2026 15:40:11 +0100 Subject: [PATCH 10/20] Index Jellyfin metadata and tags in Meilisearch --- scene_search/meili.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/scene_search/meili.py b/scene_search/meili.py index 85b4e4a..b601ba4 100644 --- a/scene_search/meili.py +++ b/scene_search/meili.py @@ -42,11 +42,22 @@ class Meili: )) self._check(self.session.put( f"{self.url}/indexes/{self.index}/settings/filterable-attributes", - json=["media_id", "title", "caption_model", "caption_route", "route_model"], + json=[ + "media_id", + "title", + "caption_model", + "caption_route", + "route_model", + "source_kind", + "jellyfin_item_id", + "jellyfin_library_id", + "jellyfin_item_type", + "jellyfin_tags", + ], )) self._check(self.session.put( f"{self.url}/indexes/{self.index}/settings/searchable-attributes", - json=["title", "caption"], + json=["title", "caption", "jellyfin_tags"], )) def add_documents(self, documents: list[dict]) -> None: From 12327f09e68bbb15b035b860548df518ea67b02f Mon Sep 17 00:00:00 2001 From: Sleeps Date: Tue, 6 Oct 2026 15:40:39 +0100 Subject: [PATCH 11/20] Show Jellyfin item identity in search results --- scene_search/cli.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scene_search/cli.py b/scene_search/cli.py index 422f404..c544c05 100644 --- a/scene_search/cli.py +++ b/scene_search/cli.py @@ -678,6 +678,11 @@ def cmd_search(args: argparse.Namespace) -> None: else "" ) print(f" route={route}{score_text}") + jellyfin_item_id = hit.get("jellyfin_item_id") + if jellyfin_item_id: + tags = hit.get("jellyfin_tags") or [] + tags_text = f" tags={', '.join(tags)}" if tags else "" + print(f" jellyfin={jellyfin_item_id}{tags_text}") print(f" {hit.get('thumb_path')}") From 20ccb79203ad9b8f05011d5820077f65e156519d Mon Sep 17 00:00:00 2001 From: Sleeps Date: Tue, 6 Oct 2026 15:40:57 +0100 Subject: [PATCH 12/20] Document Jellyfin ingestion settings --- .env.example | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.env.example b/.env.example index a53701c..764d826 100644 --- a/.env.example +++ b/.env.example @@ -22,3 +22,22 @@ CAPTION_ROUTER_DEVICE=cuda CAPTION_ROUTER_BATCH_SIZE=32 CAPTION_ROUTER_THRESHOLD=0.50 CAPTION_ROUTER_CONTEXT_SECONDS=6 + +# Jellyfin ingestion +JELLYFIN_URL=http://127.0.0.1:8096 +JELLYFIN_API_KEY= +JELLYFIN_LIBRARY= +JELLYFIN_TRICKPLAY_WIDTH=320 +JELLYFIN_FALLBACK_INTERVAL_SECONDS=10 +JELLYFIN_TIMEOUT_SECONDS=60 + +# Ordinary Jellyfin tags are passed to the captioner as context hints. +JELLYFIN_TAG_CONTEXT=true + +# Optional comma-separated tags that should always use the specialist captioner. +JELLYFIN_SPECIALIST_TAGS= + +# Reserved control tags. Add these directly to a Jellyfin item when needed. +JELLYFIN_FORCE_SPECIALIST_TAG=scene-search:specialist +JELLYFIN_FORCE_GENERAL_TAG=scene-search:general +JELLYFIN_SKIP_TAG=scene-search:skip From 9bb1e84fb40ae4c82cfd22091e67b72fae479c3f Mon Sep 17 00:00:00 2001 From: Sleeps Date: Tue, 6 Oct 2026 15:41:36 +0100 Subject: [PATCH 13/20] Test Jellyfin schema migration on existing database --- tests/test_database.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/tests/test_database.py b/tests/test_database.py index f95a0ee..ddbfb0b 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -44,18 +44,28 @@ class DatabaseMigrationTests(unittest.TestCase): init_db(path) con = sqlite3.connect(path) - columns = {row[1] for row in con.execute("PRAGMA table_info(frames)")} + frame_columns = {row[1] for row in con.execute("PRAGMA table_info(frames)")} + media_columns = {row[1] for row in con.execute("PRAGMA table_info(media)")} + indexes = {row[1] for row in con.execute("PRAGMA index_list(media)")} con.close() - self.assertIn("caption", columns) - self.assertIn("caption_model", columns) - self.assertIn("captioned_at", columns) - self.assertIn("caption_route", columns) - self.assertIn("caption_route_reason", columns) - self.assertIn("route_score", columns) - self.assertIn("route_model", columns) - self.assertIn("route_policy", columns) - self.assertIn("routed_at", columns) + self.assertIn("caption", frame_columns) + self.assertIn("caption_model", frame_columns) + self.assertIn("captioned_at", frame_columns) + self.assertIn("caption_route", frame_columns) + self.assertIn("caption_route_reason", frame_columns) + self.assertIn("route_score", frame_columns) + self.assertIn("route_model", frame_columns) + self.assertIn("route_policy", frame_columns) + self.assertIn("routed_at", frame_columns) + + self.assertIn("source_kind", media_columns) + self.assertIn("jellyfin_item_id", media_columns) + self.assertIn("jellyfin_media_source_id", media_columns) + self.assertIn("jellyfin_library_id", media_columns) + self.assertIn("jellyfin_tags", media_columns) + self.assertIn("trickplay_width", media_columns) + self.assertIn("idx_media_jellyfin_item", indexes) if __name__ == "__main__": From 495174d9900e827830d01b8754a17961bb8da94e Mon Sep 17 00:00:00 2001 From: Sleeps Date: Tue, 6 Oct 2026 15:41:39 +0100 Subject: [PATCH 14/20] Test metadata context in caption prompts --- tests/test_captioner.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_captioner.py b/tests/test_captioner.py index 753f4d7..9f1a797 100644 --- a/tests/test_captioner.py +++ b/tests/test_captioner.py @@ -21,13 +21,15 @@ class OllamaCaptionerTests(unittest.TestCase): with tempfile.TemporaryDirectory() as tmp: image = Path(tmp) / "frame.jpg" image.write_bytes(b"fake-jpeg") - result = captioner.caption(image) + result = captioner.caption(image, context_hint="Jellyfin tags: beach, holiday") self.assertEqual(result, "A person walking down a street.") call = captioner.session.post.call_args payload = call.kwargs["json"] self.assertEqual(payload["model"], "vision-test") self.assertEqual(payload["messages"][0]["role"], "user") + self.assertIn("Jellyfin tags: beach, holiday", payload["messages"][0]["content"]) + self.assertIn("actually visible", payload["messages"][0]["content"]) self.assertTrue(payload["messages"][0]["images"][0]) self.assertEqual(payload["options"]["temperature"], 0.0) self.assertEqual(payload["options"]["num_predict"], 120) From af72b3c032ece2f3604ae8da36d5c2c5acdf9f1e Mon Sep 17 00:00:00 2001 From: Sleeps Date: Tue, 6 Oct 2026 15:42:03 +0100 Subject: [PATCH 15/20] Test Jellyfin Trickplay and tag integration --- tests/test_jellyfin.py | 113 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 tests/test_jellyfin.py diff --git a/tests/test_jellyfin.py b/tests/test_jellyfin.py new file mode 100644 index 0000000..5edc002 --- /dev/null +++ b/tests/test_jellyfin.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import io +import tempfile +import unittest +from pathlib import Path +from unittest.mock import Mock + +from PIL import Image + +from scene_search.jellyfin import ( + JellyfinClient, + TrickplaySelection, + caption_context_from_tags, + route_override_from_tags, +) + + +class JellyfinIntegrationTests(unittest.TestCase): + def test_select_trickplay_prefers_nearest_width(self) -> None: + item = { + "Trickplay": { + "source-1": { + "160": { + "Width": 160, + "Height": 90, + "Interval": 10000, + "ThumbnailCount": 20, + "TileWidth": 5, + "TileHeight": 4, + }, + "320": { + "Width": 320, + "Height": 180, + "Interval": 10000, + "ThumbnailCount": 20, + "TileWidth": 5, + "TileHeight": 4, + }, + } + } + } + + selected = JellyfinClient.select_trickplay(item, preferred_width=300) + + self.assertIsNotNone(selected) + assert selected is not None + self.assertEqual(selected.media_source_id, "source-1") + self.assertEqual(selected.width, 320) + self.assertEqual(selected.interval_ms, 10000) + self.assertEqual(selected.thumbnails_per_sheet, 20) + + def test_download_trickplay_frames_splits_sheet(self) -> None: + client = JellyfinClient("http://jellyfin.test", "token") + selection = TrickplaySelection( + media_source_id="source-1", + width=10, + height=5, + interval_ms=10000, + thumbnail_count=4, + tile_columns=2, + tile_rows=2, + ) + + sheet = Image.new("RGB", (20, 10), "white") + payload = io.BytesIO() + sheet.save(payload, "JPEG") + response = Mock() + response.ok = True + response.content = payload.getvalue() + client._get = Mock(return_value=response) + + with tempfile.TemporaryDirectory() as tmp: + paths = client.download_trickplay_frames( + "item-1", + selection, + Path(tmp), + ) + self.assertEqual(len(paths), 4) + self.assertTrue(all(path.is_file() for path in paths)) + + call = client._get.call_args + self.assertIn("/Videos/item-1/Trickplay/10/0.jpg", call.args[0]) + self.assertEqual(call.kwargs["params"]["MediaSourceId"], "source-1") + + def test_jellyfin_tags_can_force_specialist_route(self) -> None: + override = route_override_from_tags( + ["Archive", "scene-search:specialist"], + force_specialist_tag="scene-search:specialist", + force_general_tag="scene-search:general", + specialist_tags=(), + ) + self.assertEqual(override, ("specialist", "jellyfin-force-tag")) + + def test_configured_specialist_tag_routes_to_specialist(self) -> None: + override = route_override_from_tags( + ["Custom Category"], + force_specialist_tag="scene-search:specialist", + force_general_tag="scene-search:general", + specialist_tags=("custom category",), + ) + self.assertEqual(override, ("specialist", "jellyfin-specialist-tag")) + + def test_control_tags_are_not_added_to_caption_context(self) -> None: + context = caption_context_from_tags( + ["holiday", "scene-search:specialist", "beach"], + control_tags=("scene-search:specialist",), + ) + self.assertEqual(context, "Jellyfin tags: holiday, beach") + + +if __name__ == "__main__": + unittest.main() From b505dd0205d904c2d37491a138deae09ba9710e6 Mon Sep 17 00:00:00 2001 From: Sleeps Date: Tue, 6 Oct 2026 15:44:08 +0100 Subject: [PATCH 16/20] Document Jellyfin test-library ingestion workflow --- README.md | 95 +++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 88 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 78c4259..8036fd5 100644 --- a/README.md +++ b/README.md @@ -163,7 +163,88 @@ Caption work is grouped by route. The general queue is processed first, its Olla On the first run after enabling routing, existing captions are classified. A caption that already came from the model selected by the new route is retained; only frames whose selected model changes are regenerated unless `--force` is used. -## 7. Hybrid search +## 7. Jellyfin ingestion + +Jellyfin can now be the source of truth for media identity, metadata and frame sampling. + +Configure a server API key and a small test library: + +```env +JELLYFIN_URL=http://jellyfin-server:8096 +JELLYFIN_API_KEY=replace-me +JELLYFIN_LIBRARY=Scene Search Test + +JELLYFIN_TRICKPLAY_WIDTH=320 +JELLYFIN_FALLBACK_INTERVAL_SECONDS=10 +JELLYFIN_TIMEOUT_SECONDS=60 + +JELLYFIN_TAG_CONTEXT=true +JELLYFIN_SPECIALIST_TAGS= + +JELLYFIN_FORCE_SPECIALIST_TAG=scene-search:specialist +JELLYFIN_FORCE_GENERAL_TAG=scene-search:general +JELLYFIN_SKIP_TAG=scene-search:skip +``` + +Verify connectivity: + +```bash +python -m scene_search.cli jellyfin-test +python -m scene_search.cli jellyfin-libraries +``` + +Inspect the test library before doing any GPU work: + +```bash +python -m scene_search.cli jellyfin-items --library "Scene Search Test" --limit 10 +``` + +This prints each Jellyfin item ID, type, tags, path and available Trickplay information. + +Index the library: + +```bash +python -m scene_search.cli jellyfin-index --library "Scene Search Test" +``` + +The indexer prefers Jellyfin's existing Trickplay sheets. It downloads each sheet, splits the sheet into timestamped thumbnails using Jellyfin's Trickplay manifest, embeds those thumbnails with SigLIP and stores the Jellyfin item/media-source identity alongside the canonical vectors. + +If an item has no Trickplay manifest, the indexer falls back to ffmpeg only when Jellyfin's reported media path is accessible from the indexing machine. + +After indexing: + +```bash +python -m scene_search.cli route +python -m scene_search.cli caption --sync +``` + +### Jellyfin tags + +Normal Jellyfin tags are supplied to the caption model as metadata context. They are hints only: the prompt explicitly tells the model not to claim a tagged detail unless that detail is actually visible in the current frame. + +Tags also participate in routing: + +- `scene-search:specialist` forces every frame from that item through the specialist JoyCaption model. +- `scene-search:general` forces the general Qwen captioner. +- `scene-search:skip` prevents the item from being indexed. +- `JELLYFIN_SPECIALIST_TAGS` can contain a comma-separated list of your existing Jellyfin tags that should automatically use JoyCaption. + +For example, if your existing library has specific descriptive tags that are useful vocabulary for specialist material, add those exact tag names to `JELLYFIN_SPECIALIST_TAGS`. The original tag text is still passed to the specialist captioner as context. + +Jellyfin-backed media stores: + +- Jellyfin item ID +- media source ID +- library ID +- item type +- tags +- item ETag +- selected Trickplay width and sheet layout +- source path when Jellyfin exposes one + +These fields are also copied into Meilisearch. Search results therefore retain enough identity to build the Jellyfin UI integration next. + +## 8. Hybrid search After captions exist, sync them: @@ -230,11 +311,11 @@ Meilisearch does not normally display the contents of `_vectors` in its Mini Das If caption + vector retrieval proves useful: -1. group neighbouring frames into scenes/windows to remove duplicate results; -2. caption one scene/window rather than every frame; -3. replace local ffmpeg frames with Jellyfin Trickplay tiles; -4. store Jellyfin item/media-source IDs and Trickplay coordinates; -5. add subtitles/transcripts as another searchable field; -6. return Jellyfin results that jump directly to the matched timestamp. +1. group neighbouring Trickplay thumbnails into scenes/windows to remove duplicate results; +2. caption one scene/window rather than every thumbnail; +3. add subtitles/transcripts as another searchable field; +4. add the Jellyfin search-provider/plugin layer; +5. return Jellyfin results that jump directly to the matched timestamp; +6. enforce the requesting Jellyfin user's library permissions before returning results. The routing classifier is not used as user-facing metadata or a library-wide content label; it exists only to select the more suitable captioner for each frame. From 0c43cb8338a8cacac605a875a514caffd34f5887 Mon Sep 17 00:00:00 2001 From: Sleeps Date: Tue, 6 Oct 2026 15:46:03 +0100 Subject: [PATCH 17/20] Make Meili sync authoritative and accept tagged specialist captions --- scene_search/cli.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scene_search/cli.py b/scene_search/cli.py index c544c05..9476087 100644 --- a/scene_search/cli.py +++ b/scene_search/cli.py @@ -150,6 +150,7 @@ def cmd_sync_meili(_: argparse.Namespace) -> None: raise SystemExit(f"Multiple embedding dimensions found: {dims}. Use separate indexes per model.") dim = int(dims[0]) meili.configure_user_vectors(dim) + meili.clear_documents() cursor = con.execute( """ @@ -236,9 +237,10 @@ def _select_caption_rows( params: list[object] = [] if only_needing_caption and not getattr(args, "force", False): - expected_models = [settings.caption_general_model] - if settings.caption_router_enabled: - expected_models.append(settings.caption_specialist_model) + expected_models = [ + settings.caption_general_model, + settings.caption_specialist_model, + ] placeholders = ",".join("?" for _ in expected_models) conditions.append( f"(f.caption IS NULL OR f.caption_model IS NULL " From da32d35c962df55da76ddc3886bd6d872ceffc0d Mon Sep 17 00:00:00 2001 From: Sleeps Date: Tue, 6 Oct 2026 15:46:09 +0100 Subject: [PATCH 18/20] Allow authoritative Meilisearch rebuilds --- scene_search/meili.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scene_search/meili.py b/scene_search/meili.py index b601ba4..f408243 100644 --- a/scene_search/meili.py +++ b/scene_search/meili.py @@ -60,6 +60,12 @@ class Meili: json=["title", "caption", "jellyfin_tags"], )) + def clear_documents(self) -> None: + self.ensure_index() + self._check(self.session.delete( + f"{self.url}/indexes/{self.index}/documents", + )) + def add_documents(self, documents: list[dict]) -> None: if not documents: return From 30197e80c91d8321a2442f91fdcde59f3bc17018 Mon Sep 17 00:00:00 2001 From: Sleeps Date: Tue, 6 Oct 2026 15:46:27 +0100 Subject: [PATCH 19/20] Remove skipped Jellyfin items and track tag changes --- scene_search/jellyfin_index.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/scene_search/jellyfin_index.py b/scene_search/jellyfin_index.py index 5cab165..b67e284 100644 --- a/scene_search/jellyfin_index.py +++ b/scene_search/jellyfin_index.py @@ -1,6 +1,7 @@ from __future__ import annotations import hashlib +import shutil from pathlib import Path from PIL import Image @@ -167,7 +168,21 @@ def cmd_jellyfin_index(args: object) -> None: normalized_tags = {tag.casefold() for tag in tags} if settings.jellyfin_skip_tag and settings.jellyfin_skip_tag in normalized_tags: - print(f"[{number}/{len(summaries)}] SKIP {title}: Jellyfin skip tag") + with connect(settings.db_path) as con: + existing = con.execute( + "SELECT id FROM media WHERE jellyfin_item_id = ?", + (item_id,), + ).fetchone() + if existing: + con.execute("DELETE FROM media WHERE id = ?", (existing["id"],)) + shutil.rmtree( + settings.thumbs_dir / jellyfin_media_id(item_id), + ignore_errors=True, + ) + print( + f"[{number}/{len(summaries)}] SKIP {title}: " + "Jellyfin skip tag (removed from canonical index)" + ) skipped += 1 continue @@ -199,9 +214,11 @@ def cmd_jellyfin_index(args: object) -> None: (item_id,), ).fetchone() + tags_json = tags_to_json(tags) if ( existing and existing["jellyfin_etag"] == etag + and existing["jellyfin_tags"] == tags_json and existing["model_id"] == settings.model_id and int(existing["interval_ms"]) == expected_interval_ms and not force @@ -310,7 +327,7 @@ def cmd_jellyfin_index(args: object) -> None: media_source_id, str(library["Id"]), str(detail.get("Type") or ""), - tags_to_json(tags), + tags_json, etag, trickplay_width, trickplay_tile_width, From aeaba2d002f148059ac90880e2cdb978774462a2 Mon Sep 17 00:00:00 2001 From: Sleeps Date: Tue, 6 Oct 2026 15:46:53 +0100 Subject: [PATCH 20/20] Test authoritative Meilisearch rebuild clearing --- tests/test_meili.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_meili.py b/tests/test_meili.py index 0241145..97f6c9e 100644 --- a/tests/test_meili.py +++ b/tests/test_meili.py @@ -2,6 +2,7 @@ from __future__ import annotations import json import unittest +from unittest.mock import Mock import numpy as np @@ -18,6 +19,20 @@ class MeiliVectorSerializationTests(unittest.TestCase): self.assertTrue(all(type(value) is float for value in converted)) json.dumps({"vector": converted}, allow_nan=False) + def test_clear_documents_deletes_all_index_documents(self) -> None: + meili = Meili("http://localhost:7700", "key", "scenes") + response = Mock() + response.ok = True + meili.ensure_index = Mock() + meili.session.delete = Mock(return_value=response) + + meili.clear_documents() + + meili.ensure_index.assert_called_once() + meili.session.delete.assert_called_once_with( + "http://localhost:7700/indexes/scenes/documents" + ) + if __name__ == "__main__": unittest.main()