mirror of
https://github.com/MrSleeps/jellyfin-scene-search.git
synced 2026-11-19 00:03:36 +00:00
Merge pull request #6 from MrSleeps/feature/jellyfin-ingestion
Add Jellyfin library and Trickplay ingestion
This commit is contained in:
@@ -22,3 +22,22 @@ CAPTION_ROUTER_DEVICE=cuda
|
||||
CAPTION_ROUTER_BATCH_SIZE=32
|
||||
CAPTION_ROUTER_THRESHOLD=0.50
|
||||
CAPTION_ROUTER_CONTEXT_SECONDS=6
|
||||
|
||||
# Jellyfin ingestion
|
||||
JELLYFIN_URL=http://127.0.0.1:8096
|
||||
JELLYFIN_API_KEY=
|
||||
JELLYFIN_LIBRARY=
|
||||
JELLYFIN_TRICKPLAY_WIDTH=320
|
||||
JELLYFIN_FALLBACK_INTERVAL_SECONDS=10
|
||||
JELLYFIN_TIMEOUT_SECONDS=60
|
||||
|
||||
# Ordinary Jellyfin tags are passed to the captioner as context hints.
|
||||
JELLYFIN_TAG_CONTEXT=true
|
||||
|
||||
# Optional comma-separated tags that should always use the specialist captioner.
|
||||
JELLYFIN_SPECIALIST_TAGS=
|
||||
|
||||
# Reserved control tags. Add these directly to a Jellyfin item when needed.
|
||||
JELLYFIN_FORCE_SPECIALIST_TAG=scene-search:specialist
|
||||
JELLYFIN_FORCE_GENERAL_TAG=scene-search:general
|
||||
JELLYFIN_SKIP_TAG=scene-search:skip
|
||||
|
||||
@@ -163,7 +163,88 @@ Caption work is grouped by route. The general queue is processed first, its Olla
|
||||
|
||||
On the first run after enabling routing, existing captions are classified. A caption that already came from the model selected by the new route is retained; only frames whose selected model changes are regenerated unless `--force` is used.
|
||||
|
||||
## 7. Hybrid search
|
||||
## 7. Jellyfin ingestion
|
||||
|
||||
Jellyfin can now be the source of truth for media identity, metadata and frame sampling.
|
||||
|
||||
Configure a server API key and a small test library:
|
||||
|
||||
```env
|
||||
JELLYFIN_URL=http://jellyfin-server:8096
|
||||
JELLYFIN_API_KEY=replace-me
|
||||
JELLYFIN_LIBRARY=Scene Search Test
|
||||
|
||||
JELLYFIN_TRICKPLAY_WIDTH=320
|
||||
JELLYFIN_FALLBACK_INTERVAL_SECONDS=10
|
||||
JELLYFIN_TIMEOUT_SECONDS=60
|
||||
|
||||
JELLYFIN_TAG_CONTEXT=true
|
||||
JELLYFIN_SPECIALIST_TAGS=
|
||||
|
||||
JELLYFIN_FORCE_SPECIALIST_TAG=scene-search:specialist
|
||||
JELLYFIN_FORCE_GENERAL_TAG=scene-search:general
|
||||
JELLYFIN_SKIP_TAG=scene-search:skip
|
||||
```
|
||||
|
||||
Verify connectivity:
|
||||
|
||||
```bash
|
||||
python -m scene_search.cli jellyfin-test
|
||||
python -m scene_search.cli jellyfin-libraries
|
||||
```
|
||||
|
||||
Inspect the test library before doing any GPU work:
|
||||
|
||||
```bash
|
||||
python -m scene_search.cli jellyfin-items --library "Scene Search Test" --limit 10
|
||||
```
|
||||
|
||||
This prints each Jellyfin item ID, type, tags, path and available Trickplay information.
|
||||
|
||||
Index the library:
|
||||
|
||||
```bash
|
||||
python -m scene_search.cli jellyfin-index --library "Scene Search Test"
|
||||
```
|
||||
|
||||
The indexer prefers Jellyfin's existing Trickplay sheets. It downloads each sheet, splits the sheet into timestamped thumbnails using Jellyfin's Trickplay manifest, embeds those thumbnails with SigLIP and stores the Jellyfin item/media-source identity alongside the canonical vectors.
|
||||
|
||||
If an item has no Trickplay manifest, the indexer falls back to ffmpeg only when Jellyfin's reported media path is accessible from the indexing machine.
|
||||
|
||||
After indexing:
|
||||
|
||||
```bash
|
||||
python -m scene_search.cli route
|
||||
python -m scene_search.cli caption --sync
|
||||
```
|
||||
|
||||
### Jellyfin tags
|
||||
|
||||
Normal Jellyfin tags are supplied to the caption model as metadata context. They are hints only: the prompt explicitly tells the model not to claim a tagged detail unless that detail is actually visible in the current frame.
|
||||
|
||||
Tags also participate in routing:
|
||||
|
||||
- `scene-search:specialist` forces every frame from that item through the specialist JoyCaption model.
|
||||
- `scene-search:general` forces the general Qwen captioner.
|
||||
- `scene-search:skip` prevents the item from being indexed.
|
||||
- `JELLYFIN_SPECIALIST_TAGS` can contain a comma-separated list of your existing Jellyfin tags that should automatically use JoyCaption.
|
||||
|
||||
For example, if your existing library has specific descriptive tags that are useful vocabulary for specialist material, add those exact tag names to `JELLYFIN_SPECIALIST_TAGS`. The original tag text is still passed to the specialist captioner as context.
|
||||
|
||||
Jellyfin-backed media stores:
|
||||
|
||||
- Jellyfin item ID
|
||||
- media source ID
|
||||
- library ID
|
||||
- item type
|
||||
- tags
|
||||
- item ETag
|
||||
- selected Trickplay width and sheet layout
|
||||
- source path when Jellyfin exposes one
|
||||
|
||||
These fields are also copied into Meilisearch. Search results therefore retain enough identity to build the Jellyfin UI integration next.
|
||||
|
||||
## 8. Hybrid search
|
||||
|
||||
After captions exist, sync them:
|
||||
|
||||
@@ -230,11 +311,11 @@ Meilisearch does not normally display the contents of `_vectors` in its Mini Das
|
||||
|
||||
If caption + vector retrieval proves useful:
|
||||
|
||||
1. group neighbouring frames into scenes/windows to remove duplicate results;
|
||||
2. caption one scene/window rather than every frame;
|
||||
3. replace local ffmpeg frames with Jellyfin Trickplay tiles;
|
||||
4. store Jellyfin item/media-source IDs and Trickplay coordinates;
|
||||
5. add subtitles/transcripts as another searchable field;
|
||||
6. return Jellyfin results that jump directly to the matched timestamp.
|
||||
1. group neighbouring Trickplay thumbnails into scenes/windows to remove duplicate results;
|
||||
2. caption one scene/window rather than every thumbnail;
|
||||
3. add subtitles/transcripts as another searchable field;
|
||||
4. add the Jellyfin search-provider/plugin layer;
|
||||
5. return Jellyfin results that jump directly to the matched timestamp;
|
||||
6. enforce the requesting Jellyfin user's library permissions before returning results.
|
||||
|
||||
The routing classifier is not used as user-facing metadata or a library-wide content label; it exists only to select the more suitable captioner for each frame.
|
||||
|
||||
@@ -56,8 +56,15 @@ class OllamaCaptioner:
|
||||
self.max_tokens = max_tokens
|
||||
self.session = requests.Session()
|
||||
|
||||
def caption(self, image_path: Path) -> str:
|
||||
def caption(self, image_path: Path, context_hint: str | None = None) -> str:
|
||||
image_b64 = base64.b64encode(image_path.read_bytes()).decode("ascii")
|
||||
prompt = self.prompt
|
||||
if context_hint:
|
||||
prompt = (
|
||||
f"{prompt}\n\nMetadata context for this video: {context_hint}\n"
|
||||
"Use this only as a vocabulary/context hint. Do not claim a tagged "
|
||||
"detail is visible unless it is actually visible in the frame."
|
||||
)
|
||||
response = self.session.post(
|
||||
f"{self.url}/api/chat",
|
||||
json={
|
||||
@@ -66,7 +73,7 @@ class OllamaCaptioner:
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": self.prompt,
|
||||
"content": prompt,
|
||||
"images": [image_b64],
|
||||
}
|
||||
],
|
||||
|
||||
+82
-8
@@ -15,6 +15,17 @@ from .config import settings
|
||||
from .database import connect, init_db, pack_vector, unpack_vector
|
||||
from .embedder import SiglipEmbedder
|
||||
from .frames import extract_frames, media_id_for, sha256_file
|
||||
from .jellyfin import (
|
||||
caption_context_from_tags,
|
||||
route_override_from_tags,
|
||||
tags_from_json,
|
||||
)
|
||||
from .jellyfin_index import (
|
||||
cmd_jellyfin_index,
|
||||
cmd_jellyfin_items,
|
||||
cmd_jellyfin_libraries,
|
||||
cmd_jellyfin_test,
|
||||
)
|
||||
from .meili import Meili
|
||||
from .routing import ExplicitContentRouter, apply_context_routing
|
||||
|
||||
@@ -139,6 +150,7 @@ def cmd_sync_meili(_: argparse.Namespace) -> None:
|
||||
raise SystemExit(f"Multiple embedding dimensions found: {dims}. Use separate indexes per model.")
|
||||
dim = int(dims[0])
|
||||
meili.configure_user_vectors(dim)
|
||||
meili.clear_documents()
|
||||
|
||||
cursor = con.execute(
|
||||
"""
|
||||
@@ -146,7 +158,10 @@ def cmd_sync_meili(_: argparse.Namespace) -> None:
|
||||
f.caption, f.caption_model, f.captioned_at,
|
||||
f.caption_route, f.caption_route_reason, f.route_score,
|
||||
f.route_model, f.route_policy, f.routed_at,
|
||||
m.title, m.source_path, m.embedding_dim, m.model_id
|
||||
m.title, m.source_path, m.embedding_dim, m.model_id,
|
||||
m.source_kind, m.jellyfin_item_id, m.jellyfin_media_source_id,
|
||||
m.jellyfin_library_id, m.jellyfin_item_type, m.jellyfin_tags,
|
||||
m.trickplay_width
|
||||
FROM frames f
|
||||
JOIN media m ON m.id = f.media_id
|
||||
ORDER BY f.id
|
||||
@@ -178,6 +193,13 @@ def cmd_sync_meili(_: argparse.Namespace) -> None:
|
||||
"route_model": row["route_model"],
|
||||
"route_policy": row["route_policy"],
|
||||
"routed_at": row["routed_at"],
|
||||
"source_kind": row["source_kind"],
|
||||
"jellyfin_item_id": row["jellyfin_item_id"],
|
||||
"jellyfin_media_source_id": row["jellyfin_media_source_id"],
|
||||
"jellyfin_library_id": row["jellyfin_library_id"],
|
||||
"jellyfin_item_type": row["jellyfin_item_type"],
|
||||
"jellyfin_tags": tags_from_json(row["jellyfin_tags"]),
|
||||
"trickplay_width": row["trickplay_width"],
|
||||
"_vectors": {"siglip2": vec.tolist()},
|
||||
})
|
||||
meili.add_documents(docs)
|
||||
@@ -189,12 +211,20 @@ def cmd_sync_meili(_: argparse.Namespace) -> None:
|
||||
|
||||
|
||||
def _route_policy() -> str:
|
||||
tag_policy = (
|
||||
f"force-specialist={settings.jellyfin_force_specialist_tag}|"
|
||||
f"force-general={settings.jellyfin_force_general_tag}|"
|
||||
f"specialist-tags={','.join(settings.jellyfin_specialist_tags)}"
|
||||
)
|
||||
if not settings.caption_router_enabled:
|
||||
return f"disabled|general={settings.caption_general_model}"
|
||||
return (
|
||||
f"disabled|general={settings.caption_general_model}|{tag_policy}"
|
||||
)
|
||||
return (
|
||||
f"model={settings.caption_router_model}|"
|
||||
f"threshold={settings.caption_router_threshold:.6f}|"
|
||||
f"context={settings.caption_router_context_seconds:.3f}"
|
||||
f"context={settings.caption_router_context_seconds:.3f}|"
|
||||
f"{tag_policy}"
|
||||
)
|
||||
|
||||
|
||||
@@ -207,9 +237,10 @@ def _select_caption_rows(
|
||||
params: list[object] = []
|
||||
|
||||
if only_needing_caption and not getattr(args, "force", False):
|
||||
expected_models = [settings.caption_general_model]
|
||||
if settings.caption_router_enabled:
|
||||
expected_models.append(settings.caption_specialist_model)
|
||||
expected_models = [
|
||||
settings.caption_general_model,
|
||||
settings.caption_specialist_model,
|
||||
]
|
||||
placeholders = ",".join("?" for _ in expected_models)
|
||||
conditions.append(
|
||||
f"(f.caption IS NULL OR f.caption_model IS NULL "
|
||||
@@ -237,7 +268,7 @@ def _select_caption_rows(
|
||||
f"""
|
||||
SELECT f.id, f.media_id, f.timestamp_ms, f.thumb_path, f.caption,
|
||||
f.caption_model, f.caption_route, f.route_score, f.route_policy,
|
||||
m.title
|
||||
m.title, m.jellyfin_tags
|
||||
FROM frames f
|
||||
JOIN media m ON m.id = f.media_id
|
||||
{where}
|
||||
@@ -306,6 +337,17 @@ def _route_rows(rows: list, *, verbose: bool = False) -> tuple[list[tuple], int]
|
||||
}
|
||||
route_model = "disabled"
|
||||
|
||||
for row in valid_rows:
|
||||
tags = tags_from_json(row["jellyfin_tags"])
|
||||
override = route_override_from_tags(
|
||||
tags,
|
||||
force_specialist_tag=settings.jellyfin_force_specialist_tag,
|
||||
force_general_tag=settings.jellyfin_force_general_tag,
|
||||
specialist_tags=settings.jellyfin_specialist_tags,
|
||||
)
|
||||
if override is not None:
|
||||
decisions[str(row["id"])] = override
|
||||
|
||||
routed: list[tuple] = []
|
||||
with connect(settings.db_path) as con:
|
||||
for row in valid_rows:
|
||||
@@ -380,7 +422,17 @@ def _caption_queue(
|
||||
for row, route, score, reason in routed_rows:
|
||||
image_path = Path(row["thumb_path"])
|
||||
try:
|
||||
caption = captioner.caption(image_path)
|
||||
context_hint = None
|
||||
if settings.jellyfin_tag_context:
|
||||
context_hint = caption_context_from_tags(
|
||||
tags_from_json(row["jellyfin_tags"]),
|
||||
control_tags=(
|
||||
settings.jellyfin_force_specialist_tag,
|
||||
settings.jellyfin_force_general_tag,
|
||||
settings.jellyfin_skip_tag,
|
||||
),
|
||||
)
|
||||
caption = captioner.caption(image_path, context_hint=context_hint)
|
||||
except Exception as exc:
|
||||
failed += 1
|
||||
print(
|
||||
@@ -628,6 +680,11 @@ def cmd_search(args: argparse.Namespace) -> None:
|
||||
else ""
|
||||
)
|
||||
print(f" route={route}{score_text}")
|
||||
jellyfin_item_id = hit.get("jellyfin_item_id")
|
||||
if jellyfin_item_id:
|
||||
tags = hit.get("jellyfin_tags") or []
|
||||
tags_text = f" tags={', '.join(tags)}" if tags else ""
|
||||
print(f" jellyfin={jellyfin_item_id}{tags_text}")
|
||||
print(f" {hit.get('thumb_path')}")
|
||||
|
||||
|
||||
@@ -648,6 +705,23 @@ def make_parser() -> argparse.ArgumentParser:
|
||||
sync = sub.add_parser("sync-meili", help="Rebuild/feed Meilisearch from canonical SQLite vectors")
|
||||
sync.set_defaults(func=cmd_sync_meili)
|
||||
|
||||
jf_test = sub.add_parser("jellyfin-test", help="Verify Jellyfin API connectivity")
|
||||
jf_test.set_defaults(func=cmd_jellyfin_test)
|
||||
|
||||
jf_libs = sub.add_parser("jellyfin-libraries", help="List Jellyfin media libraries")
|
||||
jf_libs.set_defaults(func=cmd_jellyfin_libraries)
|
||||
|
||||
jf_items = sub.add_parser("jellyfin-items", help="Inspect video items, tags and Trickplay")
|
||||
jf_items.add_argument("--library", help="Jellyfin library name (or use JELLYFIN_LIBRARY)")
|
||||
jf_items.add_argument("--limit", type=int, help="Maximum number of items to inspect")
|
||||
jf_items.set_defaults(func=cmd_jellyfin_items)
|
||||
|
||||
jf_index = sub.add_parser("jellyfin-index", help="Index a Jellyfin library via Trickplay")
|
||||
jf_index.add_argument("--library", help="Jellyfin library name (or use JELLYFIN_LIBRARY)")
|
||||
jf_index.add_argument("--limit", type=int, help="Maximum number of items to index")
|
||||
jf_index.add_argument("--force", action="store_true", help="Rebuild matching Jellyfin items")
|
||||
jf_index.set_defaults(func=cmd_jellyfin_index)
|
||||
|
||||
route = sub.add_parser("route", help="Classify frames and show which caption model they will use")
|
||||
route.add_argument("--media-id", help="Only route frames from one media ID")
|
||||
route.add_argument("--title", help="Only route frames from one exact media title")
|
||||
|
||||
@@ -9,6 +9,15 @@ from dotenv import load_dotenv
|
||||
load_dotenv()
|
||||
|
||||
|
||||
def _csv_env(name: str, default: str = "") -> tuple[str, ...]:
|
||||
value = os.getenv(name, default)
|
||||
return tuple(
|
||||
part.strip().lower()
|
||||
for part in value.split(",")
|
||||
if part.strip()
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
meili_url: str = os.getenv("MEILI_URL", "http://127.0.0.1:7700").rstrip("/")
|
||||
@@ -63,6 +72,30 @@ class Settings:
|
||||
os.getenv("CAPTION_ROUTER_CONTEXT_SECONDS", "6")
|
||||
)
|
||||
|
||||
jellyfin_url: str = os.getenv("JELLYFIN_URL", "").rstrip("/")
|
||||
jellyfin_api_key: str = os.getenv("JELLYFIN_API_KEY", "")
|
||||
jellyfin_library: str = os.getenv("JELLYFIN_LIBRARY", "")
|
||||
jellyfin_trickplay_width: int = int(os.getenv("JELLYFIN_TRICKPLAY_WIDTH", "320"))
|
||||
jellyfin_fallback_interval_seconds: float = float(
|
||||
os.getenv("JELLYFIN_FALLBACK_INTERVAL_SECONDS", "10")
|
||||
)
|
||||
jellyfin_timeout_seconds: int = int(os.getenv("JELLYFIN_TIMEOUT_SECONDS", "60"))
|
||||
jellyfin_specialist_tags: tuple[str, ...] = _csv_env(
|
||||
"JELLYFIN_SPECIALIST_TAGS"
|
||||
)
|
||||
jellyfin_force_specialist_tag: str = os.getenv(
|
||||
"JELLYFIN_FORCE_SPECIALIST_TAG", "scene-search:specialist"
|
||||
).strip().lower()
|
||||
jellyfin_force_general_tag: str = os.getenv(
|
||||
"JELLYFIN_FORCE_GENERAL_TAG", "scene-search:general"
|
||||
).strip().lower()
|
||||
jellyfin_skip_tag: str = os.getenv(
|
||||
"JELLYFIN_SKIP_TAG", "scene-search:skip"
|
||||
).strip().lower()
|
||||
jellyfin_tag_context: bool = os.getenv(
|
||||
"JELLYFIN_TAG_CONTEXT", "true"
|
||||
).strip().lower() not in {"0", "false", "no", "off"}
|
||||
|
||||
data_dir: Path = Path(os.getenv("DATA_DIR", "./data")).resolve()
|
||||
|
||||
@property
|
||||
@@ -73,7 +106,12 @@ class Settings:
|
||||
def thumbs_dir(self) -> Path:
|
||||
return self.data_dir / "thumbs"
|
||||
|
||||
@property
|
||||
def jellyfin_cache_dir(self) -> Path:
|
||||
return self.data_dir / "jellyfin"
|
||||
|
||||
|
||||
settings = Settings()
|
||||
settings.data_dir.mkdir(parents=True, exist_ok=True)
|
||||
settings.thumbs_dir.mkdir(parents=True, exist_ok=True)
|
||||
settings.jellyfin_cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -21,6 +21,16 @@ CREATE TABLE IF NOT EXISTS media (
|
||||
interval_ms INTEGER NOT NULL,
|
||||
model_id TEXT NOT NULL,
|
||||
embedding_dim INTEGER NOT NULL,
|
||||
source_kind TEXT NOT NULL DEFAULT 'file',
|
||||
jellyfin_item_id TEXT,
|
||||
jellyfin_media_source_id TEXT,
|
||||
jellyfin_library_id TEXT,
|
||||
jellyfin_item_type TEXT,
|
||||
jellyfin_tags TEXT,
|
||||
jellyfin_etag TEXT,
|
||||
trickplay_width INTEGER,
|
||||
trickplay_tile_width INTEGER,
|
||||
trickplay_tile_height INTEGER,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
@@ -61,6 +71,32 @@ def connect(path: Path) -> Iterator[sqlite3.Connection]:
|
||||
con.close()
|
||||
|
||||
|
||||
def _ensure_media_jellyfin_columns(con: sqlite3.Connection) -> None:
|
||||
columns = {row["name"] for row in con.execute("PRAGMA table_info(media)")}
|
||||
additions = {
|
||||
"source_kind": "TEXT NOT NULL DEFAULT 'file'",
|
||||
"jellyfin_item_id": "TEXT",
|
||||
"jellyfin_media_source_id": "TEXT",
|
||||
"jellyfin_library_id": "TEXT",
|
||||
"jellyfin_item_type": "TEXT",
|
||||
"jellyfin_tags": "TEXT",
|
||||
"jellyfin_etag": "TEXT",
|
||||
"trickplay_width": "INTEGER",
|
||||
"trickplay_tile_width": "INTEGER",
|
||||
"trickplay_tile_height": "INTEGER",
|
||||
}
|
||||
for name, sql_type in additions.items():
|
||||
if name not in columns:
|
||||
con.execute(f"ALTER TABLE media ADD COLUMN {name} {sql_type}")
|
||||
con.execute(
|
||||
"""
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_media_jellyfin_item
|
||||
ON media(jellyfin_item_id)
|
||||
WHERE jellyfin_item_id IS NOT NULL
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def _ensure_frame_caption_columns(con: sqlite3.Connection) -> None:
|
||||
columns = {row["name"] for row in con.execute("PRAGMA table_info(frames)")}
|
||||
additions = {
|
||||
@@ -82,6 +118,7 @@ def _ensure_frame_caption_columns(con: sqlite3.Connection) -> None:
|
||||
def init_db(path: Path) -> None:
|
||||
with connect(path) as con:
|
||||
con.executescript(SCHEMA)
|
||||
_ensure_media_jellyfin_columns(con)
|
||||
_ensure_frame_caption_columns(con)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,296 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from PIL import Image
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TrickplaySelection:
|
||||
media_source_id: str | None
|
||||
width: int
|
||||
height: int
|
||||
interval_ms: int
|
||||
thumbnail_count: int
|
||||
tile_columns: int
|
||||
tile_rows: int
|
||||
|
||||
@property
|
||||
def thumbnails_per_sheet(self) -> int:
|
||||
return self.tile_columns * self.tile_rows
|
||||
|
||||
|
||||
class JellyfinClient:
|
||||
def __init__(self, url: str, api_key: str, timeout_seconds: int = 60) -> None:
|
||||
if not url:
|
||||
raise RuntimeError("JELLYFIN_URL is not configured")
|
||||
if not api_key:
|
||||
raise RuntimeError("JELLYFIN_API_KEY is not configured")
|
||||
|
||||
self.url = url.rstrip("/")
|
||||
self.timeout_seconds = timeout_seconds
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update({
|
||||
"X-Emby-Token": api_key,
|
||||
"Authorization": f'MediaBrowser Token="{api_key}"',
|
||||
"Accept": "application/json",
|
||||
})
|
||||
|
||||
def _get(self, path: str, *, params: dict[str, Any] | None = None) -> requests.Response:
|
||||
response = self.session.get(
|
||||
f"{self.url}{path}",
|
||||
params=params,
|
||||
timeout=self.timeout_seconds,
|
||||
)
|
||||
if not response.ok:
|
||||
raise RuntimeError(
|
||||
f"Jellyfin {response.status_code} GET {path}: {response.text[:500]}"
|
||||
)
|
||||
return response
|
||||
|
||||
def system_info(self) -> dict[str, Any]:
|
||||
return self._get("/System/Info").json()
|
||||
|
||||
def media_folders(self) -> list[dict[str, Any]]:
|
||||
data = self._get("/Library/MediaFolders").json()
|
||||
return list(data.get("Items") or [])
|
||||
|
||||
def find_library(self, name: str) -> dict[str, Any]:
|
||||
libraries = self.media_folders()
|
||||
matches = [
|
||||
item
|
||||
for item in libraries
|
||||
if str(item.get("Name") or "").casefold() == name.casefold()
|
||||
]
|
||||
if not matches:
|
||||
available = ", ".join(
|
||||
sorted(str(item.get("Name") or item.get("Id")) for item in libraries)
|
||||
)
|
||||
raise RuntimeError(
|
||||
f"Jellyfin library {name!r} not found. Available: {available or 'none'}"
|
||||
)
|
||||
return matches[0]
|
||||
|
||||
def items(
|
||||
self,
|
||||
library_id: str,
|
||||
*,
|
||||
limit: int | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
results: list[dict[str, Any]] = []
|
||||
start = 0
|
||||
page_size = 200
|
||||
|
||||
while limit is None or len(results) < limit:
|
||||
request_limit = page_size
|
||||
if limit is not None:
|
||||
request_limit = min(request_limit, limit - len(results))
|
||||
if request_limit <= 0:
|
||||
break
|
||||
|
||||
data = self._get(
|
||||
"/Items",
|
||||
params={
|
||||
"ParentId": library_id,
|
||||
"Recursive": "true",
|
||||
"IncludeItemTypes": "Movie,Episode,Video",
|
||||
"Fields": "Path",
|
||||
"EnableTotalRecordCount": "true",
|
||||
"StartIndex": start,
|
||||
"Limit": request_limit,
|
||||
},
|
||||
).json()
|
||||
page = list(data.get("Items") or [])
|
||||
if not page:
|
||||
break
|
||||
|
||||
results.extend(page)
|
||||
start += len(page)
|
||||
total = int(data.get("TotalRecordCount") or len(results))
|
||||
if start >= total or len(page) < request_limit:
|
||||
break
|
||||
|
||||
return results[:limit] if limit is not None else results
|
||||
|
||||
def item(self, item_id: str) -> dict[str, Any]:
|
||||
return self._get(f"/Items/{item_id}").json()
|
||||
|
||||
@staticmethod
|
||||
def select_trickplay(
|
||||
item: dict[str, Any],
|
||||
preferred_width: int = 320,
|
||||
) -> TrickplaySelection | None:
|
||||
manifest = item.get("Trickplay") or {}
|
||||
candidates: list[TrickplaySelection] = []
|
||||
|
||||
if not isinstance(manifest, dict):
|
||||
return None
|
||||
|
||||
for media_source_id, widths in manifest.items():
|
||||
if not isinstance(widths, dict):
|
||||
continue
|
||||
for width_key, raw_info in widths.items():
|
||||
if not isinstance(raw_info, dict):
|
||||
continue
|
||||
try:
|
||||
width = int(raw_info.get("Width") or width_key)
|
||||
height = int(raw_info.get("Height") or 0)
|
||||
interval_ms = int(raw_info.get("Interval") or 0)
|
||||
thumbnail_count = int(raw_info.get("ThumbnailCount") or 0)
|
||||
tile_columns = int(raw_info.get("TileWidth") or 0)
|
||||
tile_rows = int(raw_info.get("TileHeight") or 0)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
|
||||
if min(
|
||||
width,
|
||||
height,
|
||||
interval_ms,
|
||||
thumbnail_count,
|
||||
tile_columns,
|
||||
tile_rows,
|
||||
) <= 0:
|
||||
continue
|
||||
|
||||
candidates.append(
|
||||
TrickplaySelection(
|
||||
media_source_id=str(media_source_id) if media_source_id else None,
|
||||
width=width,
|
||||
height=height,
|
||||
interval_ms=interval_ms,
|
||||
thumbnail_count=thumbnail_count,
|
||||
tile_columns=tile_columns,
|
||||
tile_rows=tile_rows,
|
||||
)
|
||||
)
|
||||
|
||||
if not candidates:
|
||||
return None
|
||||
|
||||
return min(
|
||||
candidates,
|
||||
key=lambda candidate: (
|
||||
abs(candidate.width - preferred_width),
|
||||
-candidate.width,
|
||||
),
|
||||
)
|
||||
|
||||
def download_trickplay_frames(
|
||||
self,
|
||||
item_id: str,
|
||||
selection: TrickplaySelection,
|
||||
out_dir: Path,
|
||||
) -> list[Path]:
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
for old in out_dir.glob("frame_*.jpg"):
|
||||
old.unlink()
|
||||
|
||||
capacity = selection.thumbnails_per_sheet
|
||||
sheet_count = math.ceil(selection.thumbnail_count / capacity)
|
||||
paths: list[Path] = []
|
||||
|
||||
for sheet_index in range(sheet_count):
|
||||
params: dict[str, Any] = {}
|
||||
if selection.media_source_id:
|
||||
params["MediaSourceId"] = selection.media_source_id
|
||||
|
||||
response = self._get(
|
||||
f"/Videos/{item_id}/Trickplay/{selection.width}/{sheet_index}.jpg",
|
||||
params=params,
|
||||
)
|
||||
|
||||
with Image.open(io.BytesIO(response.content)) as sheet:
|
||||
sheet = sheet.convert("RGB")
|
||||
for local_index in range(capacity):
|
||||
global_index = sheet_index * capacity + local_index
|
||||
if global_index >= selection.thumbnail_count:
|
||||
break
|
||||
|
||||
column = local_index % selection.tile_columns
|
||||
row = local_index // selection.tile_columns
|
||||
left = column * selection.width
|
||||
top = row * selection.height
|
||||
frame = sheet.crop((
|
||||
left,
|
||||
top,
|
||||
left + selection.width,
|
||||
top + selection.height,
|
||||
))
|
||||
path = out_dir / f"frame_{global_index + 1:06d}.jpg"
|
||||
frame.save(path, "JPEG", quality=90)
|
||||
paths.append(path)
|
||||
|
||||
return paths
|
||||
|
||||
|
||||
def jellyfin_media_id(item_id: str) -> str:
|
||||
return hashlib.sha256(f"jellyfin:{item_id}".encode()).hexdigest()[:32]
|
||||
|
||||
|
||||
def item_tags(item: dict[str, Any]) -> list[str]:
|
||||
tags = item.get("Tags") or []
|
||||
if not isinstance(tags, list):
|
||||
return []
|
||||
return [str(tag).strip() for tag in tags if str(tag).strip()]
|
||||
|
||||
|
||||
def primary_media_source(item: dict[str, Any]) -> dict[str, Any]:
|
||||
sources = item.get("MediaSources") or []
|
||||
if not isinstance(sources, list):
|
||||
return {}
|
||||
for source in sources:
|
||||
if isinstance(source, dict):
|
||||
return source
|
||||
return {}
|
||||
|
||||
|
||||
def tags_to_json(tags: list[str]) -> str:
|
||||
return json.dumps(tags, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
|
||||
def tags_from_json(value: str | None) -> list[str]:
|
||||
if not value:
|
||||
return []
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
except (TypeError, json.JSONDecodeError):
|
||||
return []
|
||||
if not isinstance(parsed, list):
|
||||
return []
|
||||
return [str(tag).strip() for tag in parsed if str(tag).strip()]
|
||||
|
||||
|
||||
def route_override_from_tags(
|
||||
tags: list[str],
|
||||
*,
|
||||
force_specialist_tag: str,
|
||||
force_general_tag: str,
|
||||
specialist_tags: tuple[str, ...],
|
||||
) -> tuple[str, str] | None:
|
||||
normalized = {tag.casefold() for tag in tags}
|
||||
if force_specialist_tag and force_specialist_tag.casefold() in normalized:
|
||||
return ("specialist", "jellyfin-force-tag")
|
||||
if force_general_tag and force_general_tag.casefold() in normalized:
|
||||
return ("general", "jellyfin-force-tag")
|
||||
if normalized.intersection(tag.casefold() for tag in specialist_tags):
|
||||
return ("specialist", "jellyfin-specialist-tag")
|
||||
return None
|
||||
|
||||
|
||||
def caption_context_from_tags(
|
||||
tags: list[str],
|
||||
*,
|
||||
control_tags: tuple[str, ...] = (),
|
||||
) -> str | None:
|
||||
controls = {tag.casefold() for tag in control_tags if tag}
|
||||
visible_tags = [tag for tag in tags if tag.casefold() not in controls]
|
||||
if not visible_tags:
|
||||
return None
|
||||
return "Jellyfin tags: " + ", ".join(visible_tags)
|
||||
@@ -0,0 +1,365 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from .config import settings
|
||||
from .database import connect, init_db, pack_vector
|
||||
from .embedder import SiglipEmbedder
|
||||
from .frames import extract_frames
|
||||
from .jellyfin import (
|
||||
JellyfinClient,
|
||||
item_tags,
|
||||
jellyfin_media_id,
|
||||
primary_media_source,
|
||||
tags_to_json,
|
||||
)
|
||||
|
||||
|
||||
def build_jellyfin_client() -> JellyfinClient:
|
||||
return JellyfinClient(
|
||||
settings.jellyfin_url,
|
||||
settings.jellyfin_api_key,
|
||||
settings.jellyfin_timeout_seconds,
|
||||
)
|
||||
|
||||
|
||||
def resolve_library_name(requested: str | None) -> str:
|
||||
name = (requested or settings.jellyfin_library).strip()
|
||||
if not name:
|
||||
raise RuntimeError(
|
||||
"No Jellyfin library selected. Pass --library or set JELLYFIN_LIBRARY."
|
||||
)
|
||||
return name
|
||||
|
||||
|
||||
def _display_tags(tags: list[str]) -> str:
|
||||
return ", ".join(tags) if tags else "-"
|
||||
|
||||
|
||||
def cmd_jellyfin_test(_: object) -> None:
|
||||
client = build_jellyfin_client()
|
||||
info = client.system_info()
|
||||
libraries = client.media_folders()
|
||||
print(
|
||||
f"Connected to Jellyfin {info.get('Version', '?')} "
|
||||
f"({info.get('ServerName') or info.get('ProductName') or 'server'})"
|
||||
)
|
||||
print(f"Libraries visible to API key: {len(libraries)}")
|
||||
|
||||
|
||||
def cmd_jellyfin_libraries(_: object) -> None:
|
||||
client = build_jellyfin_client()
|
||||
libraries = client.media_folders()
|
||||
if not libraries:
|
||||
print("No Jellyfin libraries returned.")
|
||||
return
|
||||
|
||||
for item in sorted(libraries, key=lambda row: str(row.get("Name") or "")):
|
||||
print(f"{item.get('Name') or '(unnamed)'}\t{item.get('Id')}")
|
||||
|
||||
|
||||
def cmd_jellyfin_items(args: object) -> None:
|
||||
client = build_jellyfin_client()
|
||||
library_name = resolve_library_name(getattr(args, "library", None))
|
||||
library = client.find_library(library_name)
|
||||
summaries = client.items(str(library["Id"]), limit=getattr(args, "limit", None))
|
||||
|
||||
if not summaries:
|
||||
print(f"No video items found in {library_name!r}.")
|
||||
return
|
||||
|
||||
for summary in summaries:
|
||||
detail = client.item(str(summary["Id"]))
|
||||
tags = item_tags(detail)
|
||||
trickplay = client.select_trickplay(detail, settings.jellyfin_trickplay_width)
|
||||
tp = (
|
||||
f"{trickplay.width}px/{trickplay.interval_ms}ms/"
|
||||
f"{trickplay.thumbnail_count} thumbs"
|
||||
if trickplay
|
||||
else "none"
|
||||
)
|
||||
print(
|
||||
f"{detail.get('Name') or summary.get('Name') or '(unnamed)'}\n"
|
||||
f" id: {detail.get('Id')}\n"
|
||||
f" type: {detail.get('Type')}\n"
|
||||
f" tags: {_display_tags(tags)}\n"
|
||||
f" trickplay: {tp}\n"
|
||||
f" path: {detail.get('Path') or '-'}"
|
||||
)
|
||||
|
||||
|
||||
def _embed_frames(
|
||||
embedder: SiglipEmbedder,
|
||||
frame_paths: list[Path],
|
||||
media_id: str,
|
||||
interval_ms: int,
|
||||
) -> tuple[list[tuple], int]:
|
||||
rows: list[tuple] = []
|
||||
dim: int | None = None
|
||||
|
||||
for start in range(0, len(frame_paths), settings.batch_size):
|
||||
batch_paths = frame_paths[start:start + settings.batch_size]
|
||||
images: list[Image.Image] = []
|
||||
try:
|
||||
for frame_path in batch_paths:
|
||||
images.append(Image.open(frame_path).convert("RGB"))
|
||||
vectors = embedder.embed_images(images)
|
||||
finally:
|
||||
for image in images:
|
||||
image.close()
|
||||
|
||||
dim = int(vectors.shape[1])
|
||||
for offset, (frame_path, vector) in enumerate(
|
||||
zip(batch_paths, vectors, strict=True)
|
||||
):
|
||||
frame_number = start + offset
|
||||
timestamp_ms = frame_number * interval_ms
|
||||
frame_id = hashlib.sha256(
|
||||
f"{media_id}:{timestamp_ms}:{settings.model_id}".encode()
|
||||
).hexdigest()[:40]
|
||||
rows.append(
|
||||
(
|
||||
frame_id,
|
||||
media_id,
|
||||
frame_number,
|
||||
timestamp_ms,
|
||||
str(frame_path),
|
||||
pack_vector(vector),
|
||||
)
|
||||
)
|
||||
|
||||
print(
|
||||
f" embedded "
|
||||
f"{min(start + len(batch_paths), len(frame_paths))}/{len(frame_paths)}"
|
||||
)
|
||||
|
||||
if dim is None:
|
||||
raise RuntimeError("No frames were embedded")
|
||||
return rows, dim
|
||||
|
||||
|
||||
def cmd_jellyfin_index(args: object) -> None:
|
||||
init_db(settings.db_path)
|
||||
client = build_jellyfin_client()
|
||||
library_name = resolve_library_name(getattr(args, "library", None))
|
||||
library = client.find_library(library_name)
|
||||
summaries = client.items(str(library["Id"]), limit=getattr(args, "limit", None))
|
||||
force = bool(getattr(args, "force", False))
|
||||
|
||||
print(
|
||||
f"Jellyfin library {library_name!r}: {len(summaries)} video item(s) selected"
|
||||
)
|
||||
|
||||
embedder: SiglipEmbedder | None = None
|
||||
indexed = 0
|
||||
skipped = 0
|
||||
failed = 0
|
||||
|
||||
for number, summary in enumerate(summaries, 1):
|
||||
try:
|
||||
detail = client.item(str(summary["Id"]))
|
||||
item_id = str(detail.get("Id") or summary["Id"])
|
||||
title = str(detail.get("Name") or summary.get("Name") or item_id)
|
||||
tags = item_tags(detail)
|
||||
normalized_tags = {tag.casefold() for tag in tags}
|
||||
|
||||
if settings.jellyfin_skip_tag and settings.jellyfin_skip_tag in normalized_tags:
|
||||
with connect(settings.db_path) as con:
|
||||
existing = con.execute(
|
||||
"SELECT id FROM media WHERE jellyfin_item_id = ?",
|
||||
(item_id,),
|
||||
).fetchone()
|
||||
if existing:
|
||||
con.execute("DELETE FROM media WHERE id = ?", (existing["id"],))
|
||||
shutil.rmtree(
|
||||
settings.thumbs_dir / jellyfin_media_id(item_id),
|
||||
ignore_errors=True,
|
||||
)
|
||||
print(
|
||||
f"[{number}/{len(summaries)}] SKIP {title}: "
|
||||
"Jellyfin skip tag (removed from canonical index)"
|
||||
)
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
selection = client.select_trickplay(
|
||||
detail,
|
||||
settings.jellyfin_trickplay_width,
|
||||
)
|
||||
source = primary_media_source(detail)
|
||||
source_path_text = str(
|
||||
detail.get("Path")
|
||||
or source.get("Path")
|
||||
or f"jellyfin://{item_id}"
|
||||
)
|
||||
media_id = jellyfin_media_id(item_id)
|
||||
etag = str(detail.get("Etag") or "")
|
||||
content_marker = hashlib.sha256(
|
||||
f"jellyfin:{item_id}:{etag}".encode()
|
||||
).hexdigest()
|
||||
|
||||
expected_interval_ms = (
|
||||
selection.interval_ms
|
||||
if selection
|
||||
else int(round(settings.jellyfin_fallback_interval_seconds * 1000))
|
||||
)
|
||||
|
||||
with connect(settings.db_path) as con:
|
||||
existing = con.execute(
|
||||
"SELECT * FROM media WHERE jellyfin_item_id = ?",
|
||||
(item_id,),
|
||||
).fetchone()
|
||||
|
||||
tags_json = tags_to_json(tags)
|
||||
if (
|
||||
existing
|
||||
and existing["jellyfin_etag"] == etag
|
||||
and existing["jellyfin_tags"] == tags_json
|
||||
and existing["model_id"] == settings.model_id
|
||||
and int(existing["interval_ms"]) == expected_interval_ms
|
||||
and not force
|
||||
):
|
||||
frame_count = 0
|
||||
with connect(settings.db_path) as con:
|
||||
frame_count = int(
|
||||
con.execute(
|
||||
"SELECT COUNT(*) FROM frames WHERE media_id = ?",
|
||||
(existing["id"],),
|
||||
).fetchone()[0]
|
||||
)
|
||||
print(
|
||||
f"[{number}/{len(summaries)}] {title}: already indexed "
|
||||
f"({frame_count} frames)"
|
||||
)
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
out_dir = settings.thumbs_dir / media_id
|
||||
if selection:
|
||||
print(
|
||||
f"[{number}/{len(summaries)}] {title}: Trickplay "
|
||||
f"{selection.width}px every {selection.interval_ms / 1000:g}s"
|
||||
)
|
||||
frame_paths = client.download_trickplay_frames(
|
||||
item_id,
|
||||
selection,
|
||||
out_dir,
|
||||
)
|
||||
interval_ms = selection.interval_ms
|
||||
media_source_id = selection.media_source_id
|
||||
trickplay_width = selection.width
|
||||
trickplay_tile_width = selection.tile_columns
|
||||
trickplay_tile_height = selection.tile_rows
|
||||
else:
|
||||
local_path = Path(source_path_text)
|
||||
if not local_path.is_file():
|
||||
print(
|
||||
f"[{number}/{len(summaries)}] SKIP {title}: "
|
||||
"no Trickplay and media path is not locally accessible"
|
||||
)
|
||||
failed += 1
|
||||
continue
|
||||
|
||||
print(
|
||||
f"[{number}/{len(summaries)}] {title}: no Trickplay; "
|
||||
f"ffmpeg fallback every "
|
||||
f"{settings.jellyfin_fallback_interval_seconds:g}s"
|
||||
)
|
||||
frame_paths = extract_frames(
|
||||
local_path,
|
||||
out_dir,
|
||||
settings.jellyfin_fallback_interval_seconds,
|
||||
)
|
||||
interval_ms = int(
|
||||
round(settings.jellyfin_fallback_interval_seconds * 1000)
|
||||
)
|
||||
media_source_id = str(source.get("Id") or "") or None
|
||||
trickplay_width = None
|
||||
trickplay_tile_width = None
|
||||
trickplay_tile_height = None
|
||||
|
||||
if not frame_paths:
|
||||
print(f"[{number}/{len(summaries)}] SKIP {title}: no frames")
|
||||
failed += 1
|
||||
continue
|
||||
|
||||
if embedder is None:
|
||||
print(f"Loading {settings.model_id} on {settings.device}...")
|
||||
embedder = SiglipEmbedder(settings.model_id, settings.device)
|
||||
|
||||
frame_rows, embedding_dim = _embed_frames(
|
||||
embedder,
|
||||
frame_paths,
|
||||
media_id,
|
||||
interval_ms,
|
||||
)
|
||||
|
||||
size_bytes = int(source.get("Size") or 0)
|
||||
with connect(settings.db_path) as con:
|
||||
con.execute("DELETE FROM media WHERE id = ?", (media_id,))
|
||||
con.execute(
|
||||
"""
|
||||
INSERT INTO media (
|
||||
id, source_path, title, content_sha256, size_bytes, mtime_ns,
|
||||
interval_ms, model_id, embedding_dim, source_kind,
|
||||
jellyfin_item_id, jellyfin_media_source_id, jellyfin_library_id,
|
||||
jellyfin_item_type, jellyfin_tags, jellyfin_etag,
|
||||
trickplay_width, trickplay_tile_width, trickplay_tile_height
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
media_id,
|
||||
source_path_text,
|
||||
title,
|
||||
content_marker,
|
||||
size_bytes,
|
||||
0,
|
||||
interval_ms,
|
||||
settings.model_id,
|
||||
embedding_dim,
|
||||
"jellyfin",
|
||||
item_id,
|
||||
media_source_id,
|
||||
str(library["Id"]),
|
||||
str(detail.get("Type") or ""),
|
||||
tags_json,
|
||||
etag,
|
||||
trickplay_width,
|
||||
trickplay_tile_width,
|
||||
trickplay_tile_height,
|
||||
),
|
||||
)
|
||||
con.executemany(
|
||||
"""
|
||||
INSERT INTO frames (
|
||||
id, media_id, frame_number, timestamp_ms,
|
||||
thumb_path, embedding_f16
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
frame_rows,
|
||||
)
|
||||
|
||||
print(
|
||||
f" stored {len(frame_rows)} frames; tags={_display_tags(tags)}"
|
||||
)
|
||||
indexed += 1
|
||||
|
||||
except Exception as exc:
|
||||
failed += 1
|
||||
print(
|
||||
f"[{number}/{len(summaries)}] FAIL "
|
||||
f"{summary.get('Name') or summary.get('Id')}: {exc}"
|
||||
)
|
||||
|
||||
print(
|
||||
f"Jellyfin indexing complete: {indexed} indexed, "
|
||||
f"{skipped} skipped, {failed} failed."
|
||||
)
|
||||
if indexed:
|
||||
print("Next: python -m scene_search.cli caption --sync")
|
||||
+19
-2
@@ -42,11 +42,28 @@ class Meili:
|
||||
))
|
||||
self._check(self.session.put(
|
||||
f"{self.url}/indexes/{self.index}/settings/filterable-attributes",
|
||||
json=["media_id", "title", "caption_model", "caption_route", "route_model"],
|
||||
json=[
|
||||
"media_id",
|
||||
"title",
|
||||
"caption_model",
|
||||
"caption_route",
|
||||
"route_model",
|
||||
"source_kind",
|
||||
"jellyfin_item_id",
|
||||
"jellyfin_library_id",
|
||||
"jellyfin_item_type",
|
||||
"jellyfin_tags",
|
||||
],
|
||||
))
|
||||
self._check(self.session.put(
|
||||
f"{self.url}/indexes/{self.index}/settings/searchable-attributes",
|
||||
json=["title", "caption"],
|
||||
json=["title", "caption", "jellyfin_tags"],
|
||||
))
|
||||
|
||||
def clear_documents(self) -> None:
|
||||
self.ensure_index()
|
||||
self._check(self.session.delete(
|
||||
f"{self.url}/indexes/{self.index}/documents",
|
||||
))
|
||||
|
||||
def add_documents(self, documents: list[dict]) -> None:
|
||||
|
||||
@@ -21,13 +21,15 @@ class OllamaCaptionerTests(unittest.TestCase):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
image = Path(tmp) / "frame.jpg"
|
||||
image.write_bytes(b"fake-jpeg")
|
||||
result = captioner.caption(image)
|
||||
result = captioner.caption(image, context_hint="Jellyfin tags: beach, holiday")
|
||||
|
||||
self.assertEqual(result, "A person walking down a street.")
|
||||
call = captioner.session.post.call_args
|
||||
payload = call.kwargs["json"]
|
||||
self.assertEqual(payload["model"], "vision-test")
|
||||
self.assertEqual(payload["messages"][0]["role"], "user")
|
||||
self.assertIn("Jellyfin tags: beach, holiday", payload["messages"][0]["content"])
|
||||
self.assertIn("actually visible", payload["messages"][0]["content"])
|
||||
self.assertTrue(payload["messages"][0]["images"][0])
|
||||
self.assertEqual(payload["options"]["temperature"], 0.0)
|
||||
self.assertEqual(payload["options"]["num_predict"], 120)
|
||||
|
||||
+20
-10
@@ -44,18 +44,28 @@ class DatabaseMigrationTests(unittest.TestCase):
|
||||
init_db(path)
|
||||
|
||||
con = sqlite3.connect(path)
|
||||
columns = {row[1] for row in con.execute("PRAGMA table_info(frames)")}
|
||||
frame_columns = {row[1] for row in con.execute("PRAGMA table_info(frames)")}
|
||||
media_columns = {row[1] for row in con.execute("PRAGMA table_info(media)")}
|
||||
indexes = {row[1] for row in con.execute("PRAGMA index_list(media)")}
|
||||
con.close()
|
||||
|
||||
self.assertIn("caption", columns)
|
||||
self.assertIn("caption_model", columns)
|
||||
self.assertIn("captioned_at", columns)
|
||||
self.assertIn("caption_route", columns)
|
||||
self.assertIn("caption_route_reason", columns)
|
||||
self.assertIn("route_score", columns)
|
||||
self.assertIn("route_model", columns)
|
||||
self.assertIn("route_policy", columns)
|
||||
self.assertIn("routed_at", columns)
|
||||
self.assertIn("caption", frame_columns)
|
||||
self.assertIn("caption_model", frame_columns)
|
||||
self.assertIn("captioned_at", frame_columns)
|
||||
self.assertIn("caption_route", frame_columns)
|
||||
self.assertIn("caption_route_reason", frame_columns)
|
||||
self.assertIn("route_score", frame_columns)
|
||||
self.assertIn("route_model", frame_columns)
|
||||
self.assertIn("route_policy", frame_columns)
|
||||
self.assertIn("routed_at", frame_columns)
|
||||
|
||||
self.assertIn("source_kind", media_columns)
|
||||
self.assertIn("jellyfin_item_id", media_columns)
|
||||
self.assertIn("jellyfin_media_source_id", media_columns)
|
||||
self.assertIn("jellyfin_library_id", media_columns)
|
||||
self.assertIn("jellyfin_tags", media_columns)
|
||||
self.assertIn("trickplay_width", media_columns)
|
||||
self.assertIn("idx_media_jellyfin_item", indexes)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from scene_search.jellyfin import (
|
||||
JellyfinClient,
|
||||
TrickplaySelection,
|
||||
caption_context_from_tags,
|
||||
route_override_from_tags,
|
||||
)
|
||||
|
||||
|
||||
class JellyfinIntegrationTests(unittest.TestCase):
|
||||
def test_select_trickplay_prefers_nearest_width(self) -> None:
|
||||
item = {
|
||||
"Trickplay": {
|
||||
"source-1": {
|
||||
"160": {
|
||||
"Width": 160,
|
||||
"Height": 90,
|
||||
"Interval": 10000,
|
||||
"ThumbnailCount": 20,
|
||||
"TileWidth": 5,
|
||||
"TileHeight": 4,
|
||||
},
|
||||
"320": {
|
||||
"Width": 320,
|
||||
"Height": 180,
|
||||
"Interval": 10000,
|
||||
"ThumbnailCount": 20,
|
||||
"TileWidth": 5,
|
||||
"TileHeight": 4,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
selected = JellyfinClient.select_trickplay(item, preferred_width=300)
|
||||
|
||||
self.assertIsNotNone(selected)
|
||||
assert selected is not None
|
||||
self.assertEqual(selected.media_source_id, "source-1")
|
||||
self.assertEqual(selected.width, 320)
|
||||
self.assertEqual(selected.interval_ms, 10000)
|
||||
self.assertEqual(selected.thumbnails_per_sheet, 20)
|
||||
|
||||
def test_download_trickplay_frames_splits_sheet(self) -> None:
|
||||
client = JellyfinClient("http://jellyfin.test", "token")
|
||||
selection = TrickplaySelection(
|
||||
media_source_id="source-1",
|
||||
width=10,
|
||||
height=5,
|
||||
interval_ms=10000,
|
||||
thumbnail_count=4,
|
||||
tile_columns=2,
|
||||
tile_rows=2,
|
||||
)
|
||||
|
||||
sheet = Image.new("RGB", (20, 10), "white")
|
||||
payload = io.BytesIO()
|
||||
sheet.save(payload, "JPEG")
|
||||
response = Mock()
|
||||
response.ok = True
|
||||
response.content = payload.getvalue()
|
||||
client._get = Mock(return_value=response)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
paths = client.download_trickplay_frames(
|
||||
"item-1",
|
||||
selection,
|
||||
Path(tmp),
|
||||
)
|
||||
self.assertEqual(len(paths), 4)
|
||||
self.assertTrue(all(path.is_file() for path in paths))
|
||||
|
||||
call = client._get.call_args
|
||||
self.assertIn("/Videos/item-1/Trickplay/10/0.jpg", call.args[0])
|
||||
self.assertEqual(call.kwargs["params"]["MediaSourceId"], "source-1")
|
||||
|
||||
def test_jellyfin_tags_can_force_specialist_route(self) -> None:
|
||||
override = route_override_from_tags(
|
||||
["Archive", "scene-search:specialist"],
|
||||
force_specialist_tag="scene-search:specialist",
|
||||
force_general_tag="scene-search:general",
|
||||
specialist_tags=(),
|
||||
)
|
||||
self.assertEqual(override, ("specialist", "jellyfin-force-tag"))
|
||||
|
||||
def test_configured_specialist_tag_routes_to_specialist(self) -> None:
|
||||
override = route_override_from_tags(
|
||||
["Custom Category"],
|
||||
force_specialist_tag="scene-search:specialist",
|
||||
force_general_tag="scene-search:general",
|
||||
specialist_tags=("custom category",),
|
||||
)
|
||||
self.assertEqual(override, ("specialist", "jellyfin-specialist-tag"))
|
||||
|
||||
def test_control_tags_are_not_added_to_caption_context(self) -> None:
|
||||
context = caption_context_from_tags(
|
||||
["holiday", "scene-search:specialist", "beach"],
|
||||
control_tags=("scene-search:specialist",),
|
||||
)
|
||||
self.assertEqual(context, "Jellyfin tags: holiday, beach")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import unittest
|
||||
from unittest.mock import Mock
|
||||
|
||||
import numpy as np
|
||||
|
||||
@@ -18,6 +19,20 @@ class MeiliVectorSerializationTests(unittest.TestCase):
|
||||
self.assertTrue(all(type(value) is float for value in converted))
|
||||
json.dumps({"vector": converted}, allow_nan=False)
|
||||
|
||||
def test_clear_documents_deletes_all_index_documents(self) -> None:
|
||||
meili = Meili("http://localhost:7700", "key", "scenes")
|
||||
response = Mock()
|
||||
response.ok = True
|
||||
meili.ensure_index = Mock()
|
||||
meili.session.delete = Mock(return_value=response)
|
||||
|
||||
meili.clear_documents()
|
||||
|
||||
meili.ensure_index.assert_called_once()
|
||||
meili.session.delete.assert_called_once_with(
|
||||
"http://localhost:7700/indexes/scenes/documents"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user