mirror of
https://github.com/MrSleeps/jellyfin-scene-search.git
synced 2026-11-17 23:41:28 +00:00
Merge pull request #17 from MrSleeps/feature/force-specialist-captioner
Add global specialist caption mode
This commit is contained in:
@@ -11,6 +11,8 @@ BATCH_SIZE=8
|
||||
OLLAMA_URL=http://127.0.0.1:11434
|
||||
CAPTION_GENERAL_MODEL=huihui_ai/qwen3-vl-abliterated:4b-instruct-q4_K_M
|
||||
CAPTION_SPECIALIST_MODEL=aha2025/llama-joycaption-beta-one-hf-llava:Q4_K_M
|
||||
# Hard override: send every frame to the specialist captioner and skip the router.
|
||||
CAPTION_FORCE_SPECIALIST=false
|
||||
CAPTION_TIMEOUT_SECONDS=180
|
||||
CAPTION_MAX_TOKENS=120
|
||||
CAPTION_PROMPT_FILE=./data/caption_prompt.txt
|
||||
|
||||
@@ -186,6 +186,7 @@ A lightweight local image classifier routes each frame to the appropriate captio
|
||||
OLLAMA_URL=http://127.0.0.1:11434
|
||||
CAPTION_GENERAL_MODEL=huihui_ai/qwen3-vl-abliterated:4b-instruct-q4_K_M
|
||||
CAPTION_SPECIALIST_MODEL=aha2025/llama-joycaption-beta-one-hf-llava:Q4_K_M
|
||||
CAPTION_FORCE_SPECIALIST=false
|
||||
CAPTION_TIMEOUT_SECONDS=180
|
||||
CAPTION_MAX_TOKENS=120
|
||||
CAPTION_PROMPT_FILE=./data/caption_prompt.txt
|
||||
@@ -210,6 +211,8 @@ ollama pull aha2025/llama-joycaption-beta-one-hf-llava:Q4_K_M
|
||||
|
||||
The router model is downloaded automatically by Transformers on the machine running this project the first time it is used.
|
||||
|
||||
Set `CAPTION_FORCE_SPECIALIST=true` to bypass the routing classifier entirely and send every selected frame to `CAPTION_SPECIALIST_MODEL`. This is a hard global override: per-item general/specialist tags and the normal router threshold/context are ignored while it is enabled. The override is included in the saved routing policy, so enabling it makes old general routes stale and they are converted to specialist routes on the next caption/resume run without re-running SigLIP. Frames that already have a caption from the specialist model can be retained.
|
||||
|
||||
Inspect routing scores without generating captions:
|
||||
|
||||
```bash
|
||||
|
||||
@@ -212,20 +212,29 @@ class CaptionJobManager:
|
||||
try:
|
||||
self._progress({"stage": "routing"})
|
||||
|
||||
# The router uses the local GPU. Block SigLIP searches/indexing only
|
||||
# for this stage; the actual VLM caption requests run on Ollama.
|
||||
self._set_gpu_busy(True)
|
||||
try:
|
||||
with self.search_service.model_lock:
|
||||
with FrameSource() as frame_source:
|
||||
routed, missing = _route_rows(
|
||||
rows,
|
||||
progress=self._progress,
|
||||
frame_source=frame_source,
|
||||
before_batch=lambda: self._wait_if_paused("routing"),
|
||||
)
|
||||
finally:
|
||||
self._set_gpu_busy(False)
|
||||
# The normal router uses the local GPU. A global specialist
|
||||
# override has no classifier work, so do not block semantic search.
|
||||
if settings.caption_force_specialist:
|
||||
with FrameSource() as frame_source:
|
||||
routed, missing = _route_rows(
|
||||
rows,
|
||||
progress=self._progress,
|
||||
frame_source=frame_source,
|
||||
before_batch=lambda: self._wait_if_paused("routing"),
|
||||
)
|
||||
else:
|
||||
self._set_gpu_busy(True)
|
||||
try:
|
||||
with self.search_service.model_lock:
|
||||
with FrameSource() as frame_source:
|
||||
routed, missing = _route_rows(
|
||||
rows,
|
||||
progress=self._progress,
|
||||
frame_source=frame_source,
|
||||
before_batch=lambda: self._wait_if_paused("routing"),
|
||||
)
|
||||
finally:
|
||||
self._set_gpu_busy(False)
|
||||
|
||||
if not routed:
|
||||
self._progress(
|
||||
|
||||
+36
-12
@@ -165,6 +165,12 @@ def cmd_sync_meili(_: argparse.Namespace) -> None:
|
||||
|
||||
|
||||
def _route_policy() -> str:
|
||||
if settings.caption_force_specialist:
|
||||
return (
|
||||
"forced-specialist|"
|
||||
f"specialist={settings.caption_specialist_model}"
|
||||
)
|
||||
|
||||
tag_policy = (
|
||||
f"force-specialist={settings.jellyfin_force_specialist_tag}|"
|
||||
f"force-general={settings.jellyfin_force_general_tag}|"
|
||||
@@ -299,7 +305,19 @@ def _route_rows(
|
||||
)
|
||||
return reusable, 0
|
||||
|
||||
if settings.caption_router_enabled:
|
||||
if settings.caption_force_specialist:
|
||||
print(
|
||||
f"Forcing {len(pending_rows)} frame(s) to specialist captioner "
|
||||
f"{settings.caption_specialist_model}; router skipped."
|
||||
)
|
||||
valid_rows = list(pending_rows)
|
||||
scores = {str(row["id"]): 0.0 for row in valid_rows}
|
||||
decisions = {
|
||||
str(row["id"]): ("specialist", "global-specialist")
|
||||
for row in valid_rows
|
||||
}
|
||||
route_model = "forced-specialist"
|
||||
elif settings.caption_router_enabled:
|
||||
print(
|
||||
f"Routing {len(pending_rows)} frame(s) with "
|
||||
f"{settings.caption_router_model} on {settings.caption_router_device}..."
|
||||
@@ -382,16 +400,17 @@ def _route_rows(
|
||||
}
|
||||
route_model = "disabled"
|
||||
|
||||
for row in valid_rows:
|
||||
tags = tags_from_json(row["jellyfin_tags"])
|
||||
override = route_override_from_tags(
|
||||
tags,
|
||||
force_specialist_tag=settings.jellyfin_force_specialist_tag,
|
||||
force_general_tag=settings.jellyfin_force_general_tag,
|
||||
specialist_tags=settings.jellyfin_specialist_tags,
|
||||
)
|
||||
if override is not None:
|
||||
decisions[str(row["id"])] = override
|
||||
if not settings.caption_force_specialist:
|
||||
for row in valid_rows:
|
||||
tags = tags_from_json(row["jellyfin_tags"])
|
||||
override = route_override_from_tags(
|
||||
tags,
|
||||
force_specialist_tag=settings.jellyfin_force_specialist_tag,
|
||||
force_general_tag=settings.jellyfin_force_general_tag,
|
||||
specialist_tags=settings.jellyfin_specialist_tags,
|
||||
)
|
||||
if override is not None:
|
||||
decisions[str(row["id"])] = override
|
||||
|
||||
routed_by_id = {
|
||||
str(row["id"]): (row, route, score, reason)
|
||||
@@ -438,7 +457,12 @@ def _route_rows(
|
||||
1 for _, route, _, _ in routed if route == "general"
|
||||
)
|
||||
specialist_count = len(routed) - general_count
|
||||
if settings.caption_router_enabled:
|
||||
if settings.caption_force_specialist:
|
||||
print(
|
||||
f"Routing complete: {specialist_count} specialist "
|
||||
"(global specialist override)"
|
||||
)
|
||||
elif settings.caption_router_enabled:
|
||||
print(
|
||||
f"Routing complete: {general_count} general, "
|
||||
f"{specialist_count} specialist "
|
||||
|
||||
@@ -50,6 +50,9 @@ class Settings:
|
||||
"CAPTION_SPECIALIST_MODEL",
|
||||
"aha2025/llama-joycaption-beta-one-hf-llava:Q4_K_M",
|
||||
)
|
||||
caption_force_specialist: bool = os.getenv(
|
||||
"CAPTION_FORCE_SPECIALIST", "false"
|
||||
).strip().lower() not in {"0", "false", "no", "off"}
|
||||
caption_timeout_seconds: int = int(os.getenv("CAPTION_TIMEOUT_SECONDS", "180"))
|
||||
caption_max_tokens: int = int(os.getenv("CAPTION_MAX_TOKENS", "120"))
|
||||
caption_prompt_file: Path = Path(
|
||||
|
||||
@@ -20,6 +20,7 @@ class CaptionSelectionTests(unittest.TestCase):
|
||||
db_path=db_path,
|
||||
caption_general_model="general-model",
|
||||
caption_specialist_model="specialist-model",
|
||||
caption_force_specialist=False,
|
||||
caption_router_enabled=False,
|
||||
jellyfin_force_specialist_tag="scene-search:specialist",
|
||||
jellyfin_force_general_tag="scene-search:general",
|
||||
@@ -136,6 +137,7 @@ class CaptionSelectionTests(unittest.TestCase):
|
||||
db_path=db_path,
|
||||
caption_general_model="general-model",
|
||||
caption_specialist_model="specialist-model",
|
||||
caption_force_specialist=False,
|
||||
caption_router_enabled=True,
|
||||
caption_router_model="router-model",
|
||||
caption_router_threshold=0.5,
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from scene_search.cli import _route_policy, _route_rows
|
||||
from scene_search.database import init_db
|
||||
|
||||
|
||||
class SavedRouteReuseTests(unittest.TestCase):
|
||||
@@ -12,6 +15,7 @@ class SavedRouteReuseTests(unittest.TestCase):
|
||||
fake_settings = SimpleNamespace(
|
||||
caption_general_model="general-model",
|
||||
caption_specialist_model="specialist-model",
|
||||
caption_force_specialist=False,
|
||||
caption_router_enabled=True,
|
||||
caption_router_model="router-model",
|
||||
caption_router_device="cuda",
|
||||
@@ -48,6 +52,49 @@ class SavedRouteReuseTests(unittest.TestCase):
|
||||
self.assertAlmostEqual(routed[0][2], 0.91)
|
||||
self.assertEqual(routed[0][3], "score")
|
||||
|
||||
def test_global_specialist_override_skips_classifier(self) -> None:
|
||||
fake_settings = SimpleNamespace(
|
||||
db_path=Path(self._testMethodName + ".sqlite3"),
|
||||
caption_general_model="general-model",
|
||||
caption_specialist_model="specialist-model",
|
||||
caption_force_specialist=True,
|
||||
caption_router_enabled=True,
|
||||
caption_router_model="router-model",
|
||||
caption_router_device="cuda",
|
||||
caption_router_batch_size=32,
|
||||
caption_router_threshold=0.5,
|
||||
caption_router_context_seconds=6,
|
||||
jellyfin_force_specialist_tag="scene-search:specialist",
|
||||
jellyfin_force_general_tag="scene-search:general",
|
||||
jellyfin_specialist_tags=(),
|
||||
)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
fake_settings.db_path = Path(tmp) / "scenes.sqlite3"
|
||||
init_db(fake_settings.db_path)
|
||||
row = {
|
||||
"id": "frame-1",
|
||||
"media_id": "media-1",
|
||||
"timestamp_ms": 10000,
|
||||
"caption_route": None,
|
||||
"caption_route_reason": None,
|
||||
"route_score": None,
|
||||
"route_model": None,
|
||||
"route_policy": None,
|
||||
"jellyfin_tags": "[]",
|
||||
}
|
||||
|
||||
with patch("scene_search.cli.settings", fake_settings):
|
||||
with patch(
|
||||
"scene_search.cli.ExplicitContentRouter",
|
||||
side_effect=AssertionError("classifier should not be loaded"),
|
||||
):
|
||||
routed, missing = _route_rows([row])
|
||||
|
||||
self.assertEqual(missing, 0)
|
||||
self.assertEqual(routed[0][1], "specialist")
|
||||
self.assertEqual(routed[0][3], "global-specialist")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user