Merge pull request #17 from MrSleeps/feature/force-specialist-captioner

Add global specialist caption mode
This commit is contained in:
Sleeps
2026-10-07 13:26:19 +01:00
committed by GitHub
7 changed files with 116 additions and 26 deletions
+2
View File
@@ -11,6 +11,8 @@ BATCH_SIZE=8
OLLAMA_URL=http://127.0.0.1:11434
CAPTION_GENERAL_MODEL=huihui_ai/qwen3-vl-abliterated:4b-instruct-q4_K_M
CAPTION_SPECIALIST_MODEL=aha2025/llama-joycaption-beta-one-hf-llava:Q4_K_M
# Hard override: send every frame to the specialist captioner and skip the router.
CAPTION_FORCE_SPECIALIST=false
CAPTION_TIMEOUT_SECONDS=180
CAPTION_MAX_TOKENS=120
CAPTION_PROMPT_FILE=./data/caption_prompt.txt
+3
View File
@@ -186,6 +186,7 @@ A lightweight local image classifier routes each frame to the appropriate captio
OLLAMA_URL=http://127.0.0.1:11434
CAPTION_GENERAL_MODEL=huihui_ai/qwen3-vl-abliterated:4b-instruct-q4_K_M
CAPTION_SPECIALIST_MODEL=aha2025/llama-joycaption-beta-one-hf-llava:Q4_K_M
CAPTION_FORCE_SPECIALIST=false
CAPTION_TIMEOUT_SECONDS=180
CAPTION_MAX_TOKENS=120
CAPTION_PROMPT_FILE=./data/caption_prompt.txt
@@ -210,6 +211,8 @@ ollama pull aha2025/llama-joycaption-beta-one-hf-llava:Q4_K_M
The router model is downloaded automatically by Transformers on the machine running this project the first time it is used.
Set `CAPTION_FORCE_SPECIALIST=true` to bypass the routing classifier entirely and send every selected frame to `CAPTION_SPECIALIST_MODEL`. This is a hard global override: per-item general/specialist tags and the normal router threshold/context are ignored while it is enabled. The override is included in the saved routing policy, so enabling it makes old general routes stale and they are converted to specialist routes on the next caption/resume run without re-running SigLIP. Frames that already have a caption from the specialist model can be retained.
Inspect routing scores without generating captions:
```bash
+23 -14
View File
@@ -212,20 +212,29 @@ class CaptionJobManager:
try:
self._progress({"stage": "routing"})
# The router uses the local GPU. Block SigLIP searches/indexing only
# for this stage; the actual VLM caption requests run on Ollama.
self._set_gpu_busy(True)
try:
with self.search_service.model_lock:
with FrameSource() as frame_source:
routed, missing = _route_rows(
rows,
progress=self._progress,
frame_source=frame_source,
before_batch=lambda: self._wait_if_paused("routing"),
)
finally:
self._set_gpu_busy(False)
# The normal router uses the local GPU. A global specialist
# override has no classifier work, so do not block semantic search.
if settings.caption_force_specialist:
with FrameSource() as frame_source:
routed, missing = _route_rows(
rows,
progress=self._progress,
frame_source=frame_source,
before_batch=lambda: self._wait_if_paused("routing"),
)
else:
self._set_gpu_busy(True)
try:
with self.search_service.model_lock:
with FrameSource() as frame_source:
routed, missing = _route_rows(
rows,
progress=self._progress,
frame_source=frame_source,
before_batch=lambda: self._wait_if_paused("routing"),
)
finally:
self._set_gpu_busy(False)
if not routed:
self._progress(
+36 -12
View File
@@ -165,6 +165,12 @@ def cmd_sync_meili(_: argparse.Namespace) -> None:
def _route_policy() -> str:
if settings.caption_force_specialist:
return (
"forced-specialist|"
f"specialist={settings.caption_specialist_model}"
)
tag_policy = (
f"force-specialist={settings.jellyfin_force_specialist_tag}|"
f"force-general={settings.jellyfin_force_general_tag}|"
@@ -299,7 +305,19 @@ def _route_rows(
)
return reusable, 0
if settings.caption_router_enabled:
if settings.caption_force_specialist:
print(
f"Forcing {len(pending_rows)} frame(s) to specialist captioner "
f"{settings.caption_specialist_model}; router skipped."
)
valid_rows = list(pending_rows)
scores = {str(row["id"]): 0.0 for row in valid_rows}
decisions = {
str(row["id"]): ("specialist", "global-specialist")
for row in valid_rows
}
route_model = "forced-specialist"
elif settings.caption_router_enabled:
print(
f"Routing {len(pending_rows)} frame(s) with "
f"{settings.caption_router_model} on {settings.caption_router_device}..."
@@ -382,16 +400,17 @@ def _route_rows(
}
route_model = "disabled"
for row in valid_rows:
tags = tags_from_json(row["jellyfin_tags"])
override = route_override_from_tags(
tags,
force_specialist_tag=settings.jellyfin_force_specialist_tag,
force_general_tag=settings.jellyfin_force_general_tag,
specialist_tags=settings.jellyfin_specialist_tags,
)
if override is not None:
decisions[str(row["id"])] = override
if not settings.caption_force_specialist:
for row in valid_rows:
tags = tags_from_json(row["jellyfin_tags"])
override = route_override_from_tags(
tags,
force_specialist_tag=settings.jellyfin_force_specialist_tag,
force_general_tag=settings.jellyfin_force_general_tag,
specialist_tags=settings.jellyfin_specialist_tags,
)
if override is not None:
decisions[str(row["id"])] = override
routed_by_id = {
str(row["id"]): (row, route, score, reason)
@@ -438,7 +457,12 @@ def _route_rows(
1 for _, route, _, _ in routed if route == "general"
)
specialist_count = len(routed) - general_count
if settings.caption_router_enabled:
if settings.caption_force_specialist:
print(
f"Routing complete: {specialist_count} specialist "
"(global specialist override)"
)
elif settings.caption_router_enabled:
print(
f"Routing complete: {general_count} general, "
f"{specialist_count} specialist "
+3
View File
@@ -50,6 +50,9 @@ class Settings:
"CAPTION_SPECIALIST_MODEL",
"aha2025/llama-joycaption-beta-one-hf-llava:Q4_K_M",
)
caption_force_specialist: bool = os.getenv(
"CAPTION_FORCE_SPECIALIST", "false"
).strip().lower() not in {"0", "false", "no", "off"}
caption_timeout_seconds: int = int(os.getenv("CAPTION_TIMEOUT_SECONDS", "180"))
caption_max_tokens: int = int(os.getenv("CAPTION_MAX_TOKENS", "120"))
caption_prompt_file: Path = Path(
+2
View File
@@ -20,6 +20,7 @@ class CaptionSelectionTests(unittest.TestCase):
db_path=db_path,
caption_general_model="general-model",
caption_specialist_model="specialist-model",
caption_force_specialist=False,
caption_router_enabled=False,
jellyfin_force_specialist_tag="scene-search:specialist",
jellyfin_force_general_tag="scene-search:general",
@@ -136,6 +137,7 @@ class CaptionSelectionTests(unittest.TestCase):
db_path=db_path,
caption_general_model="general-model",
caption_specialist_model="specialist-model",
caption_force_specialist=False,
caption_router_enabled=True,
caption_router_model="router-model",
caption_router_threshold=0.5,
+47
View File
@@ -1,10 +1,13 @@
from __future__ import annotations
import tempfile
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
from scene_search.cli import _route_policy, _route_rows
from scene_search.database import init_db
class SavedRouteReuseTests(unittest.TestCase):
@@ -12,6 +15,7 @@ class SavedRouteReuseTests(unittest.TestCase):
fake_settings = SimpleNamespace(
caption_general_model="general-model",
caption_specialist_model="specialist-model",
caption_force_specialist=False,
caption_router_enabled=True,
caption_router_model="router-model",
caption_router_device="cuda",
@@ -48,6 +52,49 @@ class SavedRouteReuseTests(unittest.TestCase):
self.assertAlmostEqual(routed[0][2], 0.91)
self.assertEqual(routed[0][3], "score")
def test_global_specialist_override_skips_classifier(self) -> None:
fake_settings = SimpleNamespace(
db_path=Path(self._testMethodName + ".sqlite3"),
caption_general_model="general-model",
caption_specialist_model="specialist-model",
caption_force_specialist=True,
caption_router_enabled=True,
caption_router_model="router-model",
caption_router_device="cuda",
caption_router_batch_size=32,
caption_router_threshold=0.5,
caption_router_context_seconds=6,
jellyfin_force_specialist_tag="scene-search:specialist",
jellyfin_force_general_tag="scene-search:general",
jellyfin_specialist_tags=(),
)
with tempfile.TemporaryDirectory() as tmp:
fake_settings.db_path = Path(tmp) / "scenes.sqlite3"
init_db(fake_settings.db_path)
row = {
"id": "frame-1",
"media_id": "media-1",
"timestamp_ms": 10000,
"caption_route": None,
"caption_route_reason": None,
"route_score": None,
"route_model": None,
"route_policy": None,
"jellyfin_tags": "[]",
}
with patch("scene_search.cli.settings", fake_settings):
with patch(
"scene_search.cli.ExplicitContentRouter",
side_effect=AssertionError("classifier should not be loaded"),
):
routed, missing = _route_rows([row])
self.assertEqual(missing, 0)
self.assertEqual(routed[0][1], "specialist")
self.assertEqual(routed[0][3], "global-specialist")
if __name__ == "__main__":
unittest.main()