mirror of
https://github.com/MrSleeps/jellyfin-scene-search.git
synced 2026-11-17 07:21:28 +00:00
Merge pull request #4 from MrSleeps/improve/caption-specificity
Improve caption specificity for scene search
This commit is contained in:
@@ -11,3 +11,5 @@ BATCH_SIZE=8
|
||||
OLLAMA_URL=http://127.0.0.1:11434
|
||||
CAPTION_MODEL=huihui_ai/qwen3-vl-abliterated:4b-instruct-q4_K_M
|
||||
CAPTION_TIMEOUT_SECONDS=180
|
||||
CAPTION_MAX_TOKENS=120
|
||||
CAPTION_PROMPT_FILE=./data/caption_prompt.txt
|
||||
|
||||
@@ -14,9 +14,10 @@ Meilisearch is the fast serving index, but it is not the only copy of expensive
|
||||
|
||||
```text
|
||||
data/
|
||||
├── scenes.sqlite3 # canonical vectors, timestamps and captions
|
||||
├── thumbs/ # prototype frames used for visual verification/captioning
|
||||
└── meili/ # Meilisearch's persistent serving index
|
||||
├── scenes.sqlite3 # canonical vectors, timestamps and captions
|
||||
├── caption_prompt.txt # local editable Ollama caption prompt
|
||||
├── thumbs/ # prototype frames used for visual verification/captioning
|
||||
└── meili/ # Meilisearch's persistent serving index
|
||||
```
|
||||
|
||||
If Meilisearch is lost or rebuilt, `sync-meili` repopulates it from SQLite without re-running SigLIP or Ollama.
|
||||
@@ -92,6 +93,8 @@ It was selected for the prototype because the project requirement is content-neu
|
||||
OLLAMA_URL=http://127.0.0.1:11434
|
||||
CAPTION_MODEL=your-vision-model
|
||||
CAPTION_TIMEOUT_SECONDS=180
|
||||
CAPTION_MAX_TOKENS=120
|
||||
CAPTION_PROMPT_FILE=./data/caption_prompt.txt
|
||||
```
|
||||
|
||||
Pull the configured model before captioning:
|
||||
@@ -126,7 +129,18 @@ Use `--force` to regenerate existing captions with the currently configured mode
|
||||
python -m scene_search.cli caption --title "Test clip 1" --force
|
||||
```
|
||||
|
||||
The caption prompt deliberately asks for factual visual descriptions and tells the model not to add labels such as `NSFW`, `adult`, `safe` or `unsafe`. Mature content may be described if clearly visible, but there is no permanent adult-content classifier.
|
||||
The default caption prompt is deliberately written for retrieval rather than politeness: it asks for dense, literal visual descriptions, discourages vague euphemisms, and requests precise visible anatomy/actions when relevant. It also tells the model not to add labels such as `NSFW`, `adult`, `safe` or `unsafe`. There is no permanent adult-content classifier.
|
||||
|
||||
The first `init` or `caption` run creates `data/caption_prompt.txt` from the repository default. That local file is ignored by git and is the prompt actually used for captioning, so prompt experiments do not require commits or pulls:
|
||||
|
||||
```bash
|
||||
nano data/caption_prompt.txt
|
||||
python -m scene_search.cli caption --title "Test clip 1" --limit 5 --force
|
||||
```
|
||||
|
||||
Set `CAPTION_PROMPT_FILE` if you want the editable prompt somewhere else. If the local prompt file is deleted, the next run recreates it from `scene_search/prompts/caption.txt`.
|
||||
|
||||
If the 4B model is still too vague and the Ollama host has enough VRAM, an 8B Q4 vision model can be configured simply by changing `CAPTION_MODEL`; existing captions can then be regenerated with `caption --force`.
|
||||
|
||||
## 7. Hybrid search
|
||||
|
||||
|
||||
+26
-12
@@ -6,15 +6,26 @@ from pathlib import Path
|
||||
import requests
|
||||
|
||||
|
||||
DEFAULT_CAPTION_PROMPT = (
|
||||
"Describe only what is visibly present in this video frame in one concise sentence "
|
||||
"for later visual search. Mention people, broad age group only when visually clear, "
|
||||
"actions, setting, notable objects, clothing, weather, and other distinctive details. "
|
||||
"If nudity, sexual activity, violence, gore, or other mature content is clearly visible, "
|
||||
"describe it factually rather than omitting or euphemising it. Do not infer names, "
|
||||
"identities, relationships, intent, or anything not visible. Do not label the image as "
|
||||
"safe, unsafe, adult, NSFW, or similar. Output the description only."
|
||||
)
|
||||
DEFAULT_CAPTION_PROMPT_PATH = Path(__file__).parent / "prompts" / "caption.txt"
|
||||
|
||||
|
||||
def load_default_caption_prompt() -> str:
|
||||
prompt = DEFAULT_CAPTION_PROMPT_PATH.read_text(encoding="utf-8").strip()
|
||||
if not prompt:
|
||||
raise RuntimeError(f"Default caption prompt is empty: {DEFAULT_CAPTION_PROMPT_PATH}")
|
||||
return prompt
|
||||
|
||||
|
||||
def ensure_caption_prompt_file(prompt_path: Path) -> str:
|
||||
prompt_path = prompt_path.expanduser().resolve()
|
||||
if not prompt_path.exists():
|
||||
prompt_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
prompt_path.write_text(load_default_caption_prompt() + "\n", encoding="utf-8")
|
||||
|
||||
prompt = prompt_path.read_text(encoding="utf-8").strip()
|
||||
if not prompt:
|
||||
raise RuntimeError(f"Caption prompt is empty: {prompt_path}")
|
||||
return prompt
|
||||
|
||||
|
||||
class OllamaCaptioner:
|
||||
@@ -23,12 +34,14 @@ class OllamaCaptioner:
|
||||
url: str,
|
||||
model: str,
|
||||
timeout_seconds: int = 180,
|
||||
prompt: str = DEFAULT_CAPTION_PROMPT,
|
||||
prompt: str | None = None,
|
||||
max_tokens: int = 120,
|
||||
) -> None:
|
||||
self.url = url.rstrip("/")
|
||||
self.model = model
|
||||
self.timeout_seconds = timeout_seconds
|
||||
self.prompt = prompt
|
||||
self.prompt = prompt or load_default_caption_prompt()
|
||||
self.max_tokens = max_tokens
|
||||
self.session = requests.Session()
|
||||
|
||||
def caption(self, image_path: Path) -> str:
|
||||
@@ -46,7 +59,8 @@ class OllamaCaptioner:
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"temperature": 0.1,
|
||||
"temperature": 0.0,
|
||||
"num_predict": self.max_tokens,
|
||||
},
|
||||
},
|
||||
timeout=self.timeout_seconds,
|
||||
|
||||
+7
-1
@@ -6,7 +6,7 @@ from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from .captioner import OllamaCaptioner
|
||||
from .captioner import OllamaCaptioner, ensure_caption_prompt_file
|
||||
from .config import settings
|
||||
from .database import connect, init_db, pack_vector, unpack_vector
|
||||
from .embedder import SiglipEmbedder
|
||||
@@ -28,7 +28,9 @@ def build_embedder() -> SiglipEmbedder:
|
||||
|
||||
def cmd_init(_: argparse.Namespace) -> None:
|
||||
init_db(settings.db_path)
|
||||
ensure_caption_prompt_file(settings.caption_prompt_file)
|
||||
print(f"SQLite ready: {settings.db_path}")
|
||||
print(f"Caption prompt: {settings.caption_prompt_file}")
|
||||
|
||||
|
||||
def cmd_index_file(args: argparse.Namespace) -> None:
|
||||
@@ -210,14 +212,18 @@ def cmd_caption(args: argparse.Namespace) -> None:
|
||||
print("No frames need captioning.")
|
||||
return
|
||||
|
||||
caption_prompt = ensure_caption_prompt_file(settings.caption_prompt_file)
|
||||
print(
|
||||
f"Captioning {len(rows)} frame(s) with {settings.caption_model} "
|
||||
f"via {settings.ollama_url}"
|
||||
)
|
||||
print(f"Prompt: {settings.caption_prompt_file}")
|
||||
captioner = OllamaCaptioner(
|
||||
settings.ollama_url,
|
||||
settings.caption_model,
|
||||
settings.caption_timeout_seconds,
|
||||
caption_prompt,
|
||||
settings.caption_max_tokens,
|
||||
)
|
||||
|
||||
completed = 0
|
||||
|
||||
@@ -26,6 +26,10 @@ class Settings:
|
||||
"huihui_ai/qwen3-vl-abliterated:4b-instruct-q4_K_M",
|
||||
)
|
||||
caption_timeout_seconds: int = int(os.getenv("CAPTION_TIMEOUT_SECONDS", "180"))
|
||||
caption_max_tokens: int = int(os.getenv("CAPTION_MAX_TOKENS", "120"))
|
||||
caption_prompt_file: Path = Path(
|
||||
os.getenv("CAPTION_PROMPT_FILE", "./data/caption_prompt.txt")
|
||||
).expanduser().resolve()
|
||||
|
||||
data_dir: Path = Path(os.getenv("DATA_DIR", "./data")).resolve()
|
||||
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
Write a dense visual-search caption of up to about 50 words describing only what is clearly visible in this video frame. Include the setting, people, broad age group only when visually clear, clothing or nudity, body position, actions, notable objects, and other distinctive details that would help someone find this exact moment later.
|
||||
|
||||
Use direct, literal terminology. If explicit sexual activity or visible anatomy is clearly shown, name the visible body parts and the specific visible action rather than using vague euphemisms such as "being intimate", "adult activity", "suggestive", or "in a compromising position". Likewise describe visible violence or gore directly.
|
||||
|
||||
Do not infer names, identities, relationships, intent, emotions, or anything that cannot be seen. Do not label the frame as safe, unsafe, adult, NSFW, explicit, pornographic, or similar; describe the scene itself. If a detail is uncertain, omit it rather than guessing.
|
||||
|
||||
Output only the caption.
|
||||
+13
-1
@@ -5,7 +5,7 @@ import tempfile
|
||||
import unittest
|
||||
from unittest.mock import Mock
|
||||
|
||||
from scene_search.captioner import OllamaCaptioner
|
||||
from scene_search.captioner import OllamaCaptioner, ensure_caption_prompt_file
|
||||
|
||||
|
||||
class OllamaCaptionerTests(unittest.TestCase):
|
||||
@@ -30,6 +30,18 @@ class OllamaCaptionerTests(unittest.TestCase):
|
||||
self.assertEqual(payload["messages"][0]["role"], "user")
|
||||
self.assertTrue(payload["messages"][0]["images"][0])
|
||||
|
||||
def test_prompt_file_is_created_once_and_remains_editable(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
prompt_path = Path(tmp) / "caption_prompt.txt"
|
||||
first = ensure_caption_prompt_file(prompt_path)
|
||||
self.assertTrue(prompt_path.is_file())
|
||||
self.assertTrue(first)
|
||||
|
||||
prompt_path.write_text("My test prompt\n", encoding="utf-8")
|
||||
second = ensure_caption_prompt_file(prompt_path)
|
||||
|
||||
self.assertEqual(second, "My test prompt")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user