Merge pull request #4 from MrSleeps/improve/caption-specificity

Improve caption specificity for scene search
This commit is contained in:
Sleeps
2026-10-06 14:43:43 +01:00
committed by GitHub
7 changed files with 77 additions and 18 deletions
+2
View File
@@ -11,3 +11,5 @@ BATCH_SIZE=8
OLLAMA_URL=http://127.0.0.1:11434
CAPTION_MODEL=huihui_ai/qwen3-vl-abliterated:4b-instruct-q4_K_M
CAPTION_TIMEOUT_SECONDS=180
CAPTION_MAX_TOKENS=120
CAPTION_PROMPT_FILE=./data/caption_prompt.txt
+18 -4
View File
@@ -14,9 +14,10 @@ Meilisearch is the fast serving index, but it is not the only copy of expensive
```text
data/
├── scenes.sqlite3 # canonical vectors, timestamps and captions
├── thumbs/ # prototype frames used for visual verification/captioning
└── meili/ # Meilisearch's persistent serving index
├── scenes.sqlite3 # canonical vectors, timestamps and captions
├── caption_prompt.txt # local editable Ollama caption prompt
├── thumbs/ # prototype frames used for visual verification/captioning
└── meili/ # Meilisearch's persistent serving index
```
If Meilisearch is lost or rebuilt, `sync-meili` repopulates it from SQLite without re-running SigLIP or Ollama.
@@ -92,6 +93,8 @@ It was selected for the prototype because the project requirement is content-neu
OLLAMA_URL=http://127.0.0.1:11434
CAPTION_MODEL=your-vision-model
CAPTION_TIMEOUT_SECONDS=180
CAPTION_MAX_TOKENS=120
CAPTION_PROMPT_FILE=./data/caption_prompt.txt
```
Pull the configured model before captioning:
@@ -126,7 +129,18 @@ Use `--force` to regenerate existing captions with the currently configured mode
python -m scene_search.cli caption --title "Test clip 1" --force
```
The caption prompt deliberately asks for factual visual descriptions and tells the model not to add labels such as `NSFW`, `adult`, `safe` or `unsafe`. Mature content may be described if clearly visible, but there is no permanent adult-content classifier.
The default caption prompt is deliberately written for retrieval rather than politeness: it asks for dense, literal visual descriptions, discourages vague euphemisms, and requests precise visible anatomy/actions when relevant. It also tells the model not to add labels such as `NSFW`, `adult`, `safe` or `unsafe`. There is no permanent adult-content classifier.
The first `init` or `caption` run creates `data/caption_prompt.txt` from the repository default. That local file is ignored by git and is the prompt actually used for captioning, so prompt experiments do not require commits or pulls:
```bash
nano data/caption_prompt.txt
python -m scene_search.cli caption --title "Test clip 1" --limit 5 --force
```
Set `CAPTION_PROMPT_FILE` if you want the editable prompt somewhere else. If the local prompt file is deleted, the next run recreates it from `scene_search/prompts/caption.txt`.
If the 4B model is still too vague and the Ollama host has enough VRAM, an 8B Q4 vision model can be configured simply by changing `CAPTION_MODEL`; existing captions can then be regenerated with `caption --force`.
## 7. Hybrid search
+26 -12
View File
@@ -6,15 +6,26 @@ from pathlib import Path
import requests
DEFAULT_CAPTION_PROMPT = (
"Describe only what is visibly present in this video frame in one concise sentence "
"for later visual search. Mention people, broad age group only when visually clear, "
"actions, setting, notable objects, clothing, weather, and other distinctive details. "
"If nudity, sexual activity, violence, gore, or other mature content is clearly visible, "
"describe it factually rather than omitting or euphemising it. Do not infer names, "
"identities, relationships, intent, or anything not visible. Do not label the image as "
"safe, unsafe, adult, NSFW, or similar. Output the description only."
)
DEFAULT_CAPTION_PROMPT_PATH = Path(__file__).parent / "prompts" / "caption.txt"
def load_default_caption_prompt() -> str:
prompt = DEFAULT_CAPTION_PROMPT_PATH.read_text(encoding="utf-8").strip()
if not prompt:
raise RuntimeError(f"Default caption prompt is empty: {DEFAULT_CAPTION_PROMPT_PATH}")
return prompt
def ensure_caption_prompt_file(prompt_path: Path) -> str:
prompt_path = prompt_path.expanduser().resolve()
if not prompt_path.exists():
prompt_path.parent.mkdir(parents=True, exist_ok=True)
prompt_path.write_text(load_default_caption_prompt() + "\n", encoding="utf-8")
prompt = prompt_path.read_text(encoding="utf-8").strip()
if not prompt:
raise RuntimeError(f"Caption prompt is empty: {prompt_path}")
return prompt
class OllamaCaptioner:
@@ -23,12 +34,14 @@ class OllamaCaptioner:
url: str,
model: str,
timeout_seconds: int = 180,
prompt: str = DEFAULT_CAPTION_PROMPT,
prompt: str | None = None,
max_tokens: int = 120,
) -> None:
self.url = url.rstrip("/")
self.model = model
self.timeout_seconds = timeout_seconds
self.prompt = prompt
self.prompt = prompt or load_default_caption_prompt()
self.max_tokens = max_tokens
self.session = requests.Session()
def caption(self, image_path: Path) -> str:
@@ -46,7 +59,8 @@ class OllamaCaptioner:
}
],
"options": {
"temperature": 0.1,
"temperature": 0.0,
"num_predict": self.max_tokens,
},
},
timeout=self.timeout_seconds,
+7 -1
View File
@@ -6,7 +6,7 @@ from pathlib import Path
from PIL import Image
from .captioner import OllamaCaptioner
from .captioner import OllamaCaptioner, ensure_caption_prompt_file
from .config import settings
from .database import connect, init_db, pack_vector, unpack_vector
from .embedder import SiglipEmbedder
@@ -28,7 +28,9 @@ def build_embedder() -> SiglipEmbedder:
def cmd_init(_: argparse.Namespace) -> None:
init_db(settings.db_path)
ensure_caption_prompt_file(settings.caption_prompt_file)
print(f"SQLite ready: {settings.db_path}")
print(f"Caption prompt: {settings.caption_prompt_file}")
def cmd_index_file(args: argparse.Namespace) -> None:
@@ -210,14 +212,18 @@ def cmd_caption(args: argparse.Namespace) -> None:
print("No frames need captioning.")
return
caption_prompt = ensure_caption_prompt_file(settings.caption_prompt_file)
print(
f"Captioning {len(rows)} frame(s) with {settings.caption_model} "
f"via {settings.ollama_url}"
)
print(f"Prompt: {settings.caption_prompt_file}")
captioner = OllamaCaptioner(
settings.ollama_url,
settings.caption_model,
settings.caption_timeout_seconds,
caption_prompt,
settings.caption_max_tokens,
)
completed = 0
+4
View File
@@ -26,6 +26,10 @@ class Settings:
"huihui_ai/qwen3-vl-abliterated:4b-instruct-q4_K_M",
)
caption_timeout_seconds: int = int(os.getenv("CAPTION_TIMEOUT_SECONDS", "180"))
caption_max_tokens: int = int(os.getenv("CAPTION_MAX_TOKENS", "120"))
caption_prompt_file: Path = Path(
os.getenv("CAPTION_PROMPT_FILE", "./data/caption_prompt.txt")
).expanduser().resolve()
data_dir: Path = Path(os.getenv("DATA_DIR", "./data")).resolve()
+7
View File
@@ -0,0 +1,7 @@
Write a dense visual-search caption of up to about 50 words describing only what is clearly visible in this video frame. Include the setting, people, broad age group only when visually clear, clothing or nudity, body position, actions, notable objects, and other distinctive details that would help someone find this exact moment later.
Use direct, literal terminology. If explicit sexual activity or visible anatomy is clearly shown, name the visible body parts and the specific visible action rather than using vague euphemisms such as "being intimate", "adult activity", "suggestive", or "in a compromising position". Likewise describe visible violence or gore directly.
Do not infer names, identities, relationships, intent, emotions, or anything that cannot be seen. Do not label the frame as safe, unsafe, adult, NSFW, explicit, pornographic, or similar; describe the scene itself. If a detail is uncertain, omit it rather than guessing.
Output only the caption.
+13 -1
View File
@@ -5,7 +5,7 @@ import tempfile
import unittest
from unittest.mock import Mock
from scene_search.captioner import OllamaCaptioner
from scene_search.captioner import OllamaCaptioner, ensure_caption_prompt_file
class OllamaCaptionerTests(unittest.TestCase):
@@ -30,6 +30,18 @@ class OllamaCaptionerTests(unittest.TestCase):
self.assertEqual(payload["messages"][0]["role"], "user")
self.assertTrue(payload["messages"][0]["images"][0])
def test_prompt_file_is_created_once_and_remains_editable(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
prompt_path = Path(tmp) / "caption_prompt.txt"
first = ensure_caption_prompt_file(prompt_path)
self.assertTrue(prompt_path.is_file())
self.assertTrue(first)
prompt_path.write_text("My test prompt\n", encoding="utf-8")
second = ensure_caption_prompt_file(prompt_path)
self.assertEqual(second, "My test prompt")
if __name__ == "__main__":
unittest.main()