diff --git a/.github/workflows/jellyfin-plugin.yml b/.github/workflows/jellyfin-plugin.yml new file mode 100644 index 0000000..4a005b8 --- /dev/null +++ b/.github/workflows/jellyfin-plugin.yml @@ -0,0 +1,33 @@ +name: Jellyfin plugin build + +on: + pull_request: + paths: + - "jellyfin-plugin/**" + - ".github/workflows/jellyfin-plugin.yml" + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up .NET 10 + uses: actions/setup-dotnet@v4 + with: + dotnet-version: "10.0.x" + + - name: Restore + run: dotnet restore jellyfin-plugin/Jellyfin.Plugin.SceneSearch/Jellyfin.Plugin.SceneSearch.csproj + + - name: Build + run: dotnet build jellyfin-plugin/Jellyfin.Plugin.SceneSearch/Jellyfin.Plugin.SceneSearch.csproj -c Release --no-restore + + - name: Upload plugin DLL + uses: actions/upload-artifact@v4 + with: + name: Jellyfin.Plugin.SceneSearch + path: jellyfin-plugin/Jellyfin.Plugin.SceneSearch/bin/Release/net10.0/Jellyfin.Plugin.SceneSearch.dll diff --git a/.gitignore b/.gitignore index 90a0356..5f45cd0 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,7 @@ __pycache__/ *.pyc data/* !data/.gitkeep + +# .NET Jellyfin plugin build output +**/bin/ +**/obj/ diff --git a/README.md b/README.md index 59b5fc5..d4ea502 100644 --- a/README.md +++ b/README.md @@ -322,6 +322,55 @@ When `SCENE_API_KEY` is set, search and thumbnail endpoints accept either `Autho The API intentionally runs with one worker. Multiple workers would each load their own copy of the embedding model into GPU memory. +## Jellyfin 12 search plugin + +The repository now includes a thin Jellyfin 12 plugin under `jellyfin-plugin/`. Jellyfin 12's external search-provider API accepts real Jellyfin item IDs with relevance scores, while Jellyfin core applies the requesting user's library visibility and access filtering afterwards. The plugin therefore sends the search text to the local Python API and returns the matching Jellyfin item IDs; it does not run AI models inside Jellyfin. + +### Build + +Jellyfin 12 plugins target .NET 10: + +```bash +dotnet build jellyfin-plugin/Jellyfin.Plugin.SceneSearch/Jellyfin.Plugin.SceneSearch.csproj -c Release +``` + +The plugin DLL is then: + +```text +jellyfin-plugin/Jellyfin.Plugin.SceneSearch/bin/Release/net10.0/Jellyfin.Plugin.SceneSearch.dll +``` + +Copy that DLL into a Scene Search directory under Jellyfin's plugin directory and restart Jellyfin. + +### Initial configuration + +Open **Dashboard → Plugins → Scene Search** and configure: + +- API URL: `http://127.0.0.1:8097` when Jellyfin and scene-search run on the same machine. +- API key: the same value as `SCENE_API_KEY`. +- Require a search prefix: enabled by default. +- Prefix: `scene:`. + +The prefix is intentional for the first rollout. Jellyfin 12 uses non-empty external-provider results instead of its built-in SQL search results. While only a test library is indexed, making Scene Search answer every query could hide ordinary Jellyfin results that have not been indexed yet. + +With the default configuration: + +```text +scene: woman standing outside +``` + +uses semantic Scene Search, while: + +```text +Alien +``` + +continues to use Jellyfin's normal search. + +Once the complete libraries you care about are indexed and semantic ranking is calibrated, the prefix requirement can be disabled in the plugin settings so normal Jellyfin searches use Scene Search directly. + +The first version contributes Movies, Episodes and generic Videos because those are the item types currently ingested by `jellyfin-index`. + ## Removing indexed sources Old prototype clips and Jellyfin-backed items are stored with different source kinds. diff --git a/jellyfin-plugin/Jellyfin.Plugin.SceneSearch/Configuration/PluginConfiguration.cs b/jellyfin-plugin/Jellyfin.Plugin.SceneSearch/Configuration/PluginConfiguration.cs new file mode 100644 index 0000000..4e4ba31 --- /dev/null +++ b/jellyfin-plugin/Jellyfin.Plugin.SceneSearch/Configuration/PluginConfiguration.cs @@ -0,0 +1,55 @@ +using MediaBrowser.Model.Plugins; + +namespace Jellyfin.Plugin.SceneSearch.Configuration; + +/// +/// Configuration for the Scene Search Jellyfin plugin. +/// +public class PluginConfiguration : BasePluginConfiguration +{ + public PluginConfiguration() + { + Enabled = true; + ApiUrl = "http://127.0.0.1:8097"; + ApiKey = string.Empty; + RequirePrefix = true; + SearchPrefix = "scene:"; + MaxResults = 20; + TimeoutSeconds = 60; + } + + /// + /// Gets or sets a value indicating whether semantic scene search is enabled. + /// + public bool Enabled { get; set; } + + /// + /// Gets or sets the scene-search HTTP API base URL. + /// + public string ApiUrl { get; set; } + + /// + /// Gets or sets the optional shared API key. + /// + public string ApiKey { get; set; } + + /// + /// Gets or sets a value indicating whether search terms must begin with . + /// + public bool RequirePrefix { get; set; } + + /// + /// Gets or sets the prefix used to opt in to scene search. + /// + public string SearchPrefix { get; set; } + + /// + /// Gets or sets the maximum number of Jellyfin items returned by scene search. + /// + public int MaxResults { get; set; } + + /// + /// Gets or sets the HTTP timeout in seconds. + /// + public int TimeoutSeconds { get; set; } +} diff --git a/jellyfin-plugin/Jellyfin.Plugin.SceneSearch/Configuration/configPage.html b/jellyfin-plugin/Jellyfin.Plugin.SceneSearch/Configuration/configPage.html new file mode 100644 index 0000000..e0e579e --- /dev/null +++ b/jellyfin-plugin/Jellyfin.Plugin.SceneSearch/Configuration/configPage.html @@ -0,0 +1,101 @@ + + + + Scene Search + + +
+
+
+

Scene Search

+

Connect Jellyfin search to the local semantic scene-search service.

+ +
+ + +
+ +
Usually http://127.0.0.1:8097 when Jellyfin and scene-search run on the same host.
+
+ +
+ +
Must match SCENE_API_KEY in jellyfin-scene-search. Leave empty only if the API has no key configured.
+
+ + +
+ Recommended while only part of the Jellyfin library is indexed. When enabled, ordinary searches keep using Jellyfin's built-in search. +
+ +
+ +
Default: scene:. Example: scene: woman standing outside
+
+ +
+ +
+ +
+ +
The first semantic search may take longer while SigLIP is loaded.
+
+ + +
+
+
+ + +
+ + diff --git a/jellyfin-plugin/Jellyfin.Plugin.SceneSearch/Jellyfin.Plugin.SceneSearch.csproj b/jellyfin-plugin/Jellyfin.Plugin.SceneSearch/Jellyfin.Plugin.SceneSearch.csproj new file mode 100644 index 0000000..0ee9402 --- /dev/null +++ b/jellyfin-plugin/Jellyfin.Plugin.SceneSearch/Jellyfin.Plugin.SceneSearch.csproj @@ -0,0 +1,28 @@ + + + net10.0 + Jellyfin.Plugin.SceneSearch + enable + enable + 0.1.0.0 + 0.1.0.0 + 0.1.0.0 + + + + + runtime + + + runtime + + + runtime + + + + + + + + diff --git a/jellyfin-plugin/Jellyfin.Plugin.SceneSearch/Plugin.cs b/jellyfin-plugin/Jellyfin.Plugin.SceneSearch/Plugin.cs new file mode 100644 index 0000000..3ddfc28 --- /dev/null +++ b/jellyfin-plugin/Jellyfin.Plugin.SceneSearch/Plugin.cs @@ -0,0 +1,47 @@ +using System.Globalization; +using Jellyfin.Plugin.SceneSearch.Configuration; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Plugins; +using MediaBrowser.Model.Plugins; +using MediaBrowser.Model.Serialization; + +namespace Jellyfin.Plugin.SceneSearch; + +/// +/// Jellyfin integration for semantic scene search. +/// +public sealed class Plugin : BasePlugin, IHasWebPages +{ + public Plugin(IApplicationPaths applicationPaths, IXmlSerializer xmlSerializer) + : base(applicationPaths, xmlSerializer) + { + Instance = this; + } + + /// + public override string Name => "Scene Search"; + + /// + public override Guid Id => Guid.Parse("2d8d1bb5-3f0c-4d5a-9d59-934463c06d35"); + + /// + /// Gets the current plugin instance. + /// + public static Plugin? Instance { get; private set; } + + /// + public IEnumerable GetPages() + { + return + [ + new PluginPageInfo + { + Name = Name, + EmbeddedResourcePath = string.Format( + CultureInfo.InvariantCulture, + "{0}.Configuration.configPage.html", + GetType().Namespace) + } + ]; + } +} diff --git a/jellyfin-plugin/Jellyfin.Plugin.SceneSearch/PluginServiceRegistrator.cs b/jellyfin-plugin/Jellyfin.Plugin.SceneSearch/PluginServiceRegistrator.cs new file mode 100644 index 0000000..5a87648 --- /dev/null +++ b/jellyfin-plugin/Jellyfin.Plugin.SceneSearch/PluginServiceRegistrator.cs @@ -0,0 +1,19 @@ +using MediaBrowser.Controller; +using MediaBrowser.Controller.Plugins; +using Microsoft.Extensions.DependencyInjection; + +namespace Jellyfin.Plugin.SceneSearch; + +/// +/// Registers Scene Search plugin dependencies. +/// +public sealed class PluginServiceRegistrator : IPluginServiceRegistrator +{ + /// + public void RegisterServices( + IServiceCollection serviceCollection, + IServerApplicationHost applicationHost) + { + serviceCollection.AddSingleton(); + } +} diff --git a/jellyfin-plugin/Jellyfin.Plugin.SceneSearch/SceneSearchClient.cs b/jellyfin-plugin/Jellyfin.Plugin.SceneSearch/SceneSearchClient.cs new file mode 100644 index 0000000..f694955 --- /dev/null +++ b/jellyfin-plugin/Jellyfin.Plugin.SceneSearch/SceneSearchClient.cs @@ -0,0 +1,114 @@ +using System.Net.Http.Headers; +using System.Net.Http.Json; +using Jellyfin.Plugin.SceneSearch.Configuration; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Plugin.SceneSearch; + +/// +/// HTTP client for the local scene-search service. +/// +public sealed class SceneSearchClient : IDisposable +{ + private readonly HttpClient _httpClient; + private readonly ILogger _logger; + + public SceneSearchClient(ILogger logger) + { + _logger = logger; + _httpClient = new HttpClient(); + } + + /// + /// Gets a value indicating whether enough configuration exists to search. + /// + public bool IsConfigured + { + get + { + var config = Configuration; + return config.Enabled + && Uri.TryCreate(config.ApiUrl, UriKind.Absolute, out _); + } + } + + private static PluginConfiguration Configuration + => Plugin.Instance?.Configuration ?? new PluginConfiguration(); + + /// + /// Searches the local scene-search API. + /// + internal async Task> SearchAsync( + string searchTerm, + int limit, + CancellationToken cancellationToken) + { + var config = Configuration; + if (!config.Enabled || string.IsNullOrWhiteSpace(config.ApiUrl)) + { + return []; + } + + var safeLimit = Math.Clamp(limit, 1, Math.Clamp(config.MaxResults, 1, 50)); + var requestUri = string.Concat( + config.ApiUrl.TrimEnd('/'), + "/api/search?q=", + Uri.EscapeDataString(searchTerm), + "&scene_limit=", + Math.Min(safeLimit * 4, 100).ToString(System.Globalization.CultureInfo.InvariantCulture), + "&item_limit=", + safeLimit.ToString(System.Globalization.CultureInfo.InvariantCulture)); + + using var request = new HttpRequestMessage(HttpMethod.Get, requestUri); + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + + if (!string.IsNullOrWhiteSpace(config.ApiKey)) + { + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", config.ApiKey); + } + + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeout.CancelAfter(TimeSpan.FromSeconds(Math.Clamp(config.TimeoutSeconds, 5, 300))); + + try + { + using var response = await _httpClient.SendAsync( + request, + HttpCompletionOption.ResponseHeadersRead, + timeout.Token).ConfigureAwait(false); + + if (!response.IsSuccessStatusCode) + { + _logger.LogWarning( + "Scene Search API returned HTTP {StatusCode} for '{SearchTerm}'", + (int)response.StatusCode, + searchTerm); + return []; + } + + var payload = await response.Content.ReadFromJsonAsync( + cancellationToken: timeout.Token).ConfigureAwait(false); + + return payload?.Items ?? []; + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + _logger.LogWarning( + "Scene Search API timed out after {TimeoutSeconds}s for '{SearchTerm}'", + config.TimeoutSeconds, + searchTerm); + return []; + } + catch (HttpRequestException ex) + { + _logger.LogWarning(ex, "Scene Search API request failed for '{SearchTerm}'", searchTerm); + return []; + } + } + + public void Dispose() + { + _httpClient.Dispose(); + GC.SuppressFinalize(this); + } +} diff --git a/jellyfin-plugin/Jellyfin.Plugin.SceneSearch/SceneSearchProvider.cs b/jellyfin-plugin/Jellyfin.Plugin.SceneSearch/SceneSearchProvider.cs new file mode 100644 index 0000000..5be4109 --- /dev/null +++ b/jellyfin-plugin/Jellyfin.Plugin.SceneSearch/SceneSearchProvider.cs @@ -0,0 +1,171 @@ +using System.Runtime.CompilerServices; +using Jellyfin.Data.Enums; +using Jellyfin.Plugin.SceneSearch.Configuration; +using MediaBrowser.Controller.Library; +using MediaBrowser.Model.Configuration; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Plugin.SceneSearch; + +/// +/// External Jellyfin search provider backed by jellyfin-scene-search. +/// +public sealed class SceneSearchProvider : IExternalSearchProvider +{ + private static readonly HashSet SupportedKinds = + [ + BaseItemKind.Movie, + BaseItemKind.Episode, + BaseItemKind.Video + ]; + + private readonly SceneSearchClient _client; + private readonly ILogger _logger; + + public SceneSearchProvider( + SceneSearchClient client, + ILogger logger) + { + _client = client; + _logger = logger; + } + + /// + public string Name => "Scene Search"; + + /// + public MetadataPluginType Type => MetadataPluginType.SearchProvider; + + /// + public int Priority => 20; + + /// + public bool CanSearch(SearchProviderQuery query) + { + if (!_client.IsConfigured || string.IsNullOrWhiteSpace(query.SearchTerm)) + { + return false; + } + + var config = Configuration; + if (config.RequirePrefix && !HasPrefix(query.SearchTerm, config.SearchPrefix)) + { + return false; + } + + if (query.IncludeItemTypes.Length > 0 + && !query.IncludeItemTypes.Any(SupportedKinds.Contains)) + { + return false; + } + + return true; + } + + /// + public async IAsyncEnumerable SearchAsync( + SearchProviderQuery query, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + if (!CanSearch(query)) + { + yield break; + } + + var config = Configuration; + var searchTerm = StripPrefix(query.SearchTerm, config); + if (string.IsNullOrWhiteSpace(searchTerm)) + { + yield break; + } + + var limit = Math.Min(query.Limit ?? config.MaxResults, config.MaxResults); + var results = await _client.SearchAsync( + searchTerm, + Math.Max(limit, 1), + cancellationToken).ConfigureAwait(false); + + foreach (var result in results) + { + if (!Guid.TryParse(result.JellyfinItemId, out var itemId) || itemId == Guid.Empty) + { + _logger.LogDebug( + "Ignoring Scene Search result with invalid Jellyfin item id {ItemId}", + result.JellyfinItemId); + continue; + } + + if (!MatchesRequestedType(result, query)) + { + continue; + } + + var score = result.RankingScore ?? result.SiglipScore ?? 0.5; + yield return new SearchResult(itemId, (float)Math.Clamp(score, 0.0, 1.0)); + } + } + + /// + async Task> ISearchProvider.SearchAsync( + SearchProviderQuery query, + CancellationToken cancellationToken) + { + var results = new List(); + await foreach (var result in SearchAsync(query, cancellationToken).ConfigureAwait(false)) + { + results.Add(result); + } + + return results; + } + + private static PluginConfiguration Configuration + => Plugin.Instance?.Configuration ?? new PluginConfiguration(); + + private static bool HasPrefix(string term, string prefix) + => !string.IsNullOrWhiteSpace(prefix) + && term.TrimStart().StartsWith(prefix, StringComparison.OrdinalIgnoreCase); + + private static string StripPrefix(string term, PluginConfiguration config) + { + var trimmed = term.Trim(); + if (!config.RequirePrefix || string.IsNullOrWhiteSpace(config.SearchPrefix)) + { + return trimmed; + } + + return trimmed.StartsWith(config.SearchPrefix, StringComparison.OrdinalIgnoreCase) + ? trimmed[config.SearchPrefix.Length..].Trim() + : trimmed; + } + + private static bool MatchesRequestedType( + SceneSearchItem result, + SearchProviderQuery query) + { + if (!Enum.TryParse( + result.JellyfinItemType, + ignoreCase: true, + out var kind)) + { + return query.IncludeItemTypes.Length == 0; + } + + if (!SupportedKinds.Contains(kind)) + { + return false; + } + + if (query.IncludeItemTypes.Length > 0 && !query.IncludeItemTypes.Contains(kind)) + { + return false; + } + + if (query.IncludeItemTypes.Length == 0 && query.ExcludeItemTypes.Contains(kind)) + { + return false; + } + + return true; + } +} diff --git a/jellyfin-plugin/Jellyfin.Plugin.SceneSearch/SceneSearchResponse.cs b/jellyfin-plugin/Jellyfin.Plugin.SceneSearch/SceneSearchResponse.cs new file mode 100644 index 0000000..c914bf8 --- /dev/null +++ b/jellyfin-plugin/Jellyfin.Plugin.SceneSearch/SceneSearchResponse.cs @@ -0,0 +1,36 @@ +using System.Text.Json.Serialization; + +namespace Jellyfin.Plugin.SceneSearch; + +internal sealed class SceneSearchResponse +{ + [JsonPropertyName("items")] + public List Items { get; set; } = []; +} + +internal sealed class SceneSearchItem +{ + [JsonPropertyName("jellyfin_item_id")] + public string? JellyfinItemId { get; set; } + + [JsonPropertyName("jellyfin_item_type")] + public string? JellyfinItemType { get; set; } + + [JsonPropertyName("title")] + public string? Title { get; set; } + + [JsonPropertyName("best_timestamp_ms")] + public long BestTimestampMs { get; set; } + + [JsonPropertyName("best_caption")] + public string? BestCaption { get; set; } + + [JsonPropertyName("ranking_score")] + public double? RankingScore { get; set; } + + [JsonPropertyName("siglip_score")] + public double? SiglipScore { get; set; } + + [JsonPropertyName("scene_matches")] + public int SceneMatches { get; set; } +} diff --git a/jellyfin-plugin/build.yaml b/jellyfin-plugin/build.yaml new file mode 100644 index 0000000..da4409c --- /dev/null +++ b/jellyfin-plugin/build.yaml @@ -0,0 +1,15 @@ +name: "Scene Search" +guid: "2d8d1bb5-3f0c-4d5a-9d59-934463c06d35" +version: "0.1.0.0" +targetAbi: "12.0.0.0" +framework: "net10.0" +overview: "Semantic scene search provider for Jellyfin" +description: > + Connects Jellyfin 12 search to jellyfin-scene-search and returns real Jellyfin + library items matched by visual scene embeddings and captions. +category: "Search" +owner: "MrSleeps" +artifacts: + - "Jellyfin.Plugin.SceneSearch.dll" +changelog: > + Initial Jellyfin 12 search-provider integration.