From 61178832c1e2b95fe5df015c034b383e40eb5035 Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Mon, 31 Oct 2022 01:00:39 -0500 Subject: [PATCH 01/22] Add initial end credits detection code --- CHANGELOG.md | 2 + .../TestAudioFingerprinting.cs | 10 +- .../Analyzers/Chromaprint.cs | 54 +++-- .../Configuration/PluginConfiguration.cs | 5 + .../Controllers/VisualizationController.cs | 2 +- .../Data/QueuedEpisode.cs | 14 +- .../FFmpegWrapper.cs | 90 ++++++-- .../Plugin.cs | 63 +++++- .../QueueManager.cs | 15 +- .../ScheduledTasks/AnalyzeEpisodesTask.cs | 2 + .../ScheduledTasks/DetectCreditsTask.cs | 197 ++++++++++++++++++ README.md | 6 +- 12 files changed, 409 insertions(+), 51 deletions(-) create mode 100644 ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7727813..cd38246 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Changelog ## v0.1.8.0 (no eta) +* New features + * Detect ending credits in television episodes * Internal changes * Move Chromaprint analysis code out of the episode analysis task * Add support for multiple analysis techinques diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs index 5470b4f..e758469 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs @@ -59,7 +59,9 @@ public void TestFingerprinting() 3472417825, 3395841056, 3458735136, 3341420624, 1076496560, 1076501168, 1076501136, 1076497024 }; - var actual = FFmpegWrapper.Fingerprint(queueEpisode("audio/big_buck_bunny_intro.mp3")); + var actual = FFmpegWrapper.Fingerprint( + queueEpisode("audio/big_buck_bunny_intro.mp3"), + AnalysisMode.Introduction); Assert.Equal(expected, actual); } @@ -91,8 +93,8 @@ public void TestIntroDetection() var lhsEpisode = queueEpisode("audio/big_buck_bunny_intro.mp3"); var rhsEpisode = queueEpisode("audio/big_buck_bunny_clip.mp3"); - var lhsFingerprint = FFmpegWrapper.Fingerprint(lhsEpisode); - var rhsFingerprint = FFmpegWrapper.Fingerprint(rhsEpisode); + var lhsFingerprint = FFmpegWrapper.Fingerprint(lhsEpisode, AnalysisMode.Introduction); + var rhsFingerprint = FFmpegWrapper.Fingerprint(rhsEpisode, AnalysisMode.Introduction); var (lhs, rhs) = chromaprint.CompareEpisodes( lhsEpisode.EpisodeId, @@ -138,7 +140,7 @@ private QueuedEpisode queueEpisode(string path) { EpisodeId = Guid.NewGuid(), Path = "../../../" + path, - FingerprintDuration = 60 + IntroFingerprintEnd = 60 }; } diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/Chromaprint.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/Chromaprint.cs index 319d678..04f9da0 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/Chromaprint.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/Chromaprint.cs @@ -30,6 +30,8 @@ public class ChromaprintAnalyzer : IMediaFileAnalyzer private ILogger _logger; + private AnalysisMode _analysisMode; + /// /// Initializes a new instance of the class. /// @@ -64,12 +66,14 @@ public ReadOnlyCollection AnalyzeMediaFiles( // Episodes that were analyzed and do not have an introduction. var episodesWithoutIntros = new List(); + this._analysisMode = mode; + // Compute fingerprints for all episodes in the season foreach (var episode in episodeAnalysisQueue) { try { - fingerprintCache[episode.EpisodeId] = FFmpegWrapper.Fingerprint(episode); + fingerprintCache[episode.EpisodeId] = FFmpegWrapper.Fingerprint(episode, mode); if (cancellationToken.IsCancellationRequested) { @@ -78,6 +82,7 @@ public ReadOnlyCollection AnalyzeMediaFiles( } catch (FingerprintException ex) { + // TODO: FIXME: move to debug level? _logger.LogWarning("Caught fingerprint error: {Ex}", ex); // Fallback to an empty fingerprint on any error @@ -112,6 +117,22 @@ public ReadOnlyCollection AnalyzeMediaFiles( continue; } + /* Since the Fingerprint() function returns an array of Chromaprint points without time + * information, the times reported from the index search function start from 0. + * + * While this is desired behavior for detecting introductions, it breaks credit + * detection, as the audio we're analyzing was extracted from some point into the file. + * + * To fix this, add the starting time of the fingerprint to the reported time range. + */ + if (this._analysisMode == AnalysisMode.Credits) + { + currentIntro.IntroStart += currentEpisode.CreditsFingerprintStart; + currentIntro.IntroEnd += currentEpisode.CreditsFingerprintStart; + remainingIntro.IntroStart += remainingEpisode.CreditsFingerprintStart; + remainingIntro.IntroEnd += remainingEpisode.CreditsFingerprintStart; + } + // Only save the discovered intro if it is: // - the first intro discovered for this episode // - longer than the previously discovered intro @@ -142,10 +163,13 @@ public ReadOnlyCollection AnalyzeMediaFiles( return analysisQueue; } - // Adjust all introduction end times so that they end at silence. - seasonIntros = AdjustIntroEndTimes(analysisQueue, seasonIntros); + if (this._analysisMode == AnalysisMode.Introduction) + { + // Adjust all introduction end times so that they end at silence. + seasonIntros = AdjustIntroEndTimes(analysisQueue, seasonIntros); + } - Plugin.Instance!.UpdateTimestamps(seasonIntros); + Plugin.Instance!.UpdateTimestamps(seasonIntros, this._analysisMode); return episodesWithoutIntros.AsReadOnly(); } @@ -338,16 +362,20 @@ public ReadOnlyCollection AnalyzeMediaFiles( // Since LHS had a contiguous time range, RHS must have one also. var rContiguous = TimeRangeHelpers.FindContiguous(rhsTimes.ToArray(), maximumTimeSkip)!; - // Tweak the end timestamps just a bit to ensure as little content as possible is skipped over. - if (lContiguous.Duration >= 90) + if (this._analysisMode == AnalysisMode.Introduction) { - lContiguous.End -= 2 * maximumTimeSkip; - rContiguous.End -= 2 * maximumTimeSkip; - } - else if (lContiguous.Duration >= 30) - { - lContiguous.End -= maximumTimeSkip; - rContiguous.End -= maximumTimeSkip; + // Tweak the end timestamps just a bit to ensure as little content as possible is skipped over. + // TODO: remove this + if (lContiguous.Duration >= 90) + { + lContiguous.End -= 2 * maximumTimeSkip; + rContiguous.End -= 2 * maximumTimeSkip; + } + else if (lContiguous.Duration >= 30) + { + lContiguous.End -= maximumTimeSkip; + rContiguous.End -= maximumTimeSkip; + } } return (lContiguous, rContiguous); diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/PluginConfiguration.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/PluginConfiguration.cs index e382bf2..ce79fdd 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/PluginConfiguration.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/PluginConfiguration.cs @@ -72,6 +72,11 @@ public PluginConfiguration() /// public int MaximumIntroDuration { get; set; } = 120; + /// + /// Gets or sets the upper limit (in minutes) on the length of each episode's audio track that will be analyzed when searching for ending credits. + /// + public int MaximumEpisodeCreditsDuration { get; set; } = 4; + // ===== Playback settings ===== /// diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/VisualizationController.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/VisualizationController.cs index d4cec0d..f2331ab 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/VisualizationController.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/VisualizationController.cs @@ -113,7 +113,7 @@ public ActionResult GetEpisodeFingerprint([FromRoute] Guid id) { if (needle.EpisodeId == id) { - return FFmpegWrapper.Fingerprint(needle); + return FFmpegWrapper.Fingerprint(needle, AnalysisMode.Introduction); } } } diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Data/QueuedEpisode.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Data/QueuedEpisode.cs index 915d33f..0964258 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Data/QueuedEpisode.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Data/QueuedEpisode.cs @@ -33,7 +33,17 @@ public class QueuedEpisode public string Name { get; set; } = string.Empty; /// - /// Gets or sets the seconds of media file to fingerprint. + /// Gets or sets the timestamp (in seconds) to stop searching for an introduction. /// - public int FingerprintDuration { get; set; } + public int IntroFingerprintEnd { get; set; } + + /// + /// Gets or sets the timestamp (in seconds) to start looking for end credits. + /// + public int CreditsFingerprintStart { get; set; } + + /// + /// Gets or sets the total duration of this media file (in seconds). + /// + public int Duration { get; set; } } diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs b/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs index 86dad74..fdae152 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs @@ -134,27 +134,60 @@ private static bool CheckFFmpegRequirement( /// Fingerprint a queued episode. /// /// Queued episode to fingerprint. + /// Portion of media file to fingerprint. Introduction = first 25% / 10 minutes and Credits = last 4 minutes. /// Numerical fingerprint points. - public static uint[] Fingerprint(QueuedEpisode episode) + public static uint[] Fingerprint(QueuedEpisode episode, AnalysisMode mode) + { + int start, end; + + if (mode == AnalysisMode.Introduction) + { + start = 0; + end = episode.IntroFingerprintEnd; + } + else if (mode == AnalysisMode.Credits) + { + start = episode.CreditsFingerprintStart; + end = episode.Duration; + } + else + { + throw new ArgumentException("Unknown analysis mode " + mode.ToString()); + } + + return Fingerprint(episode, mode, start, end); + } + + /// + /// Fingerprint a queued episode. + /// + /// Queued episode to fingerprint. + /// Portion of media file to fingerprint. + /// Time (in seconds) relative to the start of the file to start fingerprinting from. + /// Time (in seconds) relative to the start of the file to stop fingerprinting at. + /// Numerical fingerprint points. + private static uint[] Fingerprint(QueuedEpisode episode, AnalysisMode mode, int start, int end) { // Try to load this episode from cache before running ffmpeg. - if (LoadCachedFingerprint(episode, out uint[] cachedFingerprint)) + if (LoadCachedFingerprint(episode, mode, out uint[] cachedFingerprint)) { Logger?.LogTrace("Fingerprint cache hit on {File}", episode.Path); return cachedFingerprint; } Logger?.LogDebug( - "Fingerprinting {Duration} seconds from \"{File}\" (id {Id})", - episode.FingerprintDuration, + "Fingerprinting [{Start}, {End}] from \"{File}\" (id {Id})", + start, + end, episode.Path, episode.EpisodeId); var args = string.Format( CultureInfo.InvariantCulture, - "-i \"{0}\" -to {1} -ac 2 -f chromaprint -fp_format raw -", + "-ss {0} -i \"{1}\" -to {2} -ac 2 -f chromaprint -fp_format raw -", + start, episode.Path, - episode.FingerprintDuration); + end - start); // Returns all fingerprint points as raw 32 bit unsigned integers (little endian). var rawPoints = GetOutput(args, string.Empty); @@ -172,7 +205,7 @@ public static uint[] Fingerprint(QueuedEpisode episode) } // Try to cache this fingerprint. - CacheFingerprint(episode, results); + CacheFingerprint(episode, mode, results); return results.ToArray(); } @@ -226,9 +259,6 @@ public static TimeRange[] DetectSilence(QueuedEpisode episode, int limit) limit, episode.EpisodeId); - // TODO: select the audio track that matches the user's preferred language, falling - // back to the first track if nothing matches - // -vn, -sn, -dn: ignore video, subtitle, and data tracks var args = string.Format( CultureInfo.InvariantCulture, @@ -367,9 +397,13 @@ private static ReadOnlySpan GetOutput( /// This function was created before the unified caching mechanism was introduced (in v0.1.7). /// /// Episode to try to load from cache. + /// Analysis mode. /// Array to store the fingerprint in. /// true if the episode was successfully loaded from cache, false on any other error. - private static bool LoadCachedFingerprint(QueuedEpisode episode, out uint[] fingerprint) + private static bool LoadCachedFingerprint( + QueuedEpisode episode, + AnalysisMode mode, + out uint[] fingerprint) { fingerprint = Array.Empty(); @@ -379,7 +413,7 @@ private static bool LoadCachedFingerprint(QueuedEpisode episode, out uint[] fing return false; } - var path = GetFingerprintCachePath(episode); + var path = GetFingerprintCachePath(episode, mode); // If this episode isn't cached, bail out. if (!File.Exists(path)) @@ -387,7 +421,6 @@ private static bool LoadCachedFingerprint(QueuedEpisode episode, out uint[] fing return false; } - // TODO: make async var raw = File.ReadAllLines(path, Encoding.UTF8); var result = new List(); @@ -421,8 +454,12 @@ private static bool LoadCachedFingerprint(QueuedEpisode episode, out uint[] fing /// This function was created before the unified caching mechanism was introduced (in v0.1.7). /// /// Episode to store in cache. + /// Analysis mode. /// Fingerprint of the episode to store. - private static void CacheFingerprint(QueuedEpisode episode, List fingerprint) + private static void CacheFingerprint( + QueuedEpisode episode, + AnalysisMode mode, + List fingerprint) { // Bail out if caching isn't enabled. if (!(Plugin.Instance?.Configuration.CacheFingerprints ?? false)) @@ -438,7 +475,10 @@ private static void CacheFingerprint(QueuedEpisode episode, List fingerpri } // Cache the episode. - File.WriteAllLinesAsync(GetFingerprintCachePath(episode), lines, Encoding.UTF8).ConfigureAwait(false); + File.WriteAllLinesAsync( + GetFingerprintCachePath(episode, mode), + lines, + Encoding.UTF8).ConfigureAwait(false); } /// @@ -446,9 +486,25 @@ private static void CacheFingerprint(QueuedEpisode episode, List fingerpri /// This function was created before the unified caching mechanism was introduced (in v0.1.7). /// /// Episode. - private static string GetFingerprintCachePath(QueuedEpisode episode) + /// Analysis mode. + private static string GetFingerprintCachePath(QueuedEpisode episode, AnalysisMode mode) { - return Path.Join(Plugin.Instance!.FingerprintCachePath, episode.EpisodeId.ToString("N")); + var basePath = Path.Join( + Plugin.Instance!.FingerprintCachePath, + episode.EpisodeId.ToString("N")); + + if (mode == AnalysisMode.Introduction) + { + return basePath; + } + else if (mode == AnalysisMode.Credits) + { + return basePath + "-credits"; + } + else + { + throw new ArgumentException("Unknown analysis mode " + mode.ToString()); + } } /// diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Plugin.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Plugin.cs index 94604d9..bf5d241 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Plugin.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Plugin.cs @@ -6,6 +6,7 @@ using MediaBrowser.Common.Plugins; using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Entities.TV; using MediaBrowser.Controller.Library; using MediaBrowser.Model.Plugins; using MediaBrowser.Model.Serialization; @@ -24,6 +25,7 @@ public class Plugin : BasePlugin, IHasWebPages private ILibraryManager _libraryManager; private ILogger _logger; private string _introPath; + private string _creditsPath; // TODO: FIXME: remove this /// /// Initializes a new instance of the class. @@ -51,6 +53,9 @@ public Plugin( FFmpegPath = serverConfiguration.GetEncodingOptions().EncoderAppPathDisplay; _introPath = Path.Join(applicationPaths.PluginConfigurationsPath, "intros", "intros.xml"); + // TODO: FIXME: remove this + _creditsPath = Path.Join(applicationPaths.PluginConfigurationsPath, "intros", "credits.csv"); + // Create the base & cache directories (if needed). if (!Directory.Exists(FingerprintCachePath)) { @@ -68,6 +73,12 @@ public Plugin( { _logger.LogWarning("Unable to load introduction timestamps: {Exception}", ex); } + + // TODO: FIXME: remove this + if (File.Exists(_creditsPath)) + { + File.Delete(_creditsPath); + } } /// @@ -163,16 +174,52 @@ internal string GetItemPath(Guid id) return GetItem(id).Path; } - internal void UpdateTimestamps(Dictionary newIntros) + internal void UpdateTimestamps(Dictionary newIntros, AnalysisMode mode) { - lock (_introsLock) + switch (mode) { - foreach (var intro in newIntros) - { - Plugin.Instance!.Intros[intro.Key] = intro.Value; - } - - Plugin.Instance!.SaveTimestamps(); + case AnalysisMode.Introduction: + lock (_introsLock) + { + foreach (var intro in newIntros) + { + Plugin.Instance!.Intros[intro.Key] = intro.Value; + } + + Plugin.Instance!.SaveTimestamps(); + } + + break; + + case AnalysisMode.Credits: + // TODO: FIXME: implement properly + + lock (_introsLock) + { + foreach (var credit in newIntros) + { + var item = GetItem(credit.Value.EpisodeId) as Episode; + if (item is null) + { + continue; + } + + // Format: series, season number, episode number, title, start, end + var contents = string.Format( + System.Globalization.CultureInfo.InvariantCulture, + "{0},{1},{2},{3},{4},{5}\n", + item.SeriesName.Replace(",", string.Empty, StringComparison.Ordinal), + item.AiredSeasonNumber ?? 0, + item.IndexNumber ?? 0, + item.Name.Replace(",", string.Empty, StringComparison.Ordinal), + Math.Round(credit.Value.IntroStart, 2), + Math.Round(credit.Value.IntroEnd, 2)); + + File.AppendAllText(_creditsPath, contents); + } + } + + break; } } diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/QueueManager.cs b/ConfusedPolarBear.Plugin.IntroSkipper/QueueManager.cs index 3395243..be905e6 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/QueueManager.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/QueueManager.cs @@ -186,17 +186,22 @@ private void QueueEpisode(Episode episode) // Limit analysis to the first X% of the episode and at most Y minutes. // X and Y default to 25% and 10 minutes. var duration = TimeSpan.FromTicks(episode.RunTimeTicks ?? 0).TotalSeconds; - if (duration >= 5 * 60) + var fingerprintDuration = duration; + + if (fingerprintDuration >= 5 * 60) { - duration *= analysisPercent; + fingerprintDuration *= analysisPercent; } - duration = Math.Min(duration, 60 * Plugin.Instance!.Configuration.AnalysisLengthLimit); + fingerprintDuration = Math.Min( + fingerprintDuration, + 60 * Plugin.Instance!.Configuration.AnalysisLengthLimit); // Allocate a new list for each new season Plugin.Instance!.AnalysisQueue.TryAdd(episode.SeasonId, new List()); // Queue the episode for analysis + var maxCreditsDuration = Plugin.Instance!.Configuration.MaximumEpisodeCreditsDuration * 60; Plugin.Instance.AnalysisQueue[episode.SeasonId].Add(new QueuedEpisode() { SeriesName = episode.SeriesName, @@ -204,7 +209,9 @@ private void QueueEpisode(Episode episode) EpisodeId = episode.Id, Name = episode.Name, Path = episode.Path, - FingerprintDuration = Convert.ToInt32(duration) + Duration = Convert.ToInt32(duration), + IntroFingerprintEnd = Convert.ToInt32(fingerprintDuration), + CreditsFingerprintStart = Convert.ToInt32(duration - maxCreditsDuration), }); Plugin.Instance!.TotalQueued++; diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs index 3b1c531..6d563fc 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs @@ -12,6 +12,7 @@ namespace ConfusedPolarBear.Plugin.IntroSkipper; /// /// Analyze all television episodes for introduction sequences. +/// TODO: FIXME: rename task and file to DetectIntroductionsTask. /// public class AnalyzeEpisodesTask : IScheduledTask { @@ -178,6 +179,7 @@ public Task ExecuteAsync(IProgress progress, CancellationToken cancellat /// /// Verify that all episodes in a season exist in Jellyfin and as a file in storage. + /// TODO: FIXME: move to queue manager. /// /// QueuedEpisodes. /// Verified QueuedEpisodes and a flag indicating if any episode in this season has not been analyzed yet. diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs new file mode 100644 index 0000000..224155f --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs @@ -0,0 +1,197 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Threading; +using System.Threading.Tasks; +using MediaBrowser.Controller.Library; +using MediaBrowser.Model.Tasks; +using Microsoft.Extensions.Logging; + +namespace ConfusedPolarBear.Plugin.IntroSkipper; + +#if !DEBUG +#error Fix all FIXMEs introduced during initial credit implementation before release +#endif + +/// +/// Analyze all television episodes for credits. +/// +public class DetectCreditsTask : IScheduledTask +{ + private readonly ILogger _logger; + + private readonly ILoggerFactory _loggerFactory; + + private readonly ILibraryManager? _libraryManager; + + /// + /// Initializes a new instance of the class. + /// + /// Logger factory. + /// Library manager. + public DetectCreditsTask( + ILoggerFactory loggerFactory, + ILibraryManager libraryManager) : this(loggerFactory) + { + _libraryManager = libraryManager; + } + + /// + /// Initializes a new instance of the class. + /// + /// Logger factory. + public DetectCreditsTask(ILoggerFactory loggerFactory) + { + _logger = loggerFactory.CreateLogger(); + _loggerFactory = loggerFactory; + } + + /// + /// Gets the task name. + /// + public string Name => "Detect Credits"; + + /// + /// Gets the task category. + /// + public string Category => "Intro Skipper"; + + /// + /// Gets the task description. + /// + public string Description => "Analyzes the audio and video of all television episodes to find credits."; + + /// + /// Gets the task key. + /// + public string Key => "CPBIntroSkipperDetectCredits"; + + /// + /// Analyze all episodes in the queue. Only one instance of this task should be run at a time. + /// + /// Task progress. + /// Cancellation token. + /// Task. + public Task ExecuteAsync(IProgress progress, CancellationToken cancellationToken) + { + if (_libraryManager is null) + { + throw new InvalidOperationException("Library manager must not be null"); + } + + // Make sure the analysis queue matches what's currently in Jellyfin. + var queueManager = new QueueManager( + _loggerFactory.CreateLogger(), + _libraryManager); + + queueManager.EnqueueAllEpisodes(); + + var queue = Plugin.Instance!.AnalysisQueue; + + if (queue.Count == 0) + { + throw new FingerprintException( + "No episodes to analyze. If you are limiting the list of libraries to analyze, check that all library names have been spelled correctly."); + } + + var totalProcessed = 0; + var options = new ParallelOptions() + { + MaxDegreeOfParallelism = Plugin.Instance!.Configuration.MaxParallelism + }; + + // TODO: FIXME: if the queue is modified while the task is running, the task will fail. + // clone the queue before running the task to prevent this. + + // Analyze all episodes in the queue using the degrees of parallelism the user specified. + Parallel.ForEach(queue, options, (season) => + { + // TODO: FIXME: use VerifyEpisodes + var episodes = season.Value.AsReadOnly(); + if (episodes.Count == 0) + { + return; + } + + var first = episodes[0]; + + try + { + if (cancellationToken.IsCancellationRequested) + { + return; + } + + // Increment totalProcessed by the number of episodes in this season that were actually analyzed + // (instead of just using the number of episodes in the current season). + var analyzed = AnalyzeSeason(episodes, cancellationToken); + Interlocked.Add(ref totalProcessed, analyzed); + } + catch (FingerprintException ex) + { + _logger.LogWarning( + "Unable to analyze {Series} season {Season}: unable to fingerprint: {Ex}", + first.SeriesName, + first.SeasonNumber, + ex); + } + catch (KeyNotFoundException ex) + { + _logger.LogWarning( + "Unable to analyze {Series} season {Season}: cache miss: {Ex}", + first.SeriesName, + first.SeasonNumber, + ex); + } + + progress.Report((totalProcessed * 100) / Plugin.Instance!.TotalQueued); + }); + + return Task.CompletedTask; + } + + /// + /// Fingerprints all episodes in the provided season and stores the timestamps of all introductions. + /// + /// Episodes in this season. + /// Cancellation token provided by the scheduled task. + /// Number of episodes from the provided season that were analyzed. + private int AnalyzeSeason( + ReadOnlyCollection episodes, + CancellationToken cancellationToken) + { + // Skip seasons with an insufficient number of episodes. + if (episodes.Count <= 1) + { + return episodes.Count; + } + + // Only analyze specials (season 0) if the user has opted in. + var first = episodes[0]; + if (first.SeasonNumber == 0 && !Plugin.Instance!.Configuration.AnalyzeSeasonZero) + { + return 0; + } + + _logger.LogInformation( + "Analyzing {Count} episodes from {Name} season {Season}", + episodes.Count, + first.SeriesName, + first.SeasonNumber); + + // Analyze the season with Chromaprint + var chromaprint = new ChromaprintAnalyzer(_loggerFactory.CreateLogger()); + chromaprint.AnalyzeMediaFiles(episodes, AnalysisMode.Credits, cancellationToken); + + return episodes.Count; + } + + /// + /// Get task triggers. + /// + /// Task triggers. + public IEnumerable GetDefaultTriggers() + { + return Array.Empty(); + } +} diff --git a/README.md b/README.md index 8d3589d..e3ac776 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Plugin Banner -Analyzes the audio of television episodes to detect and skip over intros. +Analyzes the audio of television episodes to detect and skip over introductions and ending credits. If you use the custom web interface on your server, you will be able to click a button to skip intros, like this: @@ -20,13 +20,15 @@ However, if you want to use an unmodified installation of Jellyfin 10.8.z or use * `linuxserver/jellyfin` 10.8.z container: preinstalled * Debian Linux based native installs: provided by the `jellyfin-ffmpeg5` package -## Introduction requirements +## Introduction and end credit requirements Show introductions will only be detected if they are: * Located within the first 25% of an episode, or the first 10 minutes, whichever is smaller * Between 15 seconds and 2 minutes long +Ending credits will only be detected if they are shorter than 4 minutes. + All of these requirements can be customized as needed. ## Installation instructions From ff9ba16300829747f5a853e4ac965da99e5fcf6e Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Mon, 31 Oct 2022 01:16:55 -0500 Subject: [PATCH 02/22] Update e2e test code --- .../e2e_tests/verifier/report_generator.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/report_generator.go b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/report_generator.go index 4814022..708f864 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/report_generator.go +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/report_generator.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "os" + "os/exec" "strings" "time" @@ -77,7 +78,6 @@ func generateReport(hostAddress, apiKey, reportDestination string, keepTimestamp fmt.Println() fmt.Println("[+] Saving report") - // TODO: also save analysis statistics // Store timing data, server information, and plugin configuration report.StartedAt = start report.FinishedAt = time.Now() @@ -95,6 +95,9 @@ func generateReport(hostAddress, apiKey, reportDestination string, keepTimestamp panic(err) } + // Change report permissions + exec.Command("chown", "1000:1000", reportDestination).Run() + fmt.Println("[+] Done") } From af89e5f2b426b7cbf36c0826e0705de74712d0fb Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Mon, 31 Oct 2022 01:47:41 -0500 Subject: [PATCH 03/22] Reorganize FFmpeg wrapper --- .../FFmpegWrapper.cs | 242 +++++++++--------- 1 file changed, 121 insertions(+), 121 deletions(-) diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs b/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs index fdae152..66fc68b 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs @@ -98,38 +98,6 @@ public static bool CheckFFmpegVersion() } } - /// - /// Run an FFmpeg command with the provided arguments and validate that the output contains - /// the provided string. - /// - /// Arguments to pass to FFmpeg. - /// String that the output must contain. Case insensitive. - /// Support bundle key to store FFmpeg's output under. - /// Error message to log if this requirement is not met. - /// true on success, false on error. - private static bool CheckFFmpegRequirement( - string arguments, - string mustContain, - string bundleName, - string errorMessage) - { - Logger?.LogDebug("Checking FFmpeg requirement {Arguments}", arguments); - - var output = Encoding.UTF8.GetString(GetOutput(arguments, string.Empty, false, 2000)); - Logger?.LogTrace("Output of ffmpeg {Arguments}: {Output}", arguments, output); - ChromaprintLogs[bundleName] = output; - - if (!output.Contains(mustContain, StringComparison.OrdinalIgnoreCase)) - { - Logger?.LogError("{ErrorMessage}", errorMessage); - return false; - } - - Logger?.LogDebug("FFmpeg requirement {Arguments} met", arguments); - - return true; - } - /// /// Fingerprint a queued episode. /// @@ -158,58 +126,6 @@ public static uint[] Fingerprint(QueuedEpisode episode, AnalysisMode mode) return Fingerprint(episode, mode, start, end); } - /// - /// Fingerprint a queued episode. - /// - /// Queued episode to fingerprint. - /// Portion of media file to fingerprint. - /// Time (in seconds) relative to the start of the file to start fingerprinting from. - /// Time (in seconds) relative to the start of the file to stop fingerprinting at. - /// Numerical fingerprint points. - private static uint[] Fingerprint(QueuedEpisode episode, AnalysisMode mode, int start, int end) - { - // Try to load this episode from cache before running ffmpeg. - if (LoadCachedFingerprint(episode, mode, out uint[] cachedFingerprint)) - { - Logger?.LogTrace("Fingerprint cache hit on {File}", episode.Path); - return cachedFingerprint; - } - - Logger?.LogDebug( - "Fingerprinting [{Start}, {End}] from \"{File}\" (id {Id})", - start, - end, - episode.Path, - episode.EpisodeId); - - var args = string.Format( - CultureInfo.InvariantCulture, - "-ss {0} -i \"{1}\" -to {2} -ac 2 -f chromaprint -fp_format raw -", - start, - episode.Path, - end - start); - - // Returns all fingerprint points as raw 32 bit unsigned integers (little endian). - var rawPoints = GetOutput(args, string.Empty); - if (rawPoints.Length == 0 || rawPoints.Length % 4 != 0) - { - Logger?.LogWarning("Chromaprint returned {Count} points for \"{Path}\"", rawPoints.Length, episode.Path); - throw new FingerprintException("chromaprint output for \"" + episode.Path + "\" was malformed"); - } - - var results = new List(); - for (var i = 0; i < rawPoints.Length; i += 4) - { - var rawPoint = rawPoints.Slice(i, 4); - results.Add(BitConverter.ToUInt32(rawPoint)); - } - - // Try to cache this fingerprint. - CacheFingerprint(episode, mode, results); - - return results.ToArray(); - } - /// /// Transforms a Chromaprint into an inverted index of fingerprint points to the last index it appeared at. /// @@ -295,6 +211,75 @@ public static TimeRange[] DetectSilence(QueuedEpisode episode, int limit) return silenceRanges.ToArray(); } + /// + /// Gets Chromaprint debugging logs. + /// + /// Markdown formatted logs. + public static string GetChromaprintLogs() + { + // Print the FFmpeg detection status at the top. + // Format: "* FFmpeg: `error`" + // Append two newlines to separate the bulleted list from the logs + var logs = string.Format( + CultureInfo.InvariantCulture, + "* FFmpeg: `{0}`\n\n", + ChromaprintLogs["error"]); + + // Always include ffmpeg version information + logs += FormatFFmpegLog("version"); + + // Don't print feature detection logs if the plugin started up okay + if (ChromaprintLogs["error"] == "okay") + { + return logs; + } + + // Print all remaining logs + foreach (var kvp in ChromaprintLogs) + { + if (kvp.Key == "error" || kvp.Key == "version") + { + continue; + } + + logs += FormatFFmpegLog(kvp.Key); + } + + return logs; + } + + /// + /// Run an FFmpeg command with the provided arguments and validate that the output contains + /// the provided string. + /// + /// Arguments to pass to FFmpeg. + /// String that the output must contain. Case insensitive. + /// Support bundle key to store FFmpeg's output under. + /// Error message to log if this requirement is not met. + /// true on success, false on error. + private static bool CheckFFmpegRequirement( + string arguments, + string mustContain, + string bundleName, + string errorMessage) + { + Logger?.LogDebug("Checking FFmpeg requirement {Arguments}", arguments); + + var output = Encoding.UTF8.GetString(GetOutput(arguments, string.Empty, false, 2000)); + Logger?.LogTrace("Output of ffmpeg {Arguments}: {Output}", arguments, output); + ChromaprintLogs[bundleName] = output; + + if (!output.Contains(mustContain, StringComparison.OrdinalIgnoreCase)) + { + Logger?.LogError("{ErrorMessage}", errorMessage); + return false; + } + + Logger?.LogDebug("FFmpeg requirement {Arguments} met", arguments); + + return true; + } + /// /// Runs ffmpeg and returns standard output (or error). /// If caching is enabled, will use cacheFilename to cache the output of this command. @@ -392,6 +377,58 @@ private static ReadOnlySpan GetOutput( } } + /// + /// Fingerprint a queued episode. + /// + /// Queued episode to fingerprint. + /// Portion of media file to fingerprint. + /// Time (in seconds) relative to the start of the file to start fingerprinting from. + /// Time (in seconds) relative to the start of the file to stop fingerprinting at. + /// Numerical fingerprint points. + private static uint[] Fingerprint(QueuedEpisode episode, AnalysisMode mode, int start, int end) + { + // Try to load this episode from cache before running ffmpeg. + if (LoadCachedFingerprint(episode, mode, out uint[] cachedFingerprint)) + { + Logger?.LogTrace("Fingerprint cache hit on {File}", episode.Path); + return cachedFingerprint; + } + + Logger?.LogDebug( + "Fingerprinting [{Start}, {End}] from \"{File}\" (id {Id})", + start, + end, + episode.Path, + episode.EpisodeId); + + var args = string.Format( + CultureInfo.InvariantCulture, + "-ss {0} -i \"{1}\" -to {2} -ac 2 -f chromaprint -fp_format raw -", + start, + episode.Path, + end - start); + + // Returns all fingerprint points as raw 32 bit unsigned integers (little endian). + var rawPoints = GetOutput(args, string.Empty); + if (rawPoints.Length == 0 || rawPoints.Length % 4 != 0) + { + Logger?.LogWarning("Chromaprint returned {Count} points for \"{Path}\"", rawPoints.Length, episode.Path); + throw new FingerprintException("chromaprint output for \"" + episode.Path + "\" was malformed"); + } + + var results = new List(); + for (var i = 0; i < rawPoints.Length; i += 4) + { + var rawPoint = rawPoints.Slice(i, 4); + results.Add(BitConverter.ToUInt32(rawPoint)); + } + + // Try to cache this fingerprint. + CacheFingerprint(episode, mode, results); + + return results.ToArray(); + } + /// /// Tries to load an episode's fingerprint from cache. If caching is not enabled, calling this function is a no-op. /// This function was created before the unified caching mechanism was introduced (in v0.1.7). @@ -507,43 +544,6 @@ private static string GetFingerprintCachePath(QueuedEpisode episode, AnalysisMod } } - /// - /// Gets Chromaprint debugging logs. - /// - /// Markdown formatted logs. - public static string GetChromaprintLogs() - { - // Print the FFmpeg detection status at the top. - // Format: "* FFmpeg: `error`" - // Append two newlines to separate the bulleted list from the logs - var logs = string.Format( - CultureInfo.InvariantCulture, - "* FFmpeg: `{0}`\n\n", - ChromaprintLogs["error"]); - - // Always include ffmpeg version information - logs += FormatFFmpegLog("version"); - - // Don't print feature detection logs if the plugin started up okay - if (ChromaprintLogs["error"] == "okay") - { - return logs; - } - - // Print all remaining logs - foreach (var kvp in ChromaprintLogs) - { - if (kvp.Key == "error" || kvp.Key == "version") - { - continue; - } - - logs += FormatFFmpegLog(kvp.Key); - } - - return logs; - } - private static string FormatFFmpegLog(string key) { /* Format: From 8ee400f1f1b0e6e1a263787579f32285376e18d6 Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Tue, 1 Nov 2022 00:53:56 -0500 Subject: [PATCH 04/22] Add blackframe API --- .../TestBlackFrames.cs | 50 +++++++++++ .../video/rainbow.mp4 | Bin 0 -> 15787 bytes .../Configuration/PluginConfiguration.cs | 5 ++ .../Data/BlackFrame.cs | 28 ++++++ .../FFmpegWrapper.cs | 80 ++++++++++++++++-- 5 files changed, 154 insertions(+), 9 deletions(-) create mode 100644 ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestBlackFrames.cs create mode 100644 ConfusedPolarBear.Plugin.IntroSkipper.Tests/video/rainbow.mp4 create mode 100644 ConfusedPolarBear.Plugin.IntroSkipper/Data/BlackFrame.cs diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestBlackFrames.cs b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestBlackFrames.cs new file mode 100644 index 0000000..794ecd1 --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestBlackFrames.cs @@ -0,0 +1,50 @@ +namespace ConfusedPolarBear.Plugin.IntroSkipper.Tests; + +using System; +using System.Collections.Generic; +using Xunit; + +public class TestBlackFrames +{ + [FactSkipFFmpegTests] + public void TestBlackFrameDetection() + { + var expected = new List(); + expected.AddRange(CreateFrameSequence(2, 3)); + expected.AddRange(CreateFrameSequence(5, 6)); + expected.AddRange(CreateFrameSequence(8, 9.96)); + + var actual = FFmpegWrapper.DetectBlackFrames( + queueFile("rainbow.mp4"), + new TimeRange(0, 10) + ); + + for (var i = 0; i < expected.Count; i++) + { + var (e, a) = (expected[i], actual[i]); + Assert.Equal(e.Percentage, a.Percentage); + Assert.True(Math.Abs(e.Time - a.Time) <= 0.005); + } + } + + private QueuedEpisode queueFile(string path) + { + return new() + { + EpisodeId = Guid.NewGuid(), + Path = "../../../video/" + path + }; + } + + private BlackFrame[] CreateFrameSequence(double start, double end) + { + var frames = new List(); + + for (var i = start; i < end; i += 0.04) + { + frames.Add(new(100, i)); + } + + return frames.ToArray(); + } +} diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/video/rainbow.mp4 b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/video/rainbow.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8e01cce620da0f4d72f647169d5f42f3a9b67c08 GIT binary patch literal 15787 zcmeHOeQXp(6rbbxN<*KgT2xGX-MxMnHuh+$MT-LALsUHGcK3QqcW<}5)Aovy z)Cj~sBqkb5C`lU0wJz5b|ldj0xy`Fm*~I z@!rL{E1&&1SbwCm@B%mZ;LpFr{N<%w5hodz!U<(%oaL_)DmWfjmdaI?P*Lv#h2rHa z>YgYn8DMSfnO+7Uv`>f++Iwl-g@Dl3MDqdvoGET+Bq zB1G3f$1sU%MD1!$!XYe)0%s{sD5>=!YB9`)Wm=>l>YAJREnp;wEbh@Uk@uAYYf@k*$1$FyFO6+WAoS`hRPuCVva}DOPdi*m+T=*IZcd;pvh3YJWLLxV zwuB4tITX(bbZ@VlRal$?vCoA_0`|HP6c8W|0ECtT(eFZ>th;+0#FcTQxH=BvS`xxY z#Cx|Q^eydYMxbZrc!-{&)KT;lk3q~1>@H10G$!KRImjD!5mu!kG^Y-t#~z0eOCqdG z#4Br1_A(cta|EFS^o<1Mz3)Y}rWIDaB4HW=ka%?whoG^|432Z?4Pw?d0Bv)*oHfJhMH)_U#XM5XBb6~8DR$RmcRd!v$znJ&HHMR& zPj|xkbk|(nFk0c=g{HSgWXMyJ_cB5c=O7QFe`9H!0L5u?&6wt`xJd}j3_y5t6I8xH znw&npDT+xn=r=v3^>eF!JvGqin$xgAk%OYbB=ptzHhW6nOR%PNK%PM6c? z?gyPAwC1?et|%gFmJ5LpjglYXX}*?5&DZ0otSM9{TZXu48ibqMsym>m!!D5Bx_lXe zvuR^Uw0=g5rExBwU0{=t27JFw?2!$jI)~^F1t6y>71J%KfoazRONpj`Xt>2GpLq!g zb4KNWTI+$J?Zo&061?phiQWEdc$h-unAkQUr@c_hg-G(`A}`EGh!V=PiL(K~IyyJ$ zj~*aZ!o29l(3Ay4+usXR0@MIR7k3NLPM||TCxCuHXxewu7#?8JcCe97PGM4IDpJ|$ z5AsZ}XYxJLQ)n*mVBn3G{g-@Cntqx*A${b*81965OvHY$wHkZ2)1Fn7CVMAh zttOxklxq{ofd3Ji=v{#a3bn`tL;jquI61IrLFy$9E_N0+O?rTf=2lKB3$v@ciHQ!_Ged literal 0 HcmV?d00001 diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/PluginConfiguration.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/PluginConfiguration.cs index ce79fdd..f4789db 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/PluginConfiguration.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/PluginConfiguration.cs @@ -77,6 +77,11 @@ public PluginConfiguration() /// public int MaximumEpisodeCreditsDuration { get; set; } = 4; + /// + /// Gets or sets the minimum percentage of a frame that must consist of black pixels before it is considered a black frame. + /// + public int BlackFrameMinimumPercentage { get; set; } = 85; + // ===== Playback settings ===== /// diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Data/BlackFrame.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Data/BlackFrame.cs new file mode 100644 index 0000000..df3a957 --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Data/BlackFrame.cs @@ -0,0 +1,28 @@ +namespace ConfusedPolarBear.Plugin.IntroSkipper; + +/// +/// A frame of video that partially (or entirely) consists of black pixels. +/// +public class BlackFrame +{ + /// + /// Initializes a new instance of the class. + /// + /// Percentage of the frame that is black. + /// Time this frame appears at. + public BlackFrame(int percent, double time) + { + Percentage = percent; + Time = time; + } + + /// + /// Gets or sets the percentage of the frame that is black. + /// + public int Percentage { get; set; } + + /// + /// Gets or sets the time (in seconds) this frame appeared at. + /// + public double Time { get; set; } +} diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs b/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs index 66fc68b..e8ba464 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs @@ -16,16 +16,17 @@ public static class FFmpegWrapper { private static readonly object InvertedIndexCacheLock = new(); - // FFmpeg logs lines similar to the following: - // [silencedetect @ 0x000000000000] silence_start: 12.34 - // [silencedetect @ 0x000000000000] silence_end: 56.123 | silence_duration: 43.783 - /// /// Used with FFmpeg's silencedetect filter to extract the start and end times of silence. /// private static readonly Regex SilenceDetectionExpression = new( "silence_(?start|end): (? /// Media file to analyze. /// Time range to search. - /// Array of frames that are at least 50% black. - public static BlackFrame[] DetectBlackFrames(QueuedEpisode episode, TimeRange range) + /// Percentage of the frame that must be black. + /// Array of frames that are mostly black. + public static BlackFrame[] DetectBlackFrames( + QueuedEpisode episode, + TimeRange range, + int minimum) { // Seek to the start of the time range and find frames that are at least 50% black. var args = string.Format( @@ -233,10 +237,10 @@ public static BlackFrame[] DetectBlackFrames(QueuedEpisode episode, TimeRange ra episode.Path, range.End - range.Start); - // Cache the results to GUID-blackframes-v1-START-END. + // Cache the results to GUID-blackframes-START-END-v1. var cacheKey = string.Format( CultureInfo.InvariantCulture, - "{0}-blackframes-v1-{1}-{2}", + "{0}-blackframes-{1}-{2}-v1", episode.EpisodeId.ToString("N"), range.Start, range.End); @@ -263,10 +267,14 @@ public static BlackFrame[] DetectBlackFrames(QueuedEpisode episode, TimeRange ra matches[1].Value.Split(':')[1] ); - blackFrames.Add(new( + var bf = new BlackFrame( Convert.ToInt32(strPercent, CultureInfo.InvariantCulture), - Convert.ToDouble(strTime, CultureInfo.InvariantCulture) - )); + Convert.ToDouble(strTime, CultureInfo.InvariantCulture)); + + if (bf.Percentage > minimum) + { + blackFrames.Add(bf); + } } return blackFrames.ToArray(); diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs index 224155f..d4da24e 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs @@ -122,10 +122,8 @@ public Task ExecuteAsync(IProgress progress, CancellationToken cancellat return; } - // Increment totalProcessed by the number of episodes in this season that were actually analyzed - // (instead of just using the number of episodes in the current season). - var analyzed = AnalyzeSeason(episodes, cancellationToken); - Interlocked.Add(ref totalProcessed, analyzed); + AnalyzeSeason(episodes, cancellationToken); + Interlocked.Add(ref totalProcessed, episodes.Count); } catch (FingerprintException ex) { @@ -151,39 +149,49 @@ public Task ExecuteAsync(IProgress progress, CancellationToken cancellat } /// - /// Fingerprints all episodes in the provided season and stores the timestamps of all introductions. + /// Analyzes all episodes in the season for end credits. /// /// Episodes in this season. /// Cancellation token provided by the scheduled task. - /// Number of episodes from the provided season that were analyzed. - private int AnalyzeSeason( + private void AnalyzeSeason( ReadOnlyCollection episodes, CancellationToken cancellationToken) { - // Skip seasons with an insufficient number of episodes. - if (episodes.Count <= 1) + // Only analyze specials (season 0) if the user has opted in. + if (episodes[0].SeasonNumber == 0 && !Plugin.Instance!.Configuration.AnalyzeSeasonZero) { - return episodes.Count; + return; } - // Only analyze specials (season 0) if the user has opted in. - var first = episodes[0]; - if (first.SeasonNumber == 0 && !Plugin.Instance!.Configuration.AnalyzeSeasonZero) + // Analyze with Chromaprint first and fall back to the black frame detector + var analyzers = new IMediaFileAnalyzer[] + { + // TODO: FIXME: new ChromaprintAnalyzer(_loggerFactory.CreateLogger()), + new BlackFrameAnalyzer(_loggerFactory.CreateLogger()) + }; + + // Use each analyzer to find credits in all media files, removing successfully analyzed files + // from the queue. + var remaining = new ReadOnlyCollection(episodes); + foreach (var analyzer in analyzers) { - return 0; + remaining = AnalyzeFiles(remaining, analyzer, cancellationToken); } + } + private ReadOnlyCollection AnalyzeFiles( + ReadOnlyCollection episodes, + IMediaFileAnalyzer analyzer, + CancellationToken cancellationToken) + { _logger.LogInformation( - "Analyzing {Count} episodes from {Name} season {Season}", + "Analyzing {Count} episodes from {Name} season {Season} with {Analyzer}", episodes.Count, - first.SeriesName, - first.SeasonNumber); - - // Analyze the season with Chromaprint - var chromaprint = new ChromaprintAnalyzer(_loggerFactory.CreateLogger()); - chromaprint.AnalyzeMediaFiles(episodes, AnalysisMode.Credits, cancellationToken); + episodes[0].SeriesName, + episodes[0].SeasonNumber, + analyzer.GetType().Name); - return episodes.Count; + return analyzer.AnalyzeMediaFiles(episodes, AnalysisMode.Credits, cancellationToken); } /// From 546bf7578c0e0c0be9c2bd8866f6738db5a6b806 Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Wed, 2 Nov 2022 17:07:45 -0500 Subject: [PATCH 07/22] Rename DetectIntroductionsTask.cs --- ...EpisodesTask.cs => DetectIntroductionsTask.cs} | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) rename ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/{AnalyzeEpisodesTask.cs => DetectIntroductionsTask.cs} (94%) diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectIntroductionsTask.cs similarity index 94% rename from ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs rename to ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectIntroductionsTask.cs index 6d563fc..416cd80 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/AnalyzeEpisodesTask.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectIntroductionsTask.cs @@ -12,22 +12,21 @@ namespace ConfusedPolarBear.Plugin.IntroSkipper; /// /// Analyze all television episodes for introduction sequences. -/// TODO: FIXME: rename task and file to DetectIntroductionsTask. /// -public class AnalyzeEpisodesTask : IScheduledTask +public class DetectIntroductionsTask : IScheduledTask { - private readonly ILogger _logger; + private readonly ILogger _logger; private readonly ILoggerFactory _loggerFactory; private readonly ILibraryManager? _libraryManager; /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// Logger factory. /// Library manager. - public AnalyzeEpisodesTask( + public DetectIntroductionsTask( ILoggerFactory loggerFactory, ILibraryManager libraryManager) : this(loggerFactory) { @@ -35,12 +34,12 @@ public AnalyzeEpisodesTask( } /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// Logger factory. - public AnalyzeEpisodesTask(ILoggerFactory loggerFactory) + public DetectIntroductionsTask(ILoggerFactory loggerFactory) { - _logger = loggerFactory.CreateLogger(); + _logger = loggerFactory.CreateLogger(); _loggerFactory = loggerFactory; EdlManager.Initialize(_logger); From 2c3577ea23fbf7fa827226a20ce11288b04f3c14 Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Tue, 8 Nov 2022 20:55:23 -0600 Subject: [PATCH 08/22] Change skip button selector --- .../e2e_tests/selenium/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/selenium/main.py b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/selenium/main.py index e602cd8..ceaa25b 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/selenium/main.py +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/selenium/main.py @@ -152,7 +152,7 @@ def test_skip_button(driver, server): # Find the skip intro button and click it, logging the new video position after the seek is preformed print(" [+] Clicking skip intro button") - driver.find_element(By.CSS_SELECTOR, "button.btnSkipIntro").click() + driver.find_element(By.CSS_SELECTOR, "div#skipIntro").click() time.sleep(1) screenshot(driver, "skip_button_post_skip") assert_video_playing(driver) From bcb5e5ea300c9e4a3efe3c9517d937ab6667e065 Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Wed, 23 Nov 2022 02:34:28 -0600 Subject: [PATCH 09/22] Remove dependency on shared media item queue --- .../Controllers/TroubleshootingController.cs | 5 +-- .../Controllers/VisualizationController.cs | 8 ++--- .../Entrypoint.cs | 7 +--- .../Plugin.cs | 4 +-- .../QueueManager.cs | 34 +++++++++++-------- .../ScheduledTasks/DetectCreditsTask.cs | 10 +++--- .../ScheduledTasks/DetectIntroductionsTask.cs | 10 +++--- 7 files changed, 38 insertions(+), 40 deletions(-) diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/TroubleshootingController.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/TroubleshootingController.cs index 8923daf..1b1b901 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/TroubleshootingController.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/TroubleshootingController.cs @@ -53,10 +53,7 @@ public ActionResult GetSupportBundle() bundle.Append("* Queue contents: "); bundle.Append(Plugin.Instance!.TotalQueued); - bundle.Append(" episodes, "); - bundle.Append(Plugin.Instance!.AnalysisQueue.Count); - bundle.Append(" seasons"); - bundle.Append('\n'); + bundle.Append(" episodes\n"); bundle.Append(FFmpegWrapper.GetChromaprintLogs()); diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/VisualizationController.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/VisualizationController.cs index f2331ab..daef672 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/VisualizationController.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/VisualizationController.cs @@ -40,7 +40,7 @@ public ActionResult>> GetShowSeasons() var showSeasons = new Dictionary>(); // Loop through all seasons in the analysis queue - foreach (var kvp in Plugin.Instance!.AnalysisQueue) + foreach (var kvp in Plugin.Instance!.QueuedMediaItems) { // Check that this season contains at least one episode. var episodes = kvp.Value; @@ -104,10 +104,8 @@ public ActionResult> GetSeasonEpisodes( [HttpGet("Episode/{Id}/Chromaprint")] public ActionResult GetEpisodeFingerprint([FromRoute] Guid id) { - var queue = Plugin.Instance!.AnalysisQueue; - // Search through all queued episodes to find the requested id - foreach (var season in queue) + foreach (var season in Plugin.Instance!.QueuedMediaItems) { foreach (var needle in season.Value) { @@ -180,7 +178,7 @@ private string GetSeasonName(QueuedEpisode episode) /// Boolean indicating if the requested season was found. private bool LookupSeasonByName(string series, string season, out List episodes) { - foreach (var queuedEpisodes in Plugin.Instance!.AnalysisQueue) + foreach (var queuedEpisodes in Plugin.Instance!.QueuedMediaItems) { var first = queuedEpisodes.Value[0]; var firstSeasonName = GetSeasonName(first); diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Entrypoint.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Entrypoint.cs index 6a5dff2..a7a176a 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Entrypoint.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Entrypoint.cs @@ -57,7 +57,7 @@ public Task RunAsync() try { - // Enqueue all episodes at startup so the fingerprint visualizer works before the task is started. + // Enqueue all episodes at startup to ensure any FFmpeg errors appear as early as possible _logger.LogInformation("Running startup enqueue"); var queueManager = new QueueManager(_loggerFactory.CreateLogger(), _libraryManager); queueManager.EnqueueAllEpisodes(); @@ -67,11 +67,6 @@ public Task RunAsync() _logger.LogError("Unable to run startup enqueue: {Exception}", ex); } - _logger.LogDebug( - "Total enqueued seasons: {Count} ({Episodes} episodes)", - Plugin.Instance!.AnalysisQueue.Count, - Plugin.Instance!.TotalQueued); - return Task.CompletedTask; } diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Plugin.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Plugin.cs index bf5d241..84ea81c 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Plugin.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Plugin.cs @@ -92,9 +92,9 @@ public Plugin( public Dictionary Intros { get; } = new(); /// - /// Gets the mapping of season ids to episodes that have been queued for fingerprinting. + /// Gets the most recent media item queue. /// - public Dictionary> AnalysisQueue { get; } = new(); + public Dictionary> QueuedMediaItems { get; } = new(); /// /// Gets or sets the total number of episodes in the queue. diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/QueueManager.cs b/ConfusedPolarBear.Plugin.IntroSkipper/QueueManager.cs index be905e6..4f3308f 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/QueueManager.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/QueueManager.cs @@ -2,12 +2,12 @@ namespace ConfusedPolarBear.Plugin.IntroSkipper; using System; using System.Collections.Generic; +using System.Collections.ObjectModel; using System.Linq; using Jellyfin.Data.Enums; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Entities.TV; using MediaBrowser.Controller.Library; -using MediaBrowser.Model.Entities; using Microsoft.Extensions.Logging; /// @@ -19,7 +19,8 @@ public class QueueManager private ILogger _logger; private double analysisPercent; - private IList selectedLibraries; + private List selectedLibraries; + private Dictionary> _queuedEpisodes; /// /// Initializes a new instance of the class. @@ -31,13 +32,15 @@ public QueueManager(ILogger logger, ILibraryManager libraryManager _logger = logger; _libraryManager = libraryManager; - selectedLibraries = new List(); + selectedLibraries = new(); + _queuedEpisodes = new(); } /// /// Iterates through all libraries on the server and queues all episodes for analysis. /// - public void EnqueueAllEpisodes() + /// Queued media items. + public ReadOnlyDictionary> EnqueueAllEpisodes() { // Assert that ffmpeg with chromaprint is installed if (!FFmpegWrapper.CheckFFmpegVersion()) @@ -46,20 +49,13 @@ public void EnqueueAllEpisodes() "ffmpeg with chromaprint is not installed on this system - episodes will not be analyzed. If Jellyfin is running natively, install jellyfin-ffmpeg5. If Jellyfin is running in a container, upgrade it to the latest version of 10.8.0."); } - Plugin.Instance!.AnalysisQueue.Clear(); Plugin.Instance!.TotalQueued = 0; LoadAnalysisSettings(); - // For all selected TV show libraries, enqueue all contained items. + // For all selected libraries, enqueue all contained episodes. foreach (var folder in _libraryManager.GetVirtualFolders()) { - if (folder.CollectionType != CollectionTypeOptions.TvShows) - { - _logger.LogDebug("Not analyzing library \"{Name}\": not a TV show library", folder.Name); - continue; - } - // If libraries have been selected for analysis, ensure this library was selected. if (selectedLibraries.Count > 0 && !selectedLibraries.Contains(folder.Name)) { @@ -81,6 +77,14 @@ public void EnqueueAllEpisodes() _logger.LogError("Failed to enqueue items from library {Name}: {Exception}", folder.Name, ex); } } + + Plugin.Instance!.QueuedMediaItems.Clear(); + foreach (var kvp in _queuedEpisodes) + { + Plugin.Instance!.QueuedMediaItems[kvp.Key] = kvp.Value; + } + + return new(_queuedEpisodes); } /// @@ -156,7 +160,7 @@ private void QueueLibraryContents(string rawId) { if (item is not Episode episode) { - _logger.LogError("Item {Name} is not an episode", item.Name); + _logger.LogDebug("Item {Name} is not an episode", item.Name); continue; } @@ -198,11 +202,11 @@ private void QueueEpisode(Episode episode) 60 * Plugin.Instance!.Configuration.AnalysisLengthLimit); // Allocate a new list for each new season - Plugin.Instance!.AnalysisQueue.TryAdd(episode.SeasonId, new List()); + _queuedEpisodes.TryAdd(episode.SeasonId, new List()); // Queue the episode for analysis var maxCreditsDuration = Plugin.Instance!.Configuration.MaximumEpisodeCreditsDuration * 60; - Plugin.Instance.AnalysisQueue[episode.SeasonId].Add(new QueuedEpisode() + _queuedEpisodes[episode.SeasonId].Add(new QueuedEpisode() { SeriesName = episode.SeriesName, SeasonNumber = episode.AiredSeasonNumber ?? 0, diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs index d4da24e..92f9bbc 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs @@ -84,9 +84,7 @@ public Task ExecuteAsync(IProgress progress, CancellationToken cancellat _loggerFactory.CreateLogger(), _libraryManager); - queueManager.EnqueueAllEpisodes(); - - var queue = Plugin.Instance!.AnalysisQueue; + var queue = queueManager.EnqueueAllEpisodes(); if (queue.Count == 0) { @@ -142,7 +140,11 @@ public Task ExecuteAsync(IProgress progress, CancellationToken cancellat ex); } - progress.Report((totalProcessed * 100) / Plugin.Instance!.TotalQueued); + var total = Plugin.Instance!.TotalQueued; + if (total > 0) + { + progress.Report((totalProcessed * 100) / total); + } }); return Task.CompletedTask; diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectIntroductionsTask.cs b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectIntroductionsTask.cs index 416cd80..5b232b8 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectIntroductionsTask.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectIntroductionsTask.cs @@ -83,9 +83,7 @@ public Task ExecuteAsync(IProgress progress, CancellationToken cancellat _loggerFactory.CreateLogger(), _libraryManager); - queueManager.EnqueueAllEpisodes(); - - var queue = Plugin.Instance!.AnalysisQueue; + var queue = queueManager.EnqueueAllEpisodes(); if (queue.Count == 0) { @@ -162,7 +160,11 @@ public Task ExecuteAsync(IProgress progress, CancellationToken cancellat EdlManager.UpdateEDLFiles(episodes); } - progress.Report((totalProcessed * 100) / Plugin.Instance!.TotalQueued); + var total = Plugin.Instance!.TotalQueued; + if (total > 0) + { + progress.Report((totalProcessed * 100) / total); + } }); // Turn the regenerate EDL flag off after the scan completes. From a68914ca8f74ad82e62844054aa6dd00ac7eec7c Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Thu, 24 Nov 2022 00:43:23 -0600 Subject: [PATCH 10/22] Add chapter name based media file analyzer Closes #58 --- .../TestChapterAnalyzer.cs | 83 +++++++++++ .../Analyzers/ChapterAnalyzer.cs | 138 ++++++++++++++++++ .../Analyzers/ChromaprintAnalyzer.cs | 1 - .../Configuration/PluginConfiguration.cs | 16 +- .../Controllers/SkipIntroController.cs | 14 +- .../Plugin.cs | 119 ++++++++------- .../ScheduledTasks/DetectCreditsTask.cs | 3 - .../ScheduledTasks/DetectIntroductionsTask.cs | 4 + 8 files changed, 312 insertions(+), 66 deletions(-) create mode 100644 ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestChapterAnalyzer.cs create mode 100644 ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/ChapterAnalyzer.cs diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestChapterAnalyzer.cs b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestChapterAnalyzer.cs new file mode 100644 index 0000000..e1da0ab --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestChapterAnalyzer.cs @@ -0,0 +1,83 @@ +namespace ConfusedPolarBear.Plugin.IntroSkipper.Tests; + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using MediaBrowser.Model.Entities; +using Microsoft.Extensions.Logging; +using Xunit; + +public class TestChapterAnalyzer +{ + [Theory] + [InlineData("Opening")] + [InlineData("OP")] + [InlineData("Intro")] + [InlineData("Intro Start")] + [InlineData("Introduction")] + public void TestIntroductionExpression(string chapterName) + { + var chapters = CreateChapters(chapterName, AnalysisMode.Introduction); + var introChapter = FindChapter(chapters, AnalysisMode.Introduction); + + Assert.NotNull(introChapter); + Assert.Equal(60, introChapter.IntroStart); + Assert.Equal(90, introChapter.IntroEnd); + } + + [Theory] + [InlineData("End Credits")] + [InlineData("Ending")] + [InlineData("Credit start")] + [InlineData("Closing Credits")] + [InlineData("Credits")] + public void TestEndCreditsExpression(string chapterName) + { + var chapters = CreateChapters(chapterName, AnalysisMode.Credits); + var creditsChapter = FindChapter(chapters, AnalysisMode.Credits); + + Assert.NotNull(creditsChapter); + Assert.Equal(1890, creditsChapter.IntroStart); + Assert.Equal(2000, creditsChapter.IntroEnd); + } + + private Intro? FindChapter(Collection chapters, AnalysisMode mode) + { + var logger = new LoggerFactory().CreateLogger(); + var analyzer = new ChapterAnalyzer(logger); + + var config = new Configuration.PluginConfiguration(); + var expression = mode == AnalysisMode.Introduction ? + config.ChapterAnalyzerIntroductionPattern : + config.ChapterAnalyzerEndCreditsPattern; + + return analyzer.FindMatchingChapter(Guid.Empty, 2000, chapters, expression, mode); + } + + private Collection CreateChapters(string name, AnalysisMode mode) + { + var chapters = new[]{ + CreateChapter("Cold Open", 0), + CreateChapter(mode == AnalysisMode.Introduction ? name : "Introduction", 60), + CreateChapter("Main Episode", 90), + CreateChapter(mode == AnalysisMode.Credits ? name : "Credits", 1890) + }; + + return new(new List(chapters)); + } + + /// + /// Create a ChapterInfo object. + /// + /// Chapter name. + /// Chapter position (in seconds). + /// ChapterInfo. + private ChapterInfo CreateChapter(string name, int position) + { + return new() + { + Name = name, + StartPositionTicks = TimeSpan.FromSeconds(position).Ticks + }; + } +} diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/ChapterAnalyzer.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/ChapterAnalyzer.cs new file mode 100644 index 0000000..86efd53 --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/ChapterAnalyzer.cs @@ -0,0 +1,138 @@ +namespace ConfusedPolarBear.Plugin.IntroSkipper; + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Text.RegularExpressions; +using System.Threading; +using Microsoft.Extensions.Logging; +using MediaBrowser.Model.Entities; + +/// +/// Chapter name analyzer. +/// +public class ChapterAnalyzer : IMediaFileAnalyzer +{ + private ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + /// Logger. + public ChapterAnalyzer(ILogger logger) + { + _logger = logger; + } + + /// + public ReadOnlyCollection AnalyzeMediaFiles( + ReadOnlyCollection analysisQueue, + AnalysisMode mode, + CancellationToken cancellationToken) + { + var skippableRanges = new Dictionary(); + var expression = mode == AnalysisMode.Introduction ? + Plugin.Instance!.Configuration.ChapterAnalyzerIntroductionPattern : + Plugin.Instance!.Configuration.ChapterAnalyzerEndCreditsPattern; + + foreach (var episode in analysisQueue) + { + if (cancellationToken.IsCancellationRequested) + { + break; + } + + var skipRange = FindMatchingChapter( + episode.EpisodeId, + episode.Duration, + new(Plugin.Instance!.GetChapters(episode.EpisodeId)), + expression, + mode); + + if (skipRange is null) + { + continue; + } + + skippableRanges.Add(episode.EpisodeId, skipRange); + } + + Plugin.Instance!.UpdateTimestamps(skippableRanges, mode); + + return analysisQueue; + } + + /// + /// Searches a list of chapter names for one that matches the provided regular expression. + /// Only public to allow for unit testing. + /// + /// Item id. + /// Duration of media file in seconds. + /// Media item chapters. + /// Regular expression pattern. + /// Analysis mode. + /// Intro object containing skippable time range, or null if no chapter matched. + public Intro? FindMatchingChapter( + Guid id, + int duration, + Collection chapters, + string expression, + AnalysisMode mode) + { + Intro? matchingChapter = null; + + var config = Plugin.Instance?.Configuration ?? new Configuration.PluginConfiguration(); + var minDuration = config.MinimumIntroDuration; + int maxDuration = mode == AnalysisMode.Introduction ? + config.MaximumIntroDuration : + config.MaximumEpisodeCreditsDuration; + + if (mode == AnalysisMode.Credits) + { + // Since the ending credits chapter may be the last chapter in the file, append a virtual + // chapter at the very end of the file. + chapters.Add(new ChapterInfo() + { + StartPositionTicks = TimeSpan.FromSeconds(duration).Ticks + }); + } + + // Check all chapters + for (int i = 0; i < chapters.Count - 1; i++) + { + // Calculate chapter position and duration + var current = chapters[i]; + var next = chapters[i + 1]; + + var currentRange = new TimeRange( + TimeSpan.FromTicks(current.StartPositionTicks).TotalSeconds, + TimeSpan.FromTicks(next.StartPositionTicks).TotalSeconds); + + // Skip chapters with that don't have a name or are too short/long + if (string.IsNullOrEmpty(current.Name) || + currentRange.Duration < minDuration || + currentRange.Duration > maxDuration) + { + continue; + } + + // Regex.IsMatch() is used here in order to allow the runtime to cache the compiled regex + // between function invocations. + var match = Regex.IsMatch( + current.Name, + expression, + RegexOptions.None, + TimeSpan.FromSeconds(1)); + + if (!match) + { + continue; + } + + matchingChapter = new Intro(id, currentRange); + break; + } + + return matchingChapter; + } +} diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/ChromaprintAnalyzer.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/ChromaprintAnalyzer.cs index 04f9da0..f226835 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/ChromaprintAnalyzer.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/ChromaprintAnalyzer.cs @@ -82,7 +82,6 @@ public ReadOnlyCollection AnalyzeMediaFiles( } catch (FingerprintException ex) { - // TODO: FIXME: move to debug level? _logger.LogWarning("Caught fingerprint error: {Ex}", ex); // Fallback to an empty fingerprint on any error diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/PluginConfiguration.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/PluginConfiguration.cs index f4789db..463d5b6 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/PluginConfiguration.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/PluginConfiguration.cs @@ -73,15 +73,27 @@ public PluginConfiguration() public int MaximumIntroDuration { get; set; } = 120; /// - /// Gets or sets the upper limit (in minutes) on the length of each episode's audio track that will be analyzed when searching for ending credits. + /// Gets or sets the upper limit (in seconds) on the length of each episode's audio track that will be analyzed when searching for ending credits. /// - public int MaximumEpisodeCreditsDuration { get; set; } = 4; + public int MaximumEpisodeCreditsDuration { get; set; } = 240; /// /// Gets or sets the minimum percentage of a frame that must consist of black pixels before it is considered a black frame. /// public int BlackFrameMinimumPercentage { get; set; } = 85; + /// + /// Gets or sets the regular expression used to detect introduction chapters. + /// + public string ChapterAnalyzerIntroductionPattern { get; set; } = + @"(^|\s)(Intro|Introduction|OP|Opening)(\s|$)"; + + /// + /// Gets or sets the regular expression used to detect ending credit chapters. + /// + public string ChapterAnalyzerEndCreditsPattern { get; set; } = + @"(^|\s)(Credits?|Ending)(\s|$)"; + // ===== Playback settings ===== /// diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/SkipIntroController.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/SkipIntroController.cs index d6c97f6..b4631b7 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/SkipIntroController.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/SkipIntroController.cs @@ -73,18 +73,24 @@ public ActionResult ResetIntroTimestamps() } /// - /// Get all introductions. Only used by the end to end testing script. + /// Get all introductions or credits. Only used by the end to end testing script. /// - /// All introductions have been returned. + /// Mode. + /// All timestamps have been returned. /// List of IntroWithMetadata objects. [Authorize(Policy = "RequiresElevation")] [HttpGet("Intros/All")] - public ActionResult> GetAllIntros() + public ActionResult> GetAllTimestamps( + [FromQuery] AnalysisMode mode = AnalysisMode.Introduction) { List intros = new(); + var timestamps = mode == AnalysisMode.Introduction ? + Plugin.Instance!.Intros : + Plugin.Instance!.Credits; + // Get metadata for all intros - foreach (var intro in Plugin.Instance!.Intros) + foreach (var intro in timestamps) { // Get the details of the item from Jellyfin var rawItem = Plugin.Instance!.GetItem(intro.Key); diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Plugin.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Plugin.cs index 84ea81c..be89666 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Plugin.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Plugin.cs @@ -8,6 +8,8 @@ using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Entities.TV; using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.Persistence; +using MediaBrowser.Model.Entities; using MediaBrowser.Model.Plugins; using MediaBrowser.Model.Serialization; using Microsoft.Extensions.Logging; @@ -23,9 +25,10 @@ public class Plugin : BasePlugin, IHasWebPages private readonly object _introsLock = new(); private IXmlSerializer _xmlSerializer; private ILibraryManager _libraryManager; + private IItemRepository _itemRepository; private ILogger _logger; private string _introPath; - private string _creditsPath; // TODO: FIXME: remove this + private string _creditsPath; /// /// Initializes a new instance of the class. @@ -34,12 +37,14 @@ public class Plugin : BasePlugin, IHasWebPages /// Instance of the interface. /// Server configuration manager. /// Library manager. + /// Item repository. /// Logger. public Plugin( IApplicationPaths applicationPaths, IXmlSerializer xmlSerializer, IServerConfigurationManager serverConfiguration, ILibraryManager libraryManager, + IItemRepository itemRepository, ILogger logger) : base(applicationPaths, xmlSerializer) { @@ -47,14 +52,13 @@ public Plugin( _xmlSerializer = xmlSerializer; _libraryManager = libraryManager; + _itemRepository = itemRepository; _logger = logger; FingerprintCachePath = Path.Join(applicationPaths.PluginConfigurationsPath, "intros", "cache"); FFmpegPath = serverConfiguration.GetEncodingOptions().EncoderAppPathDisplay; _introPath = Path.Join(applicationPaths.PluginConfigurationsPath, "intros", "intros.xml"); - - // TODO: FIXME: remove this - _creditsPath = Path.Join(applicationPaths.PluginConfigurationsPath, "intros", "credits.csv"); + _creditsPath = Path.Join(applicationPaths.PluginConfigurationsPath, "intros", "credits.xml"); // Create the base & cache directories (if needed). if (!Directory.Exists(FingerprintCachePath)) @@ -73,12 +77,6 @@ public Plugin( { _logger.LogWarning("Unable to load introduction timestamps: {Exception}", ex); } - - // TODO: FIXME: remove this - if (File.Exists(_creditsPath)) - { - File.Delete(_creditsPath); - } } /// @@ -91,6 +89,11 @@ public Plugin( /// public Dictionary Intros { get; } = new(); + /// + /// Gets all discovered ending credits. + /// + public Dictionary Credits { get; } = new(); + /// /// Gets the most recent media item queue. /// @@ -131,12 +134,23 @@ public void SaveTimestamps() { var introList = new List(); + // Serialize intros foreach (var intro in Plugin.Instance!.Intros) { introList.Add(intro.Value); } _xmlSerializer.SerializeToFile(introList, _introPath); + + // Serialize credits + introList.Clear(); + + foreach (var intro in Plugin.Instance!.Credits) + { + introList.Add(intro.Value); + } + + _xmlSerializer.SerializeToFile(introList, _creditsPath); } } @@ -145,17 +159,29 @@ public void SaveTimestamps() /// public void RestoreTimestamps() { - if (!File.Exists(_introPath)) + if (File.Exists(_introPath)) { - return; - } + // Since dictionaries can't be easily serialized, analysis results are stored on disk as a list. + var introList = (List)_xmlSerializer.DeserializeFromFile( + typeof(List), + _introPath); - // Since dictionaries can't be easily serialized, analysis results are stored on disk as a list. - var introList = (List)_xmlSerializer.DeserializeFromFile(typeof(List), _introPath); + foreach (var intro in introList) + { + Plugin.Instance!.Intros[intro.EpisodeId] = intro; + } + } - foreach (var intro in introList) + if (File.Exists(_creditsPath)) { - Plugin.Instance!.Intros[intro.EpisodeId] = intro; + var creditList = (List)_xmlSerializer.DeserializeFromFile( + typeof(List), + _creditsPath); + + foreach (var credit in creditList) + { + Plugin.Instance!.Credits[credit.EpisodeId] = credit; + } } } @@ -174,52 +200,33 @@ internal string GetItemPath(Guid id) return GetItem(id).Path; } - internal void UpdateTimestamps(Dictionary newIntros, AnalysisMode mode) + /// + /// Gets all chapters for this item. + /// + /// Item id. + /// List of chapters. + internal List GetChapters(Guid id) { - switch (mode) + return _itemRepository.GetChapters(GetItem(id)); + } + + internal void UpdateTimestamps(Dictionary newTimestamps, AnalysisMode mode) + { + lock (_introsLock) { - case AnalysisMode.Introduction: - lock (_introsLock) + foreach (var intro in newTimestamps) + { + if (mode == AnalysisMode.Introduction) { - foreach (var intro in newIntros) - { - Plugin.Instance!.Intros[intro.Key] = intro.Value; - } - - Plugin.Instance!.SaveTimestamps(); + Plugin.Instance!.Intros[intro.Key] = intro.Value; } - - break; - - case AnalysisMode.Credits: - // TODO: FIXME: implement properly - - lock (_introsLock) + else if (mode == AnalysisMode.Credits) { - foreach (var credit in newIntros) - { - var item = GetItem(credit.Value.EpisodeId) as Episode; - if (item is null) - { - continue; - } - - // Format: series, season number, episode number, title, start, end - var contents = string.Format( - System.Globalization.CultureInfo.InvariantCulture, - "{0},{1},{2},{3},{4},{5}\n", - item.SeriesName.Replace(",", string.Empty, StringComparison.Ordinal), - item.AiredSeasonNumber ?? 0, - item.IndexNumber ?? 0, - item.Name.Replace(",", string.Empty, StringComparison.Ordinal), - Math.Round(credit.Value.IntroStart, 2), - Math.Round(credit.Value.IntroEnd, 2)); - - File.AppendAllText(_creditsPath, contents); - } + Plugin.Instance!.Credits[intro.Key] = intro.Value; } + } - break; + Plugin.Instance!.SaveTimestamps(); } } diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs index 92f9bbc..4a9fd9e 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs @@ -98,9 +98,6 @@ public Task ExecuteAsync(IProgress progress, CancellationToken cancellat MaxDegreeOfParallelism = Plugin.Instance!.Configuration.MaxParallelism }; - // TODO: FIXME: if the queue is modified while the task is running, the task will fail. - // clone the queue before running the task to prevent this. - // Analyze all episodes in the queue using the degrees of parallelism the user specified. Parallel.ForEach(queue, options, (season) => { diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectIntroductionsTask.cs b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectIntroductionsTask.cs index 5b232b8..240feb7 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectIntroductionsTask.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectIntroductionsTask.cs @@ -252,6 +252,10 @@ private int AnalyzeSeason( first.SeriesName, first.SeasonNumber); + // Chapter analyzer + var chapter = new ChapterAnalyzer(_loggerFactory.CreateLogger()); + chapter.AnalyzeMediaFiles(episodes, AnalysisMode.Introduction, cancellationToken); + // Analyze the season with Chromaprint var chromaprint = new ChromaprintAnalyzer(_loggerFactory.CreateLogger()); chromaprint.AnalyzeMediaFiles(episodes, AnalysisMode.Introduction, cancellationToken); From bfb821f31119edd525312de18e57eb61df686533 Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Thu, 24 Nov 2022 00:44:33 -0600 Subject: [PATCH 11/22] Update CHANGELOG.md --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd38246..821a8f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,11 @@ ## v0.1.8.0 (no eta) * New features + * Support adding skip intro button to web interface without using a fork + * Add localization support for the skip intro button and the automatic skip notification message * Detect ending credits in television episodes + * Add support for using chapter names to locate introductions and ending credits + * Add support for using black frames to locate ending credits * Internal changes * Move Chromaprint analysis code out of the episode analysis task * Add support for multiple analysis techinques From 7439720b3a94d72fae998f152d53cd88937183d7 Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Fri, 25 Nov 2022 00:37:30 -0600 Subject: [PATCH 12/22] Add chaining and logs to chapter analyzer --- .../TestChapterAnalyzer.cs | 2 +- .../Analyzers/ChapterAnalyzer.cs | 44 ++++++++++++------- .../ScheduledTasks/DetectIntroductionsTask.cs | 2 +- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestChapterAnalyzer.cs b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestChapterAnalyzer.cs index e1da0ab..96feb8e 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestChapterAnalyzer.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestChapterAnalyzer.cs @@ -51,7 +51,7 @@ public void TestEndCreditsExpression(string chapterName) config.ChapterAnalyzerIntroductionPattern : config.ChapterAnalyzerEndCreditsPattern; - return analyzer.FindMatchingChapter(Guid.Empty, 2000, chapters, expression, mode); + return analyzer.FindMatchingChapter(new() { Duration = 2000 }, chapters, expression, mode); } private Collection CreateChapters(string name, AnalysisMode mode) diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/ChapterAnalyzer.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/ChapterAnalyzer.cs index 86efd53..73ec9aa 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/ChapterAnalyzer.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/ChapterAnalyzer.cs @@ -3,6 +3,7 @@ namespace ConfusedPolarBear.Plugin.IntroSkipper; using System; using System.Collections.Generic; using System.Collections.ObjectModel; +using System.Globalization; using System.Text.RegularExpressions; using System.Threading; using Microsoft.Extensions.Logging; @@ -30,7 +31,9 @@ public ReadOnlyCollection AnalyzeMediaFiles( AnalysisMode mode, CancellationToken cancellationToken) { + var unsuccessful = new List(); var skippableRanges = new Dictionary(); + var expression = mode == AnalysisMode.Introduction ? Plugin.Instance!.Configuration.ChapterAnalyzerIntroductionPattern : Plugin.Instance!.Configuration.ChapterAnalyzerEndCreditsPattern; @@ -43,14 +46,14 @@ public ReadOnlyCollection AnalyzeMediaFiles( } var skipRange = FindMatchingChapter( - episode.EpisodeId, - episode.Duration, + episode, new(Plugin.Instance!.GetChapters(episode.EpisodeId)), expression, mode); if (skipRange is null) { + unsuccessful.Add(episode); continue; } @@ -59,22 +62,20 @@ public ReadOnlyCollection AnalyzeMediaFiles( Plugin.Instance!.UpdateTimestamps(skippableRanges, mode); - return analysisQueue; + return unsuccessful.AsReadOnly(); } /// /// Searches a list of chapter names for one that matches the provided regular expression. /// Only public to allow for unit testing. /// - /// Item id. - /// Duration of media file in seconds. + /// Episode. /// Media item chapters. /// Regular expression pattern. /// Analysis mode. /// Intro object containing skippable time range, or null if no chapter matched. public Intro? FindMatchingChapter( - Guid id, - int duration, + QueuedEpisode episode, Collection chapters, string expression, AnalysisMode mode) @@ -82,6 +83,7 @@ public ReadOnlyCollection AnalyzeMediaFiles( Intro? matchingChapter = null; var config = Plugin.Instance?.Configuration ?? new Configuration.PluginConfiguration(); + var minDuration = config.MinimumIntroDuration; int maxDuration = mode == AnalysisMode.Introduction ? config.MaximumIntroDuration : @@ -91,28 +93,38 @@ public ReadOnlyCollection AnalyzeMediaFiles( { // Since the ending credits chapter may be the last chapter in the file, append a virtual // chapter at the very end of the file. - chapters.Add(new ChapterInfo() + chapters.Add(new() { - StartPositionTicks = TimeSpan.FromSeconds(duration).Ticks + StartPositionTicks = TimeSpan.FromSeconds(episode.Duration).Ticks }); } // Check all chapters for (int i = 0; i < chapters.Count - 1; i++) { - // Calculate chapter position and duration var current = chapters[i]; var next = chapters[i + 1]; + if (string.IsNullOrWhiteSpace(current.Name)) + { + continue; + } + var currentRange = new TimeRange( TimeSpan.FromTicks(current.StartPositionTicks).TotalSeconds, TimeSpan.FromTicks(next.StartPositionTicks).TotalSeconds); - // Skip chapters with that don't have a name or are too short/long - if (string.IsNullOrEmpty(current.Name) || - currentRange.Duration < minDuration || - currentRange.Duration > maxDuration) + var baseMessage = string.Format( + CultureInfo.InvariantCulture, + "{0}: Chapter \"{1}\" ({2} - {3})", + episode.Path, + current.Name, + currentRange.Start, + currentRange.End); + + if (currentRange.Duration < minDuration || currentRange.Duration > maxDuration) { + _logger.LogTrace("{Base}: ignoring (invalid duration)", baseMessage); continue; } @@ -126,10 +138,12 @@ public ReadOnlyCollection AnalyzeMediaFiles( if (!match) { + _logger.LogTrace("{Base}: ignoring (does not match regular expression)", baseMessage); continue; } - matchingChapter = new Intro(id, currentRange); + matchingChapter = new(episode.EpisodeId, currentRange); + _logger.LogTrace("{Base}: okay", baseMessage); break; } diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectIntroductionsTask.cs b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectIntroductionsTask.cs index 240feb7..83df267 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectIntroductionsTask.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectIntroductionsTask.cs @@ -254,7 +254,7 @@ private int AnalyzeSeason( // Chapter analyzer var chapter = new ChapterAnalyzer(_loggerFactory.CreateLogger()); - chapter.AnalyzeMediaFiles(episodes, AnalysisMode.Introduction, cancellationToken); + episodes = chapter.AnalyzeMediaFiles(episodes, AnalysisMode.Introduction, cancellationToken); // Analyze the season with Chromaprint var chromaprint = new ChromaprintAnalyzer(_loggerFactory.CreateLogger()); From ae16603aeedddf1d586660ce0540d1fe753b7aef Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Fri, 25 Nov 2022 00:40:02 -0600 Subject: [PATCH 13/22] Support returning credit timestamps --- .../Controllers/SkipIntroController.cs | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/SkipIntroController.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/SkipIntroController.cs index b4631b7..85b9804 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/SkipIntroController.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/SkipIntroController.cs @@ -26,14 +26,17 @@ public SkipIntroController() /// Returns the timestamps of the introduction in a television episode. Responses are in API version 1 format. /// /// ID of the episode. Required. + /// Timestamps to return. Optional. Defaults to Introduction for backwards compatibility. /// Episode contains an intro. /// Failed to find an intro in the provided episode. /// Detected intro. [HttpGet("Episode/{id}/IntroTimestamps")] [HttpGet("Episode/{id}/IntroTimestamps/v1")] - public ActionResult GetIntroTimestamps([FromRoute] Guid id) + public ActionResult GetIntroTimestamps( + [FromRoute] Guid id, + [FromQuery] AnalysisMode mode = AnalysisMode.Introduction) { - var intro = GetIntro(id); + var intro = GetIntro(id, mode); if (intro is null || !intro.Valid) { @@ -49,13 +52,25 @@ public ActionResult GetIntroTimestamps([FromRoute] Guid id) return intro; } - /// Lookup and return the intro timestamps for the provided item. + /// Lookup and return the skippable timestamps for the provided item. /// Unique identifier of this episode. + /// Mode. /// Intro object if the provided item has an intro, null otherwise. - private Intro? GetIntro(Guid id) + private Intro? GetIntro(Guid id, AnalysisMode mode) { - // Returns a copy to avoid mutating the original Intro object stored in the dictionary. - return Plugin.Instance!.Intros.TryGetValue(id, out var intro) ? new Intro(intro) : null; + try + { + var timestamp = mode == AnalysisMode.Introduction ? + Plugin.Instance!.Intros[id] : + Plugin.Instance!.Credits[id]; + + // A copy is returned to avoid mutating the original Intro object stored in the dictionary. + return new(timestamp); + } + catch (KeyNotFoundException) + { + return null; + } } /// From 3e2dc377b7f791dd4e90b476d0f836670539aad6 Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Fri, 25 Nov 2022 00:41:48 -0600 Subject: [PATCH 14/22] Add new task ID --- .../e2e_tests/verifier/report_generator.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/report_generator.go b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/report_generator.go index 708f864..2c4fb3b 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/report_generator.go +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/e2e_tests/verifier/report_generator.go @@ -113,11 +113,13 @@ func runAnalysisAndWait(hostAddress, apiKey string, pollInterval time.Duration) SendRequest("POST", hostAddress+"/Intros/EraseTimestamps", apiKey) fmt.Println() - // The task ID changed with v0.1.7. - // Old task ID: 8863329048cc357f7dfebf080f2fe204 - // New task ID: 6adda26c5261c40e8fa4a7e7df568be2 + var taskIds = []string{ + "f64d8ad58e3d7b98548e1a07697eb100", // v0.1.8 + "8863329048cc357f7dfebf080f2fe204", + "6adda26c5261c40e8fa4a7e7df568be2"} + fmt.Println("[+] Starting analysis task") - for _, id := range []string{"8863329048cc357f7dfebf080f2fe204", "6adda26c5261c40e8fa4a7e7df568be2"} { + for _, id := range taskIds { body := SendRequest("POST", hostAddress+"/ScheduledTasks/Running/"+id, apiKey) fmt.Println() From cb234bedd3f137c82aa6c3ea0c225f04b502b3d6 Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Sat, 26 Nov 2022 02:28:40 -0600 Subject: [PATCH 15/22] Update black frame and chapter analyzer --- .../Analyzers/BlackFrameAnalyzer.cs | 12 +++++++++--- .../Analyzers/ChapterAnalyzer.cs | 8 +++++--- .../Data/QueuedEpisode.cs | 4 ++-- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/BlackFrameAnalyzer.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/BlackFrameAnalyzer.cs index 1c28c5f..ba2ee30 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/BlackFrameAnalyzer.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/BlackFrameAnalyzer.cs @@ -76,8 +76,10 @@ public ReadOnlyCollection AnalyzeMediaFiles( /// Credits timestamp. public Intro? AnalyzeMediaFile(QueuedEpisode episode, AnalysisMode mode, int minimum) { - // Start by analyzing the last four minutes of the file. - var start = TimeSpan.FromMinutes(4); + var config = Plugin.Instance?.Configuration ?? new Configuration.PluginConfiguration(); + + // Start by analyzing the last N minutes of the file. + var start = TimeSpan.FromSeconds(config.MaximumEpisodeCreditsDuration); var end = TimeSpan.Zero; var firstFrameTime = 0.0; @@ -100,7 +102,11 @@ public ReadOnlyCollection AnalyzeMediaFiles( tr.End); var frames = FFmpegWrapper.DetectBlackFrames(episode, tr, minimum); - _logger.LogTrace("{Episode}, black frames: {Count}", episode.Name, frames.Length); + _logger.LogTrace( + "{Episode} at {Start} has {Count} black frames", + episode.Name, + tr.Start, + frames.Length); if (frames.Length == 0) { diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/ChapterAnalyzer.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/ChapterAnalyzer.cs index 73ec9aa..5accbb4 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/ChapterAnalyzer.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/ChapterAnalyzer.cs @@ -4,6 +4,7 @@ namespace ConfusedPolarBear.Plugin.IntroSkipper; using System.Collections.Generic; using System.Collections.ObjectModel; using System.Globalization; +using System.Linq; using System.Text.RegularExpressions; using System.Threading; using Microsoft.Extensions.Logging; @@ -31,7 +32,6 @@ public ReadOnlyCollection AnalyzeMediaFiles( AnalysisMode mode, CancellationToken cancellationToken) { - var unsuccessful = new List(); var skippableRanges = new Dictionary(); var expression = mode == AnalysisMode.Introduction ? @@ -53,7 +53,6 @@ public ReadOnlyCollection AnalyzeMediaFiles( if (skipRange is null) { - unsuccessful.Add(episode); continue; } @@ -62,7 +61,10 @@ public ReadOnlyCollection AnalyzeMediaFiles( Plugin.Instance!.UpdateTimestamps(skippableRanges, mode); - return unsuccessful.AsReadOnly(); + return analysisQueue + .Where(x => !skippableRanges.ContainsKey(x.EpisodeId)) + .ToList() + .AsReadOnly(); } /// diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Data/QueuedEpisode.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Data/QueuedEpisode.cs index 0964258..bb65789 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Data/QueuedEpisode.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Data/QueuedEpisode.cs @@ -33,12 +33,12 @@ public class QueuedEpisode public string Name { get; set; } = string.Empty; /// - /// Gets or sets the timestamp (in seconds) to stop searching for an introduction. + /// Gets or sets the timestamp (in seconds) to stop searching for an introduction at. /// public int IntroFingerprintEnd { get; set; } /// - /// Gets or sets the timestamp (in seconds) to start looking for end credits. + /// Gets or sets the timestamp (in seconds) to start looking for end credits at. /// public int CreditsFingerprintStart { get; set; } From 94bd2a059e3ba2e908abb98c67165e528bebabd7 Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Tue, 29 Nov 2022 02:29:27 -0600 Subject: [PATCH 16/22] Use chromaprint analyzer when detecting credits --- ConfusedPolarBear.Plugin.IntroSkipper/QueueManager.cs | 2 +- .../ScheduledTasks/DetectCreditsTask.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/QueueManager.cs b/ConfusedPolarBear.Plugin.IntroSkipper/QueueManager.cs index 4f3308f..dfc5639 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/QueueManager.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/QueueManager.cs @@ -205,7 +205,7 @@ private void QueueEpisode(Episode episode) _queuedEpisodes.TryAdd(episode.SeasonId, new List()); // Queue the episode for analysis - var maxCreditsDuration = Plugin.Instance!.Configuration.MaximumEpisodeCreditsDuration * 60; + var maxCreditsDuration = Plugin.Instance!.Configuration.MaximumEpisodeCreditsDuration; _queuedEpisodes[episode.SeasonId].Add(new QueuedEpisode() { SeriesName = episode.SeriesName, diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs index 4a9fd9e..74ef0c0 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs @@ -165,7 +165,7 @@ private void AnalyzeSeason( // Analyze with Chromaprint first and fall back to the black frame detector var analyzers = new IMediaFileAnalyzer[] { - // TODO: FIXME: new ChromaprintAnalyzer(_loggerFactory.CreateLogger()), + new ChromaprintAnalyzer(_loggerFactory.CreateLogger()), new BlackFrameAnalyzer(_loggerFactory.CreateLogger()) }; From 16251f0735c37e596debc8ff1b3eb4f6d5dbc889 Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Tue, 29 Nov 2022 02:31:24 -0600 Subject: [PATCH 17/22] Support erasing times for intros and end credits --- .../Configuration/configPage.html | 61 +++++++++++++------ .../Controllers/SkipIntroController.cs | 13 +++- 2 files changed, 53 insertions(+), 21 deletions(-) diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/configPage.html b/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/configPage.html index 401bd92..ad4fc31 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/configPage.html +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/configPage.html @@ -201,7 +201,8 @@
- If checked, intros will be automatically skipped. If you access Jellyfin through a reverse proxy, it must be configured to proxy web + If checked, intros will be automatically skipped. If you access Jellyfin through a + reverse proxy, it must be configured to proxy web sockets.
@@ -253,10 +254,6 @@ Save
- - @@ -302,7 +299,17 @@

Introduction timestamp editor

+
+ + +
+ +

@@ -376,7 +383,8 @@

Fingerprint Visualizer

// settings elements var visualizer = document.querySelector("details#visualizer"); var support = document.querySelector("details#support"); - var btnEraseTimestamps = document.querySelector("button#btnEraseTimestamps"); + var btnEraseIntroTimestamps = document.querySelector("button#btnEraseIntroTimestamps"); + var btnEraseCreditTimestamps = document.querySelector("button#btnEraseCreditTimestamps"); // all plugin configuration fields that can be get or set with .value (i.e. strings or numbers). var configurationFields = [ @@ -662,6 +670,28 @@

Fingerprint Visualizer

return new Date(seconds * 1000).toISOString().substr(14, 5); } + // erase all intro/credits timestamps + function eraseTimestamps(mode) { + const lower = mode.toLocaleLowerCase(); + const title = "Confirm timestamp erasure"; + const body = "Are you sure you want to erase all previously discovered " + + mode.toLocaleLowerCase() + + " timestamps?"; + + Dashboard.confirm( + body, + title, + (result) => { + if (!result) { + return; + } + + fetchWithAuth("Intros/EraseTimestamps?mode=" + mode, "POST", null); + + Dashboard.alert(mode + " timestamps erased"); + }); + } + document.querySelector('#TemplateConfigPage') .addEventListener('pageshow', function () { Dashboard.showLoadingMsg(); @@ -707,19 +737,12 @@

Fingerprint Visualizer

selectSeason.addEventListener("change", seasonChanged); selectEpisode1.addEventListener("change", episodeChanged); selectEpisode2.addEventListener("change", episodeChanged); - btnEraseTimestamps.addEventListener("click", (e) => { - Dashboard.confirm( - "Are you sure you want to erase all previously discovered introduction timestamps?", - "Confirm timestamp erasure", - (result) => { - if (!result) { - return; - } - - // reset all intro timestamps on the server so a new fingerprint comparison algorithm can be tested - fetchWithAuth("Intros/EraseTimestamps", "POST", null); - }); - + btnEraseIntroTimestamps.addEventListener("click", (e) => { + eraseTimestamps("Introduction"); + e.preventDefault(); + }); + btnEraseCreditTimestamps.addEventListener("click", (e) => { + eraseTimestamps("Credits"); e.preventDefault(); }); btnSeasonEraseTimestamps.addEventListener("click", () => { diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/SkipIntroController.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/SkipIntroController.cs index 85b9804..3f12bb9 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/SkipIntroController.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Controllers/SkipIntroController.cs @@ -76,13 +76,22 @@ public ActionResult GetIntroTimestamps( /// /// Erases all previously discovered introduction timestamps. /// + /// Mode. /// Operation successful. /// No content. [Authorize(Policy = "RequiresElevation")] [HttpPost("Intros/EraseTimestamps")] - public ActionResult ResetIntroTimestamps() + public ActionResult ResetIntroTimestamps([FromQuery] AnalysisMode mode) { - Plugin.Instance!.Intros.Clear(); + if (mode == AnalysisMode.Introduction) + { + Plugin.Instance!.Intros.Clear(); + } + else if (mode == AnalysisMode.Credits) + { + Plugin.Instance!.Credits.Clear(); + } + Plugin.Instance!.SaveTimestamps(); return NoContent(); } From 0afc4cab41d2e6737bb1ddb83643417753f1d2f2 Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Tue, 29 Nov 2022 20:11:22 -0600 Subject: [PATCH 18/22] Add minimum end credit duration --- .../TestBlackFrames.cs | 4 +++- .../Analyzers/BlackFrameAnalyzer.cs | 2 +- .../Configuration/PluginConfiguration.cs | 5 +++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestBlackFrames.cs b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestBlackFrames.cs index bdd34e5..16caf27 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestBlackFrames.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestBlackFrames.cs @@ -30,6 +30,8 @@ public void TestBlackFrameDetection() [FactSkipFFmpegTests] public void TestEndCreditDetection() { + var range = 1; + var analyzer = CreateBlackFrameAnalyzer(); var episode = queueFile("credits.mp4"); @@ -37,7 +39,7 @@ public void TestEndCreditDetection() var result = analyzer.AnalyzeMediaFile(episode, AnalysisMode.Credits, 85); Assert.NotNull(result); - Assert.Equal(300, result.IntroStart); + Assert.InRange(result.IntroStart, 300 - range, 300 + range); } private QueuedEpisode queueFile(string path) diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/BlackFrameAnalyzer.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/BlackFrameAnalyzer.cs index ba2ee30..89a4018 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/BlackFrameAnalyzer.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/BlackFrameAnalyzer.cs @@ -80,7 +80,7 @@ public ReadOnlyCollection AnalyzeMediaFiles( // Start by analyzing the last N minutes of the file. var start = TimeSpan.FromSeconds(config.MaximumEpisodeCreditsDuration); - var end = TimeSpan.Zero; + var end = TimeSpan.FromSeconds(config.MinimumCreditsDuration); var firstFrameTime = 0.0; // Continue bisecting the end of the file until the range that contains the first black diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/PluginConfiguration.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/PluginConfiguration.cs index 463d5b6..d6b38bb 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/PluginConfiguration.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Configuration/PluginConfiguration.cs @@ -72,6 +72,11 @@ public PluginConfiguration() ///
public int MaximumIntroDuration { get; set; } = 120; + /// + /// Gets or sets the minimum length of similar audio that will be considered ending credits. + /// + public int MinimumCreditsDuration { get; set; } = 15; + /// /// Gets or sets the upper limit (in seconds) on the length of each episode's audio track that will be analyzed when searching for ending credits. /// From 4bb978639c26b15235bdfe3f11a54b24e1293c76 Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Tue, 29 Nov 2022 20:31:41 -0600 Subject: [PATCH 19/22] Don't run chapter analyzer with an empty regex --- .../Analyzers/ChapterAnalyzer.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/ChapterAnalyzer.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/ChapterAnalyzer.cs index 5accbb4..558c0e9 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/ChapterAnalyzer.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/ChapterAnalyzer.cs @@ -38,6 +38,11 @@ public ReadOnlyCollection AnalyzeMediaFiles( Plugin.Instance!.Configuration.ChapterAnalyzerIntroductionPattern : Plugin.Instance!.Configuration.ChapterAnalyzerEndCreditsPattern; + if (string.IsNullOrWhiteSpace(expression)) + { + return analysisQueue; + } + foreach (var episode in analysisQueue) { if (cancellationToken.IsCancellationRequested) From 1966357e29f8f7102b92a86faab7dee41b0c9953 Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Mon, 5 Dec 2022 22:35:01 -0600 Subject: [PATCH 20/22] Resolve all remaining FIXMEs with credit detection --- .../Entrypoint.cs | 2 +- .../QueueManager.cs | 52 ++++++++++++++++- .../ScheduledTasks/DetectCreditsTask.cs | 14 ++--- .../ScheduledTasks/DetectIntroductionsTask.cs | 57 +++---------------- 4 files changed, 64 insertions(+), 61 deletions(-) diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Entrypoint.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Entrypoint.cs index a7a176a..a2c1877 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/Entrypoint.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Entrypoint.cs @@ -60,7 +60,7 @@ public Task RunAsync() // Enqueue all episodes at startup to ensure any FFmpeg errors appear as early as possible _logger.LogInformation("Running startup enqueue"); var queueManager = new QueueManager(_loggerFactory.CreateLogger(), _libraryManager); - queueManager.EnqueueAllEpisodes(); + queueManager.GetMediaItems(); } catch (Exception ex) { diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/QueueManager.cs b/ConfusedPolarBear.Plugin.IntroSkipper/QueueManager.cs index dfc5639..bfe2105 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/QueueManager.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/QueueManager.cs @@ -3,6 +3,7 @@ namespace ConfusedPolarBear.Plugin.IntroSkipper; using System; using System.Collections.Generic; using System.Collections.ObjectModel; +using System.IO; using System.Linq; using Jellyfin.Data.Enums; using MediaBrowser.Controller.Entities; @@ -37,10 +38,10 @@ public QueueManager(ILogger logger, ILibraryManager libraryManager } /// - /// Iterates through all libraries on the server and queues all episodes for analysis. + /// Gets all media items on the server. /// /// Queued media items. - public ReadOnlyDictionary> EnqueueAllEpisodes() + public ReadOnlyDictionary> GetMediaItems() { // Assert that ffmpeg with chromaprint is installed if (!FFmpegWrapper.CheckFFmpegVersion()) @@ -220,4 +221,51 @@ private void QueueEpisode(Episode episode) Plugin.Instance!.TotalQueued++; } + + /// + /// Verify that a collection of queued media items still exist in Jellyfin and in storage. + /// This is done to ensure that we don't analyze items that were deleted between the call to GetMediaItems() and popping them from the queue. + /// + /// Queued media items. + /// Analysis mode. + /// Media items that have been verified to exist in Jellyfin and in storage. + public (ReadOnlyCollection VerifiedItems, bool AnyUnanalyzed) + VerifyQueue(ReadOnlyCollection candidates, AnalysisMode mode) + { + var unanalyzed = false; + var verified = new List(); + + var timestamps = mode == AnalysisMode.Introduction ? + Plugin.Instance!.Intros : + Plugin.Instance!.Credits; + + foreach (var candidate in candidates) + { + try + { + var path = Plugin.Instance!.GetItemPath(candidate.EpisodeId); + + if (File.Exists(path)) + { + verified.Add(candidate); + } + + if (!timestamps.ContainsKey(candidate.EpisodeId)) + { + unanalyzed = true; + } + } + catch (Exception ex) + { + _logger.LogDebug( + "Skipping {Mode} analysis of {Name} ({Id}): {Exception}", + mode, + candidate.Name, + candidate.EpisodeId, + ex); + } + } + + return (verified.AsReadOnly(), unanalyzed); + } } diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs index 74ef0c0..fa05f34 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs @@ -9,10 +9,6 @@ namespace ConfusedPolarBear.Plugin.IntroSkipper; -#if !DEBUG -#error Fix all FIXMEs introduced during initial credit implementation before release -#endif - /// /// Analyze all television episodes for credits. /// @@ -84,7 +80,7 @@ public Task ExecuteAsync(IProgress progress, CancellationToken cancellat _loggerFactory.CreateLogger(), _libraryManager); - var queue = queueManager.EnqueueAllEpisodes(); + var queue = queueManager.GetMediaItems(); if (queue.Count == 0) { @@ -101,9 +97,11 @@ public Task ExecuteAsync(IProgress progress, CancellationToken cancellat // Analyze all episodes in the queue using the degrees of parallelism the user specified. Parallel.ForEach(queue, options, (season) => { - // TODO: FIXME: use VerifyEpisodes - var episodes = season.Value.AsReadOnly(); - if (episodes.Count == 0) + var (episodes, unanalyzed) = queueManager.VerifyQueue( + season.Value.AsReadOnly(), + AnalysisMode.Credits); + + if (episodes.Count == 0 || unanalyzed) { return; } diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectIntroductionsTask.cs b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectIntroductionsTask.cs index 83df267..23890a0 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectIntroductionsTask.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectIntroductionsTask.cs @@ -83,7 +83,7 @@ public Task ExecuteAsync(IProgress progress, CancellationToken cancellat _loggerFactory.CreateLogger(), _libraryManager); - var queue = queueManager.EnqueueAllEpisodes(); + var queue = queueManager.GetMediaItems(); if (queue.Count == 0) { @@ -100,13 +100,15 @@ public Task ExecuteAsync(IProgress progress, CancellationToken cancellat MaxDegreeOfParallelism = Plugin.Instance!.Configuration.MaxParallelism }; - // TODO: if the queue is modified while the task is running, the task will fail. - // clone the queue before running the task to prevent this. - // Analyze all episodes in the queue using the degrees of parallelism the user specified. Parallel.ForEach(queue, options, (season) => { - var (episodes, unanalyzed) = VerifyEpisodes(season.Value.AsReadOnly()); + // Since the first run of the task can run for multiple hours, ensure that none + // of the current media items were deleted from Jellyfin since the task was started. + var (episodes, unanalyzed) = queueManager.VerifyQueue( + season.Value.AsReadOnly(), + AnalysisMode.Introduction); + if (episodes.Count == 0) { return; @@ -178,51 +180,6 @@ public Task ExecuteAsync(IProgress progress, CancellationToken cancellat return Task.CompletedTask; } - /// - /// Verify that all episodes in a season exist in Jellyfin and as a file in storage. - /// TODO: FIXME: move to queue manager. - /// - /// QueuedEpisodes. - /// Verified QueuedEpisodes and a flag indicating if any episode in this season has not been analyzed yet. - private ( - ReadOnlyCollection VerifiedEpisodes, - bool AnyUnanalyzed) - VerifyEpisodes(ReadOnlyCollection candidates) - { - var unanalyzed = false; - var verified = new List(); - - foreach (var candidate in candidates) - { - try - { - // Verify that the episode exists in Jellyfin and in storage - var path = Plugin.Instance!.GetItemPath(candidate.EpisodeId); - - if (File.Exists(path)) - { - verified.Add(candidate); - } - - // Flag this season for analysis if the current episode hasn't been analyzed yet - if (!Plugin.Instance.Intros.ContainsKey(candidate.EpisodeId)) - { - unanalyzed = true; - } - } - catch (Exception ex) - { - _logger.LogDebug( - "Skipping analysis of {Name} ({Id}): {Exception}", - candidate.Name, - candidate.EpisodeId, - ex); - } - } - - return (verified.AsReadOnly(), unanalyzed); - } - /// /// Fingerprints all episodes in the provided season and stores the timestamps of all introductions. /// From 431aed58ffa8705de38a919f7804b3e4350ba9c3 Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Thu, 2 Feb 2023 01:20:11 -0600 Subject: [PATCH 21/22] Combine introduction and credits detection code --- .../ScheduledTasks/BaseItemAnalyzerTask.cs | 198 ++++++++++++++++++ .../ScheduledTasks/DetectCreditsTask.cs | 136 +----------- .../ScheduledTasks/DetectIntroductionsTask.cs | 164 +-------------- 3 files changed, 217 insertions(+), 281 deletions(-) create mode 100644 ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/BaseItemAnalyzerTask.cs diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/BaseItemAnalyzerTask.cs b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/BaseItemAnalyzerTask.cs new file mode 100644 index 0000000..6a8f50e --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/BaseItemAnalyzerTask.cs @@ -0,0 +1,198 @@ +namespace ConfusedPolarBear.Plugin.IntroSkipper; + +using System; +using System.Collections.ObjectModel; +using System.Threading; +using System.Threading.Tasks; +using MediaBrowser.Controller.Library; +using Microsoft.Extensions.Logging; + +/// +/// Common code shared by all media item analyzer tasks. +/// +public class BaseItemAnalyzerTask +{ + private readonly AnalysisMode _analysisMode; + + private readonly ILogger _logger; + + private readonly ILoggerFactory _loggerFactory; + + private readonly ILibraryManager _libraryManager; + + /// + /// Initializes a new instance of the class. + /// + /// Analysis mode. + /// Task logger. + /// Logger factory. + /// Library manager. + public BaseItemAnalyzerTask( + AnalysisMode mode, + ILogger logger, + ILoggerFactory loggerFactory, + ILibraryManager libraryManager) + { + _analysisMode = mode; + _logger = logger; + _loggerFactory = loggerFactory; + _libraryManager = libraryManager; + + if (mode == AnalysisMode.Introduction) + { + EdlManager.Initialize(_logger); + } + } + + /// + /// Analyze all media items on the server. + /// + /// Progress. + /// Cancellation token. + public void AnalyzeItems( + IProgress progress, + CancellationToken cancellationToken) + { + var queueManager = new QueueManager( + _loggerFactory.CreateLogger(), + _libraryManager); + + var queue = queueManager.GetMediaItems(); + + var totalQueued = 0; + foreach (var kvp in queue) + { + totalQueued += kvp.Value.Count; + } + + if (totalQueued == 0) + { + throw new FingerprintException( + "No episodes to analyze. If you are limiting the list of libraries to analyze, check that all library names have been spelled correctly."); + } + + if (this._analysisMode == AnalysisMode.Introduction) + { + EdlManager.LogConfiguration(); + } + + var totalProcessed = 0; + var options = new ParallelOptions() + { + MaxDegreeOfParallelism = Plugin.Instance!.Configuration.MaxParallelism + }; + + Parallel.ForEach(queue, options, (season) => + { + var writeEdl = false; + + // Since the first run of the task can run for multiple hours, ensure that none + // of the current media items were deleted from Jellyfin since the task was started. + var (episodes, unanalyzed) = queueManager.VerifyQueue( + season.Value.AsReadOnly(), + this._analysisMode); + + if (episodes.Count == 0) + { + return; + } + + var first = episodes[0]; + + if (!unanalyzed) + { + _logger.LogDebug( + "All episodes in {Name} season {Season} have already been analyzed", + first.SeriesName, + first.SeasonNumber); + + return; + } + + try + { + if (cancellationToken.IsCancellationRequested) + { + return; + } + + var analyzed = AnalyzeItems(episodes, cancellationToken); + Interlocked.Add(ref totalProcessed, analyzed); + + writeEdl = analyzed > 0 || Plugin.Instance!.Configuration.RegenerateEdlFiles; + } + catch (FingerprintException ex) + { + _logger.LogWarning( + "Unable to analyze {Series} season {Season}: unable to fingerprint: {Ex}", + first.SeriesName, + first.SeasonNumber, + ex); + } + + if ( + writeEdl && + Plugin.Instance!.Configuration.EdlAction != EdlAction.None && + _analysisMode == AnalysisMode.Introduction) + { + EdlManager.UpdateEDLFiles(episodes); + } + + progress.Report((totalProcessed * 100) / totalQueued); + }); + + if ( + _analysisMode == AnalysisMode.Introduction && + Plugin.Instance!.Configuration.RegenerateEdlFiles) + { + _logger.LogInformation("Turning EDL file regeneration flag off"); + Plugin.Instance!.Configuration.RegenerateEdlFiles = false; + Plugin.Instance!.SaveConfiguration(); + } + } + + /// + /// Analyze a group of media items for skippable segments. + /// + /// Media items to analyze. + /// Cancellation token. + /// Number of items that were successfully analyzed. + private int AnalyzeItems( + ReadOnlyCollection items, + CancellationToken cancellationToken) + { + var totalItems = items.Count; + + // Only analyze specials (season 0) if the user has opted in. + var first = items[0]; + if (first.SeasonNumber == 0 && !Plugin.Instance!.Configuration.AnalyzeSeasonZero) + { + return 0; + } + + _logger.LogInformation( + "Analyzing {Count} files from {Name} season {Season}", + items.Count, + first.SeriesName, + first.SeasonNumber); + + var analyzers = new Collection(); + + analyzers.Add(new ChapterAnalyzer(_loggerFactory.CreateLogger())); + analyzers.Add(new ChromaprintAnalyzer(_loggerFactory.CreateLogger())); + + if (this._analysisMode == AnalysisMode.Credits) + { + analyzers.Add(new BlackFrameAnalyzer(_loggerFactory.CreateLogger())); + } + + // Use each analyzer to find skippable ranges in all media files, removing successfully + // analyzed items from the queue. + foreach (var analyzer in analyzers) + { + items = analyzer.AnalyzeMediaFiles(items, this._analysisMode, cancellationToken); + } + + return totalItems - items.Count; + } +} diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs index fa05f34..8e3bd9c 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Collections.ObjectModel; using System.Threading; using System.Threading.Tasks; using MediaBrowser.Controller.Library; @@ -11,14 +10,13 @@ namespace ConfusedPolarBear.Plugin.IntroSkipper; /// /// Analyze all television episodes for credits. +/// TODO: analyze all media files. /// public class DetectCreditsTask : IScheduledTask { - private readonly ILogger _logger; - private readonly ILoggerFactory _loggerFactory; - private readonly ILibraryManager? _libraryManager; + private readonly ILibraryManager _libraryManager; /// /// Initializes a new instance of the class. @@ -27,19 +25,10 @@ public class DetectCreditsTask : IScheduledTask /// Library manager. public DetectCreditsTask( ILoggerFactory loggerFactory, - ILibraryManager libraryManager) : this(loggerFactory) - { - _libraryManager = libraryManager; - } - - /// - /// Initializes a new instance of the class. - /// - /// Logger factory. - public DetectCreditsTask(ILoggerFactory loggerFactory) + ILibraryManager libraryManager) { - _logger = loggerFactory.CreateLogger(); _loggerFactory = loggerFactory; + _libraryManager = libraryManager; } /// @@ -72,125 +61,20 @@ public Task ExecuteAsync(IProgress progress, CancellationToken cancellat { if (_libraryManager is null) { - throw new InvalidOperationException("Library manager must not be null"); + throw new InvalidOperationException("Library manager was null"); } - // Make sure the analysis queue matches what's currently in Jellyfin. - var queueManager = new QueueManager( - _loggerFactory.CreateLogger(), + var baseAnalyzer = new BaseItemAnalyzerTask( + AnalysisMode.Credits, + _loggerFactory.CreateLogger(), + _loggerFactory, _libraryManager); - var queue = queueManager.GetMediaItems(); - - if (queue.Count == 0) - { - throw new FingerprintException( - "No episodes to analyze. If you are limiting the list of libraries to analyze, check that all library names have been spelled correctly."); - } - - var totalProcessed = 0; - var options = new ParallelOptions() - { - MaxDegreeOfParallelism = Plugin.Instance!.Configuration.MaxParallelism - }; - - // Analyze all episodes in the queue using the degrees of parallelism the user specified. - Parallel.ForEach(queue, options, (season) => - { - var (episodes, unanalyzed) = queueManager.VerifyQueue( - season.Value.AsReadOnly(), - AnalysisMode.Credits); - - if (episodes.Count == 0 || unanalyzed) - { - return; - } - - var first = episodes[0]; - - try - { - if (cancellationToken.IsCancellationRequested) - { - return; - } - - AnalyzeSeason(episodes, cancellationToken); - Interlocked.Add(ref totalProcessed, episodes.Count); - } - catch (FingerprintException ex) - { - _logger.LogWarning( - "Unable to analyze {Series} season {Season}: unable to fingerprint: {Ex}", - first.SeriesName, - first.SeasonNumber, - ex); - } - catch (KeyNotFoundException ex) - { - _logger.LogWarning( - "Unable to analyze {Series} season {Season}: cache miss: {Ex}", - first.SeriesName, - first.SeasonNumber, - ex); - } - - var total = Plugin.Instance!.TotalQueued; - if (total > 0) - { - progress.Report((totalProcessed * 100) / total); - } - }); + baseAnalyzer.AnalyzeItems(progress, cancellationToken); return Task.CompletedTask; } - /// - /// Analyzes all episodes in the season for end credits. - /// - /// Episodes in this season. - /// Cancellation token provided by the scheduled task. - private void AnalyzeSeason( - ReadOnlyCollection episodes, - CancellationToken cancellationToken) - { - // Only analyze specials (season 0) if the user has opted in. - if (episodes[0].SeasonNumber == 0 && !Plugin.Instance!.Configuration.AnalyzeSeasonZero) - { - return; - } - - // Analyze with Chromaprint first and fall back to the black frame detector - var analyzers = new IMediaFileAnalyzer[] - { - new ChromaprintAnalyzer(_loggerFactory.CreateLogger()), - new BlackFrameAnalyzer(_loggerFactory.CreateLogger()) - }; - - // Use each analyzer to find credits in all media files, removing successfully analyzed files - // from the queue. - var remaining = new ReadOnlyCollection(episodes); - foreach (var analyzer in analyzers) - { - remaining = AnalyzeFiles(remaining, analyzer, cancellationToken); - } - } - - private ReadOnlyCollection AnalyzeFiles( - ReadOnlyCollection episodes, - IMediaFileAnalyzer analyzer, - CancellationToken cancellationToken) - { - _logger.LogInformation( - "Analyzing {Count} episodes from {Name} season {Season} with {Analyzer}", - episodes.Count, - episodes[0].SeriesName, - episodes[0].SeasonNumber, - analyzer.GetType().Name); - - return analyzer.AnalyzeMediaFiles(episodes, AnalysisMode.Credits, cancellationToken); - } - /// /// Get task triggers. /// diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectIntroductionsTask.cs b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectIntroductionsTask.cs index 23890a0..c0f1609 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectIntroductionsTask.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectIntroductionsTask.cs @@ -1,7 +1,5 @@ using System; using System.Collections.Generic; -using System.Collections.ObjectModel; -using System.IO; using System.Threading; using System.Threading.Tasks; using MediaBrowser.Controller.Library; @@ -15,11 +13,9 @@ namespace ConfusedPolarBear.Plugin.IntroSkipper; /// public class DetectIntroductionsTask : IScheduledTask { - private readonly ILogger _logger; - private readonly ILoggerFactory _loggerFactory; - private readonly ILibraryManager? _libraryManager; + private readonly ILibraryManager _libraryManager; /// /// Initializes a new instance of the class. @@ -28,21 +24,10 @@ public class DetectIntroductionsTask : IScheduledTask /// Library manager. public DetectIntroductionsTask( ILoggerFactory loggerFactory, - ILibraryManager libraryManager) : this(loggerFactory) + ILibraryManager libraryManager) { - _libraryManager = libraryManager; - } - - /// - /// Initializes a new instance of the class. - /// - /// Logger factory. - public DetectIntroductionsTask(ILoggerFactory loggerFactory) - { - _logger = loggerFactory.CreateLogger(); _loggerFactory = loggerFactory; - - EdlManager.Initialize(_logger); + _libraryManager = libraryManager; } /// @@ -75,151 +60,20 @@ public Task ExecuteAsync(IProgress progress, CancellationToken cancellat { if (_libraryManager is null) { - throw new InvalidOperationException("Library manager must not be null"); + throw new InvalidOperationException("Library manager was null"); } - // Make sure the analysis queue matches what's currently in Jellyfin. - var queueManager = new QueueManager( - _loggerFactory.CreateLogger(), + var baseAnalyzer = new BaseItemAnalyzerTask( + AnalysisMode.Introduction, + _loggerFactory.CreateLogger(), + _loggerFactory, _libraryManager); - var queue = queueManager.GetMediaItems(); - - if (queue.Count == 0) - { - throw new FingerprintException( - "No episodes to analyze. If you are limiting the list of libraries to analyze, check that all library names have been spelled correctly."); - } - - // Log EDL settings - EdlManager.LogConfiguration(); - - var totalProcessed = 0; - var options = new ParallelOptions() - { - MaxDegreeOfParallelism = Plugin.Instance!.Configuration.MaxParallelism - }; - - // Analyze all episodes in the queue using the degrees of parallelism the user specified. - Parallel.ForEach(queue, options, (season) => - { - // Since the first run of the task can run for multiple hours, ensure that none - // of the current media items were deleted from Jellyfin since the task was started. - var (episodes, unanalyzed) = queueManager.VerifyQueue( - season.Value.AsReadOnly(), - AnalysisMode.Introduction); - - if (episodes.Count == 0) - { - return; - } - - var first = episodes[0]; - var writeEdl = false; - - if (!unanalyzed) - { - _logger.LogDebug( - "All episodes in {Name} season {Season} have already been analyzed", - first.SeriesName, - first.SeasonNumber); - - return; - } - - try - { - if (cancellationToken.IsCancellationRequested) - { - return; - } - - // Increment totalProcessed by the number of episodes in this season that were actually analyzed - // (instead of just using the number of episodes in the current season). - var analyzed = AnalyzeSeason(episodes, cancellationToken); - Interlocked.Add(ref totalProcessed, analyzed); - writeEdl = analyzed > 0 || Plugin.Instance!.Configuration.RegenerateEdlFiles; - } - catch (FingerprintException ex) - { - _logger.LogWarning( - "Unable to analyze {Series} season {Season}: unable to fingerprint: {Ex}", - first.SeriesName, - first.SeasonNumber, - ex); - } - catch (KeyNotFoundException ex) - { - _logger.LogWarning( - "Unable to analyze {Series} season {Season}: cache miss: {Ex}", - first.SeriesName, - first.SeasonNumber, - ex); - } - - if (writeEdl && Plugin.Instance!.Configuration.EdlAction != EdlAction.None) - { - EdlManager.UpdateEDLFiles(episodes); - } - - var total = Plugin.Instance!.TotalQueued; - if (total > 0) - { - progress.Report((totalProcessed * 100) / total); - } - }); - - // Turn the regenerate EDL flag off after the scan completes. - if (Plugin.Instance!.Configuration.RegenerateEdlFiles) - { - _logger.LogInformation("Turning EDL file regeneration flag off"); - Plugin.Instance!.Configuration.RegenerateEdlFiles = false; - Plugin.Instance!.SaveConfiguration(); - } + baseAnalyzer.AnalyzeItems(progress, cancellationToken); return Task.CompletedTask; } - /// - /// Fingerprints all episodes in the provided season and stores the timestamps of all introductions. - /// - /// Episodes in this season. - /// Cancellation token provided by the scheduled task. - /// Number of episodes from the provided season that were analyzed. - private int AnalyzeSeason( - ReadOnlyCollection episodes, - CancellationToken cancellationToken) - { - // Skip seasons with an insufficient number of episodes. - if (episodes.Count <= 1) - { - return episodes.Count; - } - - // Only analyze specials (season 0) if the user has opted in. - var first = episodes[0]; - if (first.SeasonNumber == 0 && !Plugin.Instance!.Configuration.AnalyzeSeasonZero) - { - return 0; - } - - _logger.LogInformation( - "Analyzing {Count} episodes from {Name} season {Season}", - episodes.Count, - first.SeriesName, - first.SeasonNumber); - - // Chapter analyzer - var chapter = new ChapterAnalyzer(_loggerFactory.CreateLogger()); - episodes = chapter.AnalyzeMediaFiles(episodes, AnalysisMode.Introduction, cancellationToken); - - // Analyze the season with Chromaprint - var chromaprint = new ChromaprintAnalyzer(_loggerFactory.CreateLogger()); - chromaprint.AnalyzeMediaFiles(episodes, AnalysisMode.Introduction, cancellationToken); - - return episodes.Count; - } - /// /// Get task triggers. /// From c6801d57c629de265f47b2eabc1ae32d46b9b24d Mon Sep 17 00:00:00 2001 From: ConfusedPolarBear <33811686+ConfusedPolarBear@users.noreply.github.com> Date: Thu, 2 Feb 2023 03:52:31 -0600 Subject: [PATCH 22/22] Fix progress bar --- .../ScheduledTasks/BaseItemAnalyzerTask.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/BaseItemAnalyzerTask.cs b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/BaseItemAnalyzerTask.cs index 6a8f50e..940e0a0 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/BaseItemAnalyzerTask.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/BaseItemAnalyzerTask.cs @@ -193,6 +193,6 @@ private int AnalyzeItems( items = analyzer.AnalyzeMediaFiles(items, this._analysisMode, cancellationToken); } - return totalItems - items.Count; + return totalItems; } }