Add initial end credits detection code

CHANGELOG.md

@@ -1,6 +1,8 @@
 # Changelog
 
 ## v0.1.8.0 (no eta)
+* New features
+  * Detect ending credits in television episodes
 * Internal changes
   * Move Chromaprint analysis code out of the episode analysis task
   * Add support for multiple analysis techinques

ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestAudioFingerprinting.cs

@@ -59,7 +59,9 @@ public void TestFingerprinting()
             3472417825, 3395841056, 3458735136, 3341420624, 1076496560, 1076501168, 1076501136, 1076497024
         };
 
-        var actual = FFmpegWrapper.Fingerprint(queueEpisode("audio/big_buck_bunny_intro.mp3"));
+        var actual = FFmpegWrapper.Fingerprint(
+            queueEpisode("audio/big_buck_bunny_intro.mp3"),
+            AnalysisMode.Introduction);
 
         Assert.Equal(expected, actual);
     }
@@ -91,8 +93,8 @@ public void TestIntroDetection()
 
         var lhsEpisode = queueEpisode("audio/big_buck_bunny_intro.mp3");
         var rhsEpisode = queueEpisode("audio/big_buck_bunny_clip.mp3");
-        var lhsFingerprint = FFmpegWrapper.Fingerprint(lhsEpisode);
-        var rhsFingerprint = FFmpegWrapper.Fingerprint(rhsEpisode);
+        var lhsFingerprint = FFmpegWrapper.Fingerprint(lhsEpisode, AnalysisMode.Introduction);
+        var rhsFingerprint = FFmpegWrapper.Fingerprint(rhsEpisode, AnalysisMode.Introduction);
 
         var (lhs, rhs) = chromaprint.CompareEpisodes(
             lhsEpisode.EpisodeId,
@@ -138,7 +140,7 @@ private QueuedEpisode queueEpisode(string path)
         {
             EpisodeId = Guid.NewGuid(),
             Path = "../../../" + path,
-            FingerprintDuration = 60
+            IntroFingerprintEnd = 60
         };
     }
 

ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/Chromaprint.cs

@@ -30,6 +30,8 @@ public class ChromaprintAnalyzer : IMediaFileAnalyzer
 
     private ILogger<ChromaprintAnalyzer> _logger;
 
+    private AnalysisMode _analysisMode;
+
     /// <summary>
     /// Initializes a new instance of the <see cref="ChromaprintAnalyzer"/> class.
     /// </summary>
@@ -64,12 +66,14 @@ public ReadOnlyCollection<QueuedEpisode> AnalyzeMediaFiles(
         // Episodes that were analyzed and do not have an introduction.
         var episodesWithoutIntros = new List<QueuedEpisode>();
 
+        this._analysisMode = mode;
+
         // Compute fingerprints for all episodes in the season
         foreach (var episode in episodeAnalysisQueue)
         {
             try
             {
-                fingerprintCache[episode.EpisodeId] = FFmpegWrapper.Fingerprint(episode);
+                fingerprintCache[episode.EpisodeId] = FFmpegWrapper.Fingerprint(episode, mode);
 
                 if (cancellationToken.IsCancellationRequested)
                 {
@@ -78,6 +82,7 @@ public ReadOnlyCollection<QueuedEpisode> AnalyzeMediaFiles(
             }
             catch (FingerprintException ex)
             {
+                // TODO: FIXME: move to debug level?
                 _logger.LogWarning("Caught fingerprint error: {Ex}", ex);
 
                 // Fallback to an empty fingerprint on any error
@@ -112,6 +117,22 @@ public ReadOnlyCollection<QueuedEpisode> AnalyzeMediaFiles(
                     continue;
                 }
 
+                /* Since the Fingerprint() function returns an array of Chromaprint points without time
+                 * information, the times reported from the index search function start from 0.
+                 *
+                 * While this is desired behavior for detecting introductions, it breaks credit
+                 * detection, as the audio we're analyzing was extracted from some point into the file.
+                 *
+                 * To fix this, add the starting time of the fingerprint to the reported time range.
+                 */
+                if (this._analysisMode == AnalysisMode.Credits)
+                {
+                    currentIntro.IntroStart += currentEpisode.CreditsFingerprintStart;
+                    currentIntro.IntroEnd += currentEpisode.CreditsFingerprintStart;
+                    remainingIntro.IntroStart += remainingEpisode.CreditsFingerprintStart;
+                    remainingIntro.IntroEnd += remainingEpisode.CreditsFingerprintStart;
+                }
+
                 // Only save the discovered intro if it is:
                 // - the first intro discovered for this episode
                 // - longer than the previously discovered intro
@@ -142,10 +163,13 @@ public ReadOnlyCollection<QueuedEpisode> AnalyzeMediaFiles(
             return analysisQueue;
         }
 
-        // Adjust all introduction end times so that they end at silence.
-        seasonIntros = AdjustIntroEndTimes(analysisQueue, seasonIntros);
+        if (this._analysisMode == AnalysisMode.Introduction)
+        {
+            // Adjust all introduction end times so that they end at silence.
+            seasonIntros = AdjustIntroEndTimes(analysisQueue, seasonIntros);
+        }
 
-        Plugin.Instance!.UpdateTimestamps(seasonIntros);
+        Plugin.Instance!.UpdateTimestamps(seasonIntros, this._analysisMode);
 
         return episodesWithoutIntros.AsReadOnly();
     }
@@ -338,16 +362,20 @@ public ReadOnlyCollection<QueuedEpisode> AnalyzeMediaFiles(
         // Since LHS had a contiguous time range, RHS must have one also.
         var rContiguous = TimeRangeHelpers.FindContiguous(rhsTimes.ToArray(), maximumTimeSkip)!;
 
-        // Tweak the end timestamps just a bit to ensure as little content as possible is skipped over.
-        if (lContiguous.Duration >= 90)
+        if (this._analysisMode == AnalysisMode.Introduction)
         {
-            lContiguous.End -= 2 * maximumTimeSkip;
-            rContiguous.End -= 2 * maximumTimeSkip;
-        }
-        else if (lContiguous.Duration >= 30)
-        {
-            lContiguous.End -= maximumTimeSkip;
-            rContiguous.End -= maximumTimeSkip;
+            // Tweak the end timestamps just a bit to ensure as little content as possible is skipped over.
+            // TODO: remove this
+            if (lContiguous.Duration >= 90)
+            {
+                lContiguous.End -= 2 * maximumTimeSkip;
+                rContiguous.End -= 2 * maximumTimeSkip;
+            }
+            else if (lContiguous.Duration >= 30)
+            {
+                lContiguous.End -= maximumTimeSkip;
+                rContiguous.End -= maximumTimeSkip;
+            }
         }
 
         return (lContiguous, rContiguous);

ConfusedPolarBear.Plugin.IntroSkipper/Configuration/PluginConfiguration.cs

@@ -72,6 +72,11 @@ public PluginConfiguration()
     /// </summary>
     public int MaximumIntroDuration { get; set; } = 120;
 
+    /// <summary>
+    /// Gets or sets the upper limit (in minutes) on the length of each episode's audio track that will be analyzed when searching for ending credits.
+    /// </summary>
+    public int MaximumEpisodeCreditsDuration { get; set; } = 4;
+
     // ===== Playback settings =====
 
     /// <summary>

ConfusedPolarBear.Plugin.IntroSkipper/Controllers/VisualizationController.cs

@@ -113,7 +113,7 @@ public ActionResult<uint[]> GetEpisodeFingerprint([FromRoute] Guid id)
             {
                 if (needle.EpisodeId == id)
                 {
-                    return FFmpegWrapper.Fingerprint(needle);
+                    return FFmpegWrapper.Fingerprint(needle, AnalysisMode.Introduction);
                 }
             }
         }

ConfusedPolarBear.Plugin.IntroSkipper/Data/QueuedEpisode.cs

@@ -33,7 +33,17 @@ public class QueuedEpisode
     public string Name { get; set; } = string.Empty;
 
     /// <summary>
-    /// Gets or sets the seconds of media file to fingerprint.
+    /// Gets or sets the timestamp (in seconds) to stop searching for an introduction.
     /// </summary>
-    public int FingerprintDuration { get; set; }
+    public int IntroFingerprintEnd { get; set; }
+
+    /// <summary>
+    /// Gets or sets the timestamp (in seconds) to start looking for end credits.
+    /// </summary>
+    public int CreditsFingerprintStart { get; set; }
+
+    /// <summary>
+    /// Gets or sets the total duration of this media file (in seconds).
+    /// </summary>
+    public int Duration { get; set; }
 }

ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs

@@ -134,27 +134,60 @@ private static bool CheckFFmpegRequirement(
     /// Fingerprint a queued episode.
     /// </summary>
     /// <param name="episode">Queued episode to fingerprint.</param>
+    /// <param name="mode">Portion of media file to fingerprint. Introduction = first 25% / 10 minutes and Credits = last 4 minutes.</param>
     /// <returns>Numerical fingerprint points.</returns>
-    public static uint[] Fingerprint(QueuedEpisode episode)
+    public static uint[] Fingerprint(QueuedEpisode episode, AnalysisMode mode)
+    {
+        int start, end;
+
+        if (mode == AnalysisMode.Introduction)
+        {
+            start = 0;
+            end = episode.IntroFingerprintEnd;
+        }
+        else if (mode == AnalysisMode.Credits)
+        {
+            start = episode.CreditsFingerprintStart;
+            end = episode.Duration;
+        }
+        else
+        {
+            throw new ArgumentException("Unknown analysis mode " + mode.ToString());
+        }
+
+        return Fingerprint(episode, mode, start, end);
+    }
+
+    /// <summary>
+    /// Fingerprint a queued episode.
+    /// </summary>
+    /// <param name="episode">Queued episode to fingerprint.</param>
+    /// <param name="mode">Portion of media file to fingerprint.</param>
+    /// <param name="start">Time (in seconds) relative to the start of the file to start fingerprinting from.</param>
+    /// <param name="end">Time (in seconds) relative to the start of the file to stop fingerprinting at.</param>
+    /// <returns>Numerical fingerprint points.</returns>
+    private static uint[] Fingerprint(QueuedEpisode episode, AnalysisMode mode, int start, int end)
     {
         // Try to load this episode from cache before running ffmpeg.
-        if (LoadCachedFingerprint(episode, out uint[] cachedFingerprint))
+        if (LoadCachedFingerprint(episode, mode, out uint[] cachedFingerprint))
         {
             Logger?.LogTrace("Fingerprint cache hit on {File}", episode.Path);
             return cachedFingerprint;
         }
 
         Logger?.LogDebug(
-            "Fingerprinting {Duration} seconds from \"{File}\" (id {Id})",
-            episode.FingerprintDuration,
+            "Fingerprinting [{Start}, {End}] from \"{File}\" (id {Id})",
+            start,
+            end,
             episode.Path,
             episode.EpisodeId);
 
         var args = string.Format(
             CultureInfo.InvariantCulture,
-            "-i \"{0}\" -to {1} -ac 2 -f chromaprint -fp_format raw -",
+            "-ss {0} -i \"{1}\" -to {2} -ac 2 -f chromaprint -fp_format raw -",
+            start,
             episode.Path,
-            episode.FingerprintDuration);
+            end - start);
 
         // Returns all fingerprint points as raw 32 bit unsigned integers (little endian).
         var rawPoints = GetOutput(args, string.Empty);
@@ -172,7 +205,7 @@ public static uint[] Fingerprint(QueuedEpisode episode)
         }
 
         // Try to cache this fingerprint.
-        CacheFingerprint(episode, results);
+        CacheFingerprint(episode, mode, results);
 
         return results.ToArray();
     }
@@ -226,9 +259,6 @@ public static TimeRange[] DetectSilence(QueuedEpisode episode, int limit)
             limit,
             episode.EpisodeId);
 
-        // TODO: select the audio track that matches the user's preferred language, falling
-        //     back to the first track if nothing matches
-
         // -vn, -sn, -dn: ignore video, subtitle, and data tracks
         var args = string.Format(
             CultureInfo.InvariantCulture,
@@ -367,9 +397,13 @@ private static ReadOnlySpan<byte> GetOutput(
     /// This function was created before the unified caching mechanism was introduced (in v0.1.7).
     /// </summary>
     /// <param name="episode">Episode to try to load from cache.</param>
+    /// <param name="mode">Analysis mode.</param>
     /// <param name="fingerprint">Array to store the fingerprint in.</param>
     /// <returns>true if the episode was successfully loaded from cache, false on any other error.</returns>
-    private static bool LoadCachedFingerprint(QueuedEpisode episode, out uint[] fingerprint)
+    private static bool LoadCachedFingerprint(
+        QueuedEpisode episode,
+        AnalysisMode mode,
+        out uint[] fingerprint)
     {
         fingerprint = Array.Empty<uint>();
 
@@ -379,15 +413,14 @@ private static bool LoadCachedFingerprint(QueuedEpisode episode, out uint[] fing
             return false;
         }
 
-        var path = GetFingerprintCachePath(episode);
+        var path = GetFingerprintCachePath(episode, mode);
 
         // If this episode isn't cached, bail out.
         if (!File.Exists(path))
         {
             return false;
         }
 
-        // TODO: make async
         var raw = File.ReadAllLines(path, Encoding.UTF8);
         var result = new List<uint>();
 
@@ -421,8 +454,12 @@ private static bool LoadCachedFingerprint(QueuedEpisode episode, out uint[] fing
     /// This function was created before the unified caching mechanism was introduced (in v0.1.7).
     /// </summary>
     /// <param name="episode">Episode to store in cache.</param>
+    /// <param name="mode">Analysis mode.</param>
     /// <param name="fingerprint">Fingerprint of the episode to store.</param>
-    private static void CacheFingerprint(QueuedEpisode episode, List<uint> fingerprint)
+    private static void CacheFingerprint(
+        QueuedEpisode episode,
+        AnalysisMode mode,
+        List<uint> fingerprint)
     {
         // Bail out if caching isn't enabled.
         if (!(Plugin.Instance?.Configuration.CacheFingerprints ?? false))
@@ -438,17 +475,36 @@ private static void CacheFingerprint(QueuedEpisode episode, List<uint> fingerpri
         }
 
         // Cache the episode.
-        File.WriteAllLinesAsync(GetFingerprintCachePath(episode), lines, Encoding.UTF8).ConfigureAwait(false);
+        File.WriteAllLinesAsync(
+            GetFingerprintCachePath(episode, mode),
+            lines,
+            Encoding.UTF8).ConfigureAwait(false);
     }
 
     /// <summary>
     /// Determines the path an episode should be cached at.
     /// This function was created before the unified caching mechanism was introduced (in v0.1.7).
     /// </summary>
     /// <param name="episode">Episode.</param>
-    private static string GetFingerprintCachePath(QueuedEpisode episode)
+    /// <param name="mode">Analysis mode.</param>
+    private static string GetFingerprintCachePath(QueuedEpisode episode, AnalysisMode mode)
     {
-        return Path.Join(Plugin.Instance!.FingerprintCachePath, episode.EpisodeId.ToString("N"));
+        var basePath = Path.Join(
+            Plugin.Instance!.FingerprintCachePath,
+            episode.EpisodeId.ToString("N"));
+
+        if (mode == AnalysisMode.Introduction)
+        {
+            return basePath;
+        }
+        else if (mode == AnalysisMode.Credits)
+        {
+            return basePath + "-credits";
+        }
+        else
+        {
+            throw new ArgumentException("Unknown analysis mode " + mode.ToString());
+        }
     }
 
     /// <summary>