diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestBlackFrames.cs b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestBlackFrames.cs index 794ecd1..bdd34e5 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestBlackFrames.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/TestBlackFrames.cs @@ -2,6 +2,7 @@ namespace ConfusedPolarBear.Plugin.IntroSkipper.Tests; using System; using System.Collections.Generic; +using Microsoft.Extensions.Logging; using Xunit; public class TestBlackFrames @@ -9,29 +10,42 @@ public class TestBlackFrames [FactSkipFFmpegTests] public void TestBlackFrameDetection() { + var range = 1e-5; + var expected = new List(); expected.AddRange(CreateFrameSequence(2, 3)); expected.AddRange(CreateFrameSequence(5, 6)); expected.AddRange(CreateFrameSequence(8, 9.96)); - var actual = FFmpegWrapper.DetectBlackFrames( - queueFile("rainbow.mp4"), - new TimeRange(0, 10) - ); + var actual = FFmpegWrapper.DetectBlackFrames(queueFile("rainbow.mp4"), new(0, 10), 85); for (var i = 0; i < expected.Count; i++) { var (e, a) = (expected[i], actual[i]); Assert.Equal(e.Percentage, a.Percentage); - Assert.True(Math.Abs(e.Time - a.Time) <= 0.005); + Assert.InRange(a.Time, e.Time - range, e.Time + range); } } + [FactSkipFFmpegTests] + public void TestEndCreditDetection() + { + var analyzer = CreateBlackFrameAnalyzer(); + + var episode = queueFile("credits.mp4"); + episode.Duration = (int)new TimeSpan(0, 5, 30).TotalSeconds; + + var result = analyzer.AnalyzeMediaFile(episode, AnalysisMode.Credits, 85); + Assert.NotNull(result); + Assert.Equal(300, result.IntroStart); + } + private QueuedEpisode queueFile(string path) { return new() { EpisodeId = Guid.NewGuid(), + Name = path, Path = "../../../video/" + path }; } @@ -47,4 +61,10 @@ private BlackFrame[] CreateFrameSequence(double start, double end) return frames.ToArray(); } + + private BlackFrameAnalyzer CreateBlackFrameAnalyzer() + { + var logger = new LoggerFactory().CreateLogger(); + return new(logger); + } } diff --git a/ConfusedPolarBear.Plugin.IntroSkipper.Tests/video/credits.mp4 b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/video/credits.mp4 new file mode 100644 index 0000000..c8fa8d2 Binary files /dev/null and b/ConfusedPolarBear.Plugin.IntroSkipper.Tests/video/credits.mp4 differ diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/BlackFrameAnalyzer.cs b/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/BlackFrameAnalyzer.cs new file mode 100644 index 0000000..1c28c5f --- /dev/null +++ b/ConfusedPolarBear.Plugin.IntroSkipper/Analyzers/BlackFrameAnalyzer.cs @@ -0,0 +1,125 @@ +namespace ConfusedPolarBear.Plugin.IntroSkipper; + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; +using System.Threading; +using Microsoft.Extensions.Logging; + +/// +/// Media file analyzer used to detect end credits that consist of text overlaid on a black background. +/// Bisects the end of the video file to perform an efficient search. +/// +public class BlackFrameAnalyzer : IMediaFileAnalyzer +{ + private readonly TimeSpan _maximumError = new(0, 0, 4); + + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + /// Logger. + public BlackFrameAnalyzer(ILogger logger) + { + _logger = logger; + } + + /// + public ReadOnlyCollection AnalyzeMediaFiles( + ReadOnlyCollection analysisQueue, + AnalysisMode mode, + CancellationToken cancellationToken) + { + if (mode != AnalysisMode.Credits) + { + throw new NotImplementedException("mode must equal Credits"); + } + + var creditTimes = new Dictionary(); + + foreach (var episode in analysisQueue) + { + if (cancellationToken.IsCancellationRequested) + { + break; + } + + var intro = AnalyzeMediaFile( + episode, + mode, + Plugin.Instance!.Configuration.BlackFrameMinimumPercentage); + + if (intro is null) + { + continue; + } + + creditTimes[episode.EpisodeId] = intro; + } + + Plugin.Instance!.UpdateTimestamps(creditTimes, mode); + + return analysisQueue + .Where(x => !creditTimes.ContainsKey(x.EpisodeId)) + .ToList() + .AsReadOnly(); + } + + /// + /// Analyzes an individual media file. Only public because of unit tests. + /// + /// Media file to analyze. + /// Analysis mode. + /// Percentage of the frame that must be black. + /// Credits timestamp. + public Intro? AnalyzeMediaFile(QueuedEpisode episode, AnalysisMode mode, int minimum) + { + // Start by analyzing the last four minutes of the file. + var start = TimeSpan.FromMinutes(4); + var end = TimeSpan.Zero; + var firstFrameTime = 0.0; + + // Continue bisecting the end of the file until the range that contains the first black + // frame is smaller than the maximum permitted error. + while (start - end > _maximumError) + { + // Analyze the middle two seconds from the current bisected range + var midpoint = (start + end) / 2; + var scanTime = episode.Duration - midpoint.TotalSeconds; + var tr = new TimeRange(scanTime, scanTime + 2); + + _logger.LogTrace( + "{Episode}, dur {Duration}, bisect [{BStart}, {BEnd}], time [{Start}, {End}]", + episode.Name, + episode.Duration, + start, + end, + tr.Start, + tr.End); + + var frames = FFmpegWrapper.DetectBlackFrames(episode, tr, minimum); + _logger.LogTrace("{Episode}, black frames: {Count}", episode.Name, frames.Length); + + if (frames.Length == 0) + { + // Since no black frames were found, slide the range closer to the end + start = midpoint; + } + else + { + // Some black frames were found, slide the range closer to the start + end = midpoint; + firstFrameTime = frames[0].Time + scanTime; + } + } + + if (firstFrameTime > 0) + { + return new(episode.EpisodeId, new TimeRange(firstFrameTime, episode.Duration)); + } + + return null; + } +} diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs b/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs index e8ba464..1cb2c4d 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/FFmpegWrapper.cs @@ -222,8 +222,12 @@ public static TimeRange[] DetectSilence(QueuedEpisode episode, int limit) /// /// Media file to analyze. /// Time range to search. - /// Array of frames that are at least 50% black. - public static BlackFrame[] DetectBlackFrames(QueuedEpisode episode, TimeRange range) + /// Percentage of the frame that must be black. + /// Array of frames that are mostly black. + public static BlackFrame[] DetectBlackFrames( + QueuedEpisode episode, + TimeRange range, + int minimum) { // Seek to the start of the time range and find frames that are at least 50% black. var args = string.Format( @@ -233,10 +237,10 @@ public static BlackFrame[] DetectBlackFrames(QueuedEpisode episode, TimeRange ra episode.Path, range.End - range.Start); - // Cache the results to GUID-blackframes-v1-START-END. + // Cache the results to GUID-blackframes-START-END-v1. var cacheKey = string.Format( CultureInfo.InvariantCulture, - "{0}-blackframes-v1-{1}-{2}", + "{0}-blackframes-{1}-{2}-v1", episode.EpisodeId.ToString("N"), range.Start, range.End); @@ -263,10 +267,14 @@ public static BlackFrame[] DetectBlackFrames(QueuedEpisode episode, TimeRange ra matches[1].Value.Split(':')[1] ); - blackFrames.Add(new( + var bf = new BlackFrame( Convert.ToInt32(strPercent, CultureInfo.InvariantCulture), - Convert.ToDouble(strTime, CultureInfo.InvariantCulture) - )); + Convert.ToDouble(strTime, CultureInfo.InvariantCulture)); + + if (bf.Percentage > minimum) + { + blackFrames.Add(bf); + } } return blackFrames.ToArray(); diff --git a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs index 224155f..d4da24e 100644 --- a/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs +++ b/ConfusedPolarBear.Plugin.IntroSkipper/ScheduledTasks/DetectCreditsTask.cs @@ -122,10 +122,8 @@ public Task ExecuteAsync(IProgress progress, CancellationToken cancellat return; } - // Increment totalProcessed by the number of episodes in this season that were actually analyzed - // (instead of just using the number of episodes in the current season). - var analyzed = AnalyzeSeason(episodes, cancellationToken); - Interlocked.Add(ref totalProcessed, analyzed); + AnalyzeSeason(episodes, cancellationToken); + Interlocked.Add(ref totalProcessed, episodes.Count); } catch (FingerprintException ex) { @@ -151,39 +149,49 @@ public Task ExecuteAsync(IProgress progress, CancellationToken cancellat } /// - /// Fingerprints all episodes in the provided season and stores the timestamps of all introductions. + /// Analyzes all episodes in the season for end credits. /// /// Episodes in this season. /// Cancellation token provided by the scheduled task. - /// Number of episodes from the provided season that were analyzed. - private int AnalyzeSeason( + private void AnalyzeSeason( ReadOnlyCollection episodes, CancellationToken cancellationToken) { - // Skip seasons with an insufficient number of episodes. - if (episodes.Count <= 1) + // Only analyze specials (season 0) if the user has opted in. + if (episodes[0].SeasonNumber == 0 && !Plugin.Instance!.Configuration.AnalyzeSeasonZero) { - return episodes.Count; + return; } - // Only analyze specials (season 0) if the user has opted in. - var first = episodes[0]; - if (first.SeasonNumber == 0 && !Plugin.Instance!.Configuration.AnalyzeSeasonZero) + // Analyze with Chromaprint first and fall back to the black frame detector + var analyzers = new IMediaFileAnalyzer[] + { + // TODO: FIXME: new ChromaprintAnalyzer(_loggerFactory.CreateLogger()), + new BlackFrameAnalyzer(_loggerFactory.CreateLogger()) + }; + + // Use each analyzer to find credits in all media files, removing successfully analyzed files + // from the queue. + var remaining = new ReadOnlyCollection(episodes); + foreach (var analyzer in analyzers) { - return 0; + remaining = AnalyzeFiles(remaining, analyzer, cancellationToken); } + } + private ReadOnlyCollection AnalyzeFiles( + ReadOnlyCollection episodes, + IMediaFileAnalyzer analyzer, + CancellationToken cancellationToken) + { _logger.LogInformation( - "Analyzing {Count} episodes from {Name} season {Season}", + "Analyzing {Count} episodes from {Name} season {Season} with {Analyzer}", episodes.Count, - first.SeriesName, - first.SeasonNumber); - - // Analyze the season with Chromaprint - var chromaprint = new ChromaprintAnalyzer(_loggerFactory.CreateLogger()); - chromaprint.AnalyzeMediaFiles(episodes, AnalysisMode.Credits, cancellationToken); + episodes[0].SeriesName, + episodes[0].SeasonNumber, + analyzer.GetType().Name); - return episodes.Count; + return analyzer.AnalyzeMediaFiles(episodes, AnalysisMode.Credits, cancellationToken); } ///