diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.cpp b/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.cpp index 64d492b1e..a84893e70 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.cpp +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.cpp @@ -36,12 +36,14 @@ struct StreamHistorySession::Data{ mutable SpinLock m_lock; std::chrono::seconds m_window; AudioChannelFormat m_audio_format; + bool m_has_video; std::shared_ptr m_current; Data(Logger& logger) : m_logger(logger) , m_window(GlobalSettings::instance().STREAM_HISTORY->HISTORY_SECONDS) , m_audio_format(AudioChannelFormat::NONE) + , m_has_video(false) {} }; @@ -53,11 +55,12 @@ StreamHistorySession::StreamHistorySession(Logger& logger) : AudioFloatStreamListener(1) , m_data(CONSTRUCT_TOKEN, logger) {} -void StreamHistorySession::start(AudioChannelFormat format){ +void StreamHistorySession::start(AudioChannelFormat format, bool has_video){ Data& data = *m_data; WriteSpinLock lg(data.m_lock); if (!data.m_current){ data.m_audio_format = format; + data.m_has_video = has_video; initialize(); } } @@ -118,7 +121,7 @@ void StreamHistorySession::on_samples(const float* samples, size_t frames){ data.m_current->on_samples(samples, frames); } } -void StreamHistorySession::on_frame(std::shared_ptr frame){ +void StreamHistorySession::on_frame(std::shared_ptr frame){ Data& data = *m_data; WriteSpinLock lg(data.m_lock); if (data.m_current){ @@ -128,12 +131,15 @@ void StreamHistorySession::on_frame(std::shared_ptr frame){ void StreamHistorySession::clear(){ +// cout << "clear()" << endl; + // Must call under lock. Data& data = *m_data; data.m_logger.log("Clearing stream history...", COLOR_ORANGE); data.m_current.reset(); - expected_samples_per_frame = 0; - data.m_audio_format = AudioChannelFormat::NONE; +// expected_samples_per_frame = 0; +// data.m_audio_format = AudioChannelFormat::NONE; +// data.m_has_video = false; } void StreamHistorySession::initialize(){ if (!GlobalSettings::instance().STREAM_HISTORY->enabled()){ @@ -143,22 +149,28 @@ void StreamHistorySession::initialize(){ // Must call under lock. Data& data = *m_data; data.m_logger.log("Starting stream history...", COLOR_ORANGE); + +// cout << "video = " << data.m_has_video << endl; + switch (data.m_audio_format){ case AudioChannelFormat::NONE: expected_samples_per_frame = 0; - data.m_current.reset(new StreamHistoryTracker(data.m_logger, 0, 0, data.m_window)); + data.m_current.reset(new StreamHistoryTracker(data.m_logger, data.m_window, 0, 0, data.m_has_video)); return; case AudioChannelFormat::MONO_48000: - data.m_current.reset(new StreamHistoryTracker(data.m_logger, 1, 48000, data.m_window)); + expected_samples_per_frame = 1; + data.m_current.reset(new StreamHistoryTracker(data.m_logger, data.m_window, 1, 48000, data.m_has_video)); return; case AudioChannelFormat::DUAL_44100: - data.m_current.reset(new StreamHistoryTracker(data.m_logger, 1, 44100, data.m_window)); + expected_samples_per_frame = 2; + data.m_current.reset(new StreamHistoryTracker(data.m_logger, data.m_window, 1, 44100, data.m_has_video)); return; case AudioChannelFormat::DUAL_48000: case AudioChannelFormat::MONO_96000: case AudioChannelFormat::INTERLEAVE_LR_96000: case AudioChannelFormat::INTERLEAVE_RL_96000: - data.m_current.reset(new StreamHistoryTracker(data.m_logger, 2, 48000, data.m_window)); + expected_samples_per_frame = 2; + data.m_current.reset(new StreamHistoryTracker(data.m_logger, data.m_window, 2, 48000, data.m_has_video)); return; default: throw InternalProgramError( @@ -168,32 +180,50 @@ void StreamHistorySession::initialize(){ } } void StreamHistorySession::pre_input_change(){ +// cout << "pre_input_change()" << endl; Data& data = *m_data; WriteSpinLock lg(data.m_lock); clear(); } void StreamHistorySession::post_input_change(const std::string& file, const AudioDeviceInfo& device, AudioChannelFormat format){ +// cout << "post_input_change()" << endl; Data& data = *m_data; WriteSpinLock lg(data.m_lock); - data.m_audio_format = format; + if (device){ + data.m_audio_format = format; + }else{ + data.m_audio_format = AudioChannelFormat::NONE; + } initialize(); } void StreamHistorySession::pre_shutdown(){ +// cout << "pre_shutdown()" << endl; Data& data = *m_data; WriteSpinLock lg(data.m_lock); clear(); } +void StreamHistorySession::post_shutdown(){ +// cout << "post_shutdown()" << endl; + Data& data = *m_data; + WriteSpinLock lg(data.m_lock); + data.m_has_video = false; + initialize(); +} void StreamHistorySession::post_new_source(const CameraInfo& device, Resolution resolution){ +// cout << "post_new_source()" << endl; Data& data = *m_data; WriteSpinLock lg(data.m_lock); + data.m_has_video = !device.device_name().empty(); initialize(); } void StreamHistorySession::pre_resolution_change(Resolution resolution){ +// cout << "pre_resolution_change()" << endl; Data& data = *m_data; WriteSpinLock lg(data.m_lock); clear(); } void StreamHistorySession::post_resolution_change(Resolution resolution){ +// cout << "post_resolution_change()" << endl; Data& data = *m_data; WriteSpinLock lg(data.m_lock); initialize(); diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.h b/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.h index 8e1536b34..dc1a2ce00 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.h +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.h @@ -26,18 +26,19 @@ class StreamHistorySession public: ~StreamHistorySession(); StreamHistorySession(Logger& logger); - void start(AudioChannelFormat format); + void start(AudioChannelFormat format, bool has_video); bool save(const std::string& filename) const; public: virtual void on_samples(const float* data, size_t frames) override; - virtual void on_frame(std::shared_ptr frame) override; + virtual void on_frame(std::shared_ptr frame) override; public: virtual void pre_input_change() override; virtual void post_input_change(const std::string& file, const AudioDeviceInfo& device, AudioChannelFormat format) override; virtual void pre_shutdown() override; + virtual void post_shutdown() override; virtual void post_new_source(const CameraInfo& device, Resolution resolution) override; virtual void pre_resolution_change(Resolution resolution) override; virtual void post_resolution_change(Resolution resolution) override; diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_ParallelStreams.h b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_ParallelStreams.h index 0fcca147e..1b053ff26 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_ParallelStreams.h +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_ParallelStreams.h @@ -39,14 +39,17 @@ class StreamHistoryTracker{ public: StreamHistoryTracker( Logger& logger, + std::chrono::seconds window, size_t audio_samples_per_frame, size_t audio_frames_per_second, - std::chrono::seconds window + bool has_video ) : m_logger(logger) - , m_window(window) + , m_window(window) // REMOVE +// , m_window(100) , m_audio_samples_per_frame(audio_samples_per_frame) , m_audio_frames_per_second(audio_frames_per_second) + , m_has_video(has_video) { update_streams(current_time()); } @@ -84,7 +87,7 @@ class StreamHistoryTracker{ } update_streams(now); } - void on_frame(std::shared_ptr frame){ + void on_frame(std::shared_ptr frame){ WallClock now = current_time(); SpinLockGuard lg(m_lock); for (auto& item : m_recordings){ @@ -140,9 +143,10 @@ class StreamHistoryTracker{ std::forward_as_tuple(start_time), std::forward_as_tuple(new StreamRecording( m_logger, std::chrono::milliseconds(500), + start_time, m_audio_samples_per_frame, m_audio_frames_per_second, - start_time + m_has_video )) ); } @@ -153,9 +157,10 @@ class StreamHistoryTracker{ private: Logger& m_logger; mutable SpinLock m_lock; - std::chrono::seconds m_window; + std::chrono::milliseconds m_window; const size_t m_audio_samples_per_frame; const size_t m_audio_frames_per_second; + const bool m_has_video; std::map> m_recordings; }; diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_RecordOnTheFly.h b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_RecordOnTheFly.h index f14d17e99..3b37cf564 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_RecordOnTheFly.h +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_RecordOnTheFly.h @@ -37,7 +37,7 @@ class RollingStream : public QIODevice{ public: ~RollingStream(){ waitForBytesWritten(-1); -// cout << "~RollingStream()" << endl; // REMOVE +// cout << "~RollingStream()" << endl; } RollingStream(){ setOpenMode(QIODeviceBase::WriteOnly); @@ -111,7 +111,7 @@ StreamHistoryTracker::~StreamHistoryTracker(){ pause(); }catch (...){} } -// cout << "~StreamHistoryTracker()" << endl; // REMOVE +// cout << "~StreamHistoryTracker()" << endl; } StreamHistoryTracker::StreamHistoryTracker( size_t audio_samples_per_frame, diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h index 364c43804..b481e432e 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h @@ -24,10 +24,9 @@ #include "CommonFramework/VideoPipeline/Backends/VideoFrameQt.h" -// REMOVE -#include -using std::cout; -using std::endl; +//#include +//using std::cout; +//using std::endl; namespace PokemonAutomation{ @@ -116,7 +115,7 @@ void StreamHistoryTracker::on_frame(std::shared_ptr frame){ // TODO: Find a more efficient way to buffer the frames. // It takes almost 10GB of memory to store 30 seconds of QVideoFrames // due to them caching uncompressed bitmaps. -// return; // REMOVE +// return; // TODO WriteSpinLock lg(m_lock); // cout << "on_frame() = " << m_frames.size() << endl; @@ -296,7 +295,7 @@ bool StreamHistoryTracker::save(Logger& logger, const std::string& filename) con recorder.stop(); logger.log("Done saving stream history...", COLOR_BLUE); - cout << recorder.duration() << endl; // REMOVE +// cout << recorder.duration() << endl; // }); diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamRecorder.cpp b/SerialPrograms/Source/CommonFramework/Recording/StreamRecorder.cpp index 966e428c8..74fa36e2b 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamRecorder.cpp +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamRecorder.cpp @@ -38,34 +38,53 @@ namespace PokemonAutomation{ StreamRecording::StreamRecording( Logger& logger, std::chrono::milliseconds buffer_limit, + WallClock start_time, size_t audio_samples_per_frame, size_t audio_frames_per_second, - WallClock start_time + bool has_video ) : m_logger(logger) , m_buffer_limit(buffer_limit) - , m_audio_samples_per_frame(audio_samples_per_frame) , m_start_time(start_time) + , m_audio_samples_per_frame(audio_samples_per_frame) + , m_has_video(has_video) , m_filename(GlobalSettings::instance().TEMP_FOLDER) - , m_state(State::STARTING) + , m_stopping(false) , m_last_drop(current_time()) + , m_last_frame_time(std::numeric_limits::min()) { - m_audio_format.setChannelCount((int)audio_samples_per_frame); - m_audio_format.setChannelConfig(audio_samples_per_frame == 1 ? QAudioFormat::ChannelConfigMono : QAudioFormat::ChannelConfigStereo); - m_audio_format.setSampleRate((int)audio_frames_per_second); - m_audio_format.setSampleFormat(QAudioFormat::Float); + logger.log( + "StreamRecording: Audio = " + std::to_string(audio_samples_per_frame) + + ", Video = " + std::to_string(has_video) + ); +// cout << "audio = " << audio_samples_per_frame << ", video = " << has_video << endl; + + if (audio_samples_per_frame == 0 && !has_video){ + return; + } + + if (audio_samples_per_frame > 0){ + m_audio_format.setChannelCount((int)audio_samples_per_frame); + m_audio_format.setChannelConfig(audio_samples_per_frame == 1 ? QAudioFormat::ChannelConfigMono : QAudioFormat::ChannelConfigStereo); + m_audio_format.setSampleRate((int)audio_frames_per_second); + m_audio_format.setSampleFormat(QAudioFormat::Float); + } #ifndef PA_STREAM_HISTORY_LOCAL_BUFFER QDir().mkdir(QString::fromStdString(m_filename)); #endif - m_filename += now_to_filestring() + ".mp4"; + if (has_video){ + m_filename += now_to_filestring() + ".mp4"; + }else{ + m_filename += now_to_filestring() + ".m4a"; + } start(); } StreamRecording::~StreamRecording(){ { std::lock_guard lg(m_lock); - m_state = State::STOPPING; + m_stopping = true; // cout << "signalling: ~StreamRecording()" << endl; m_cv.notify_all(); } @@ -79,7 +98,7 @@ StreamRecording::~StreamRecording(){ bool StreamRecording::stop_and_save(const std::string& filename){ { std::lock_guard lg(m_lock); - m_state = State::STOPPING; + m_stopping = true; // cout << "signalling: stop_and_save()" << endl; m_cv.notify_all(); } @@ -98,67 +117,103 @@ bool StreamRecording::stop_and_save(const std::string& filename){ void StreamRecording::push_samples(WallClock timestamp, const float* data, size_t frames){ WallClock now = current_time(); - if (now < m_start_time){ + if (m_audio_samples_per_frame == 0 || now < m_start_time){ return; } WallClock threshold = timestamp - m_buffer_limit; std::lock_guard lg(m_lock); - if (m_state != State::ACTIVE){ - return; - } - if (m_buffered_audio.empty() || m_buffered_audio.front()->timestamp > threshold){ - m_buffered_audio.emplace_back(std::make_shared( - timestamp, data, frames * m_audio_samples_per_frame - )); - m_cv.notify_all(); + if (m_stopping){ return; } - if (now - m_last_drop > std::chrono::seconds(5)){ - m_last_drop = now; - m_logger.log("Unable to keep up with audio recording. Dropping samples.", COLOR_RED); - } + + do{ + if (m_buffered_audio.empty()){ + break; + } + + // Too much has been buffered. Drop the block. + if (m_buffered_audio.front().timestamp < threshold){ + // Throttle the prints. + if (now - m_last_drop > std::chrono::seconds(5)){ + m_last_drop = now; + m_logger.log("Unable to keep up with audio recording. Dropping samples.", COLOR_RED); + } + return; + } + + }while (false); + + // Enqueue the sample block. + m_buffered_audio.emplace_back( + timestamp, + data, frames * m_audio_samples_per_frame + ); + m_cv.notify_all(); } -void StreamRecording::push_frame(std::shared_ptr frame){ +void StreamRecording::push_frame(std::shared_ptr frame){ WallClock now = current_time(); - if (now < m_start_time){ + if (!m_has_video || now < m_start_time){ return; } -// cout << "push_frame()" << endl; +// cout << "push_frame(): " << frame->frame.startTime() << " - " << frame->frame.endTime() << endl; WallClock threshold = frame->timestamp - m_buffer_limit; std::lock_guard lg(m_lock); - if (m_state != State::ACTIVE){ + if (m_stopping){ return; } - if (m_buffered_frames.empty() || m_buffered_frames.front()->timestamp > threshold){ - m_buffered_frames.emplace_back(std::move(frame)); - m_cv.notify_all(); - return; - } - if (now - m_last_drop > std::chrono::seconds(5)){ - m_last_drop = now; - m_logger.log("Unable to keep up with video recording. Dropping samples.", COLOR_RED); - } + + qint64 frame_time = frame->frame.startTime(); + do{ + if (m_buffered_frames.empty()){ + break; + } + + // Too much has been buffered. Drop the frame. + if (m_buffered_frames.front()->timestamp < threshold){ + // Throttle the prints. + if (now - m_last_drop > std::chrono::seconds(5)){ + m_last_drop = now; + m_logger.log("Unable to keep up with video recording. Dropping samples.", COLOR_RED); + } + return; + } + + // Non-increasing timestamp. Drop possible duplicate frame. + if (frame_time <= m_last_frame_time){ + return; + } + }while (false); + + // Enqueue the frame. + m_buffered_frames.emplace_back(std::move(frame)); + m_last_frame_time = frame_time; + m_cv.notify_all(); } -void StreamRecording::internal_run(){ - QAudioBufferInput audio_input; - QVideoFrameInput video_input; - m_audio_input = &audio_input; - m_video_input = &video_input; +std::unique_ptr StreamRecording::initialize_audio(){ + std::unique_ptr ret(new QAudioBufferInput()); + m_audio_input = ret.get(); + m_session->setAudioBufferInput(m_audio_input); - QMediaCaptureSession session; - QMediaRecorder recorder; - session.setAudioBufferInput(&audio_input); - session.setVideoFrameInput(&video_input); - session.setRecorder(&recorder); - recorder.setMediaFormat(QMediaFormat::MPEG4); -// recorder.setQuality(QMediaRecorder::NormalQuality); -// recorder.setQuality(QMediaRecorder::LowQuality); -// recorder.setQuality(QMediaRecorder::VeryLowQuality); + connect( + m_audio_input, &QAudioBufferInput::readyToSendAudioBuffer, + m_recorder, [this](){ + std::lock_guard lg(m_lock); + m_cv.notify_all(); + }, + Qt::DirectConnection + ); + + return ret; +} +std::unique_ptr StreamRecording::initialize_video(){ + std::unique_ptr ret(new QVideoFrameInput()); + m_video_input = ret.get(); + m_session->setVideoFrameInput(m_video_input); const StreamHistoryOption& settings = GlobalSettings::instance().STREAM_HISTORY; @@ -166,10 +221,10 @@ void StreamRecording::internal_run(){ case StreamHistoryOption::Resolution::MATCH_INPUT: break; case StreamHistoryOption::Resolution::FORCE_720p: - recorder.setVideoResolution(1280, 720); + m_recorder->setVideoResolution(1280, 720); break; case StreamHistoryOption::Resolution::FORCE_1080p: - recorder.setVideoResolution(1920, 1080); + m_recorder->setVideoResolution(1920, 1080); break; } @@ -177,89 +232,104 @@ void StreamRecording::internal_run(){ case StreamHistoryOption::EncodingMode::FIXED_QUALITY: switch (settings.VIDEO_QUALITY){ case StreamHistoryOption::VideoQuality::VERY_LOW: - recorder.setQuality(QMediaRecorder::VeryLowQuality); + m_recorder->setQuality(QMediaRecorder::VeryLowQuality); break; case StreamHistoryOption::VideoQuality::LOW: - recorder.setQuality(QMediaRecorder::LowQuality); + m_recorder->setQuality(QMediaRecorder::LowQuality); break; case StreamHistoryOption::VideoQuality::NORMAL: - recorder.setQuality(QMediaRecorder::NormalQuality); + m_recorder->setQuality(QMediaRecorder::NormalQuality); break; case StreamHistoryOption::VideoQuality::HIGH: - recorder.setQuality(QMediaRecorder::HighQuality); + m_recorder->setQuality(QMediaRecorder::HighQuality); break; case StreamHistoryOption::VideoQuality::VERY_HIGH: - recorder.setQuality(QMediaRecorder::VeryHighQuality); + m_recorder->setQuality(QMediaRecorder::VeryHighQuality); break; } break; case StreamHistoryOption::EncodingMode::FIXED_BITRATE: - recorder.setVideoBitRate(settings.VIDEO_BITRATE * 1000); - recorder.setEncodingMode(QMediaRecorder::AverageBitRateEncoding); + m_recorder->setVideoBitRate(settings.VIDEO_BITRATE * 1000); + m_recorder->setEncodingMode(QMediaRecorder::AverageBitRateEncoding); break; } - -#ifdef PA_STREAM_HISTORY_LOCAL_BUFFER - recorder.setOutputDevice(&m_write_buffer); -#else - QFileInfo file(QString::fromStdString(m_filename)); - recorder.setOutputLocation( - QUrl::fromLocalFile(file.absoluteFilePath()) - ); -#endif - -// cout << "Encoding Mode = " << (int)recorder.encodingMode() << endl; -// cout << "Bit Rate = " << (int)recorder.videoBitRate() << endl; - connect( - &audio_input, &QAudioBufferInput::readyToSendAudioBuffer, - &recorder, [this](){ + m_video_input, &QVideoFrameInput::readyToSendVideoFrame, + m_recorder, [this](){ std::lock_guard lg(m_lock); m_cv.notify_all(); }, Qt::DirectConnection ); connect( - &video_input, &QVideoFrameInput::readyToSendVideoFrame, - &recorder, [this](){ - std::lock_guard lg(m_lock); - m_cv.notify_all(); - }, - Qt::DirectConnection - ); - connect( - &recorder, &QMediaRecorder::recorderStateChanged, - &recorder, [this](QMediaRecorder::RecorderState state){ + m_recorder, &QMediaRecorder::recorderStateChanged, + m_recorder, [this](QMediaRecorder::RecorderState state){ if (state == QMediaRecorder::StoppedState){ std::lock_guard lg(m_lock); // cout << "signalling: StoppedState" << endl; - m_state = State::STOPPING; + m_stopping = true; m_cv.notify_all(); } }, Qt::DirectConnection ); + return ret; +} + +void StreamRecording::internal_run(){ + QMediaCaptureSession session; + QMediaRecorder recorder; + m_session = &session; + m_recorder = &recorder; + m_session->setRecorder(m_recorder); + + std::unique_ptr audio; + std::unique_ptr video; + + // Only initialize the streams we intend to use. + if (m_audio_samples_per_frame > 0){ + audio = initialize_audio(); + } + if (m_has_video){ + video = initialize_video(); + } + + m_recorder->setMediaFormat(QMediaFormat::MPEG4); + + + +#ifdef PA_STREAM_HISTORY_LOCAL_BUFFER + m_recorder->setOutputDevice(&m_write_buffer); +#else + QFileInfo file(QString::fromStdString(m_filename)); + m_recorder->setOutputLocation( + QUrl::fromLocalFile(file.absoluteFilePath()) + ); +#endif + +// cout << "Encoding Mode = " << (int)recorder.encodingMode() << endl; +// cout << "Bit Rate = " << (int)recorder.videoBitRate() << endl; + + // cout << "starting recording" << endl; - recorder.record(); + m_recorder->record(); - std::shared_ptr current_audio; - std::shared_ptr current_frame; + AudioBlock current_audio; + std::shared_ptr current_frame; QAudioBuffer audio_buffer; while (true){ -// cout << "recording loop" << endl; QCoreApplication::processEvents(); { std::unique_lock lg(m_lock); - if (m_state == State::STOPPING){ + if (m_stopping){ break; } - m_state = State::ACTIVE; - if (!current_audio && !m_buffered_audio.empty()){ + if (!current_audio.is_valid() && !m_buffered_audio.empty()){ current_audio = std::move(m_buffered_audio.front()); m_buffered_audio.pop_front(); } @@ -268,7 +338,7 @@ void StreamRecording::internal_run(){ m_buffered_frames.pop_front(); } - if (!current_audio && !current_frame){ + if (!current_audio.is_valid() && !current_frame){ // cout << "sleeping 0..." << endl; m_cv.wait(lg); // cout << "waking 0..." << endl; @@ -277,27 +347,29 @@ void StreamRecording::internal_run(){ bool progress_made = false; - if (current_audio){ + if (current_audio.is_valid()){ if (!audio_buffer.isValid()){ - const std::vector& samples = current_audio->samples; + const std::vector& samples = current_audio.samples; QByteArray bytes((const char*)samples.data(), samples.size() * sizeof(float)); audio_buffer = QAudioBuffer(bytes, m_audio_format); } - if (audio_buffer.isValid() && audio_input.sendAudioBuffer(audio_buffer)){ - current_audio.reset(); + if (audio_buffer.isValid() && m_audio_input->sendAudioBuffer(audio_buffer)){ + current_audio.clear(); audio_buffer = QAudioBuffer(); progress_made = true; } } - - if (current_frame && video_input.sendVideoFrame(current_frame->frame)){ +// cout << "Before: " << m_video_input << endl; + if (current_frame && m_video_input->sendVideoFrame(current_frame->frame)){ +// cout << "push frame: " << current_frame->frame.startTime() << endl; current_frame.reset(); progress_made = true; } +// cout << "After: " << m_video_input << endl; if (!progress_made){ std::unique_lock lg(m_lock); - if (m_state != State::ACTIVE){ + if (m_stopping){ break; } // cout << "sleeping 1..." << endl; @@ -306,11 +378,11 @@ void StreamRecording::internal_run(){ } } - recorder.stop(); + m_recorder->stop(); // cout << "recorder.stop()" << endl; - while (recorder.recorderState() != QMediaRecorder::StoppedState){ + while (m_recorder->recorderState() != QMediaRecorder::StoppedState){ // cout << "StreamHistoryTracker: process" << endl; QCoreApplication::processEvents(); pause(); @@ -325,7 +397,7 @@ void StreamRecording::run(){ }catch (...){ m_logger.log("Exception thrown out of stream recorder...", COLOR_RED); std::lock_guard lg(m_lock); - m_state = State::STOPPING; + m_stopping = true; } } diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamRecorder.h b/SerialPrograms/Source/CommonFramework/Recording/StreamRecorder.h index 7ce918d19..8b44d4118 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamRecorder.h +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamRecorder.h @@ -15,32 +15,44 @@ #include #include "Common/Cpp/Time.h" #include "Common/Cpp/AbstractLogger.h" +#include "CommonFramework/VideoPipeline/Backends/VideoFrameQt.h" -// REMOVE -#include -using std::cout; -using std::endl; +//#include +//using std::cout; +//using std::endl; class QAudioBufferInput; class QVideoFrameInput; +class QMediaCaptureSession; +class QMediaRecorder; namespace PokemonAutomation{ -class VideoFrame; - struct AudioBlock{ WallClock timestamp; std::vector samples; + AudioBlock(AudioBlock&&) = default; + AudioBlock& operator=(AudioBlock&&) = default; AudioBlock(const AudioBlock&) = delete; void operator=(const AudioBlock&) = delete; + AudioBlock() + : timestamp(WallClock::min()) + {} AudioBlock(WallClock p_timestamp, const float* p_samples, size_t p_count) : timestamp(p_timestamp) , samples(p_samples, p_samples + p_count) {} + + void clear(){ + samples.clear(); + } + bool is_valid() const{ + return !samples.empty(); + } }; @@ -92,45 +104,51 @@ class StreamRecording : public QThread{ StreamRecording( Logger& logger, std::chrono::milliseconds buffer_limit, + WallClock start_time, size_t audio_samples_per_frame, size_t audio_frames_per_second, - WallClock start_time + bool has_video ); ~StreamRecording(); void push_samples(WallClock timestamp, const float* data, size_t frames); - void push_frame(std::shared_ptr frame); + void push_frame(std::shared_ptr frame); bool stop_and_save(const std::string& filename); private: + std::unique_ptr initialize_audio(); + std::unique_ptr initialize_video(); void internal_run(); virtual void run() override; private: Logger& m_logger; const std::chrono::milliseconds m_buffer_limit; + const WallClock m_start_time; const size_t m_audio_samples_per_frame; + const bool m_has_video; + QAudioFormat m_audio_format; - const WallClock m_start_time; std::string m_filename; std::mutex m_lock; std::condition_variable m_cv; - enum class State{ - STARTING, - ACTIVE, - STOPPING, - }; - - State m_state; + bool m_stopping = false; WallClock m_last_drop; - std::deque> m_buffered_audio; - std::deque> m_buffered_frames; - QAudioBufferInput* m_audio_input; - QVideoFrameInput* m_video_input; + + std::deque m_buffered_audio; + + qint64 m_last_frame_time; + std::deque> m_buffered_frames; + WriteBuffer m_write_buffer; + + QAudioBufferInput* m_audio_input = nullptr; + QVideoFrameInput* m_video_input = nullptr; + QMediaCaptureSession* m_session = nullptr; + QMediaRecorder* m_recorder = nullptr; }; diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt5.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt5.cpp index d28bd7f43..eeef1e59e 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt5.cpp +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt5.cpp @@ -314,13 +314,19 @@ void CameraSession::shutdown(){ m_last_frame_timestamp = current_time(); m_last_frame_seqnum++; - SpinLockGuard lg(m_frame_lock); + { + SpinLockGuard lg(m_frame_lock); - m_last_frame = QVideoFrame(); - m_last_frame_timestamp = current_time(); - m_last_frame_seqnum++; + m_last_frame = QVideoFrame(); + m_last_frame_timestamp = current_time(); + m_last_frame_seqnum++; + + m_last_image_seqnum = m_last_frame_seqnum; + } - m_last_image_seqnum = m_last_frame_seqnum; + for (StateListener* listener : m_state_listeners){ + listener->post_shutdown(); + } } void CameraSession::startup(){ if (!m_device){ diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.cpp index d8843dfe2..187531979 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.cpp +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.cpp @@ -332,16 +332,21 @@ void CameraSession::shutdown(){ m_resolution_map.clear(); m_formats.clear(); - WriteSpinLock lg(m_frame_lock); + { + WriteSpinLock lg(m_frame_lock); - m_last_frame = QVideoFrame(); - m_last_frame_timestamp = current_time(); - m_last_frame_seqnum++; + m_last_frame = QVideoFrame(); + m_last_frame_timestamp = current_time(); + m_last_frame_seqnum++; - m_last_image = QImage(); - m_last_image_timestamp = m_last_frame_timestamp; - m_last_image_seqnum = m_last_frame_seqnum; + m_last_image = QImage(); + m_last_image_timestamp = m_last_frame_timestamp; + m_last_image_seqnum = m_last_frame_seqnum; + } + for (StateListener* listener : m_state_listeners){ + listener->post_shutdown(); + } } void CameraSession::startup(){ if (!m_device){ @@ -397,6 +402,7 @@ void CameraSession::startup(){ desired_format = m_resolution_map.rbegin()->second; } // cout << "CameraSession::m_resolutions = " << m_resolutions.size() << endl; + cout << "desired_format = " << desired_format->minFrameRate() << " - " << desired_format->maxFrameRate() << endl; // REMOVE QSize size = desired_format->resolution(); m_resolution = Resolution(size.width(), size.height()); diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.cpp index bed54793c..62d488755 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.cpp +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.cpp @@ -257,16 +257,21 @@ void CameraSession::shutdown(){ m_resolution_map.clear(); m_formats.clear(); - SpinLockGuard lg(m_frame_lock); + { + SpinLockGuard lg(m_frame_lock); - m_last_frame = QVideoFrame(); - m_last_frame_timestamp = current_time(); - m_last_frame_seqnum++; + m_last_frame = QVideoFrame(); + m_last_frame_timestamp = current_time(); + m_last_frame_seqnum++; - m_last_image = QImage(); - m_last_image_timestamp = m_last_frame_timestamp; - m_last_image_seqnum = m_last_frame_seqnum; + m_last_image = QImage(); + m_last_image_timestamp = m_last_frame_timestamp; + m_last_image_seqnum = m_last_frame_seqnum; + } + for (StateListener* listener : m_state_listeners){ + listener->post_shutdown(); + } } void CameraSession::startup(){ if (!m_device){ diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/VideoFrameQt.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/VideoFrameQt.h index 3c3dd11e6..7bc197f76 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/VideoFrameQt.h +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/VideoFrameQt.h @@ -18,13 +18,25 @@ class VideoFrame{ WallClock timestamp; QVideoFrame frame; + VideoFrame(VideoFrame&&) = default; + VideoFrame& operator=(VideoFrame&&) = default; VideoFrame(const VideoFrame&) = delete; void operator=(const VideoFrame&) = delete; + VideoFrame() + : timestamp(WallClock::min()) + {} VideoFrame(WallClock p_timestamp, QVideoFrame p_frame) : timestamp(p_timestamp) , frame(std::move(p_frame)) {} + + void clear(){ + frame = QVideoFrame(); + } + bool is_valid() const{ + return frame.isValid(); + } }; diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/CameraSession.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/CameraSession.h index f57d3a82d..ddf740704 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/CameraSession.h +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/CameraSession.h @@ -35,6 +35,7 @@ class CameraSession : public VideoFeed{ // Sent before the camera shuts down. Listeners should drop their // references to the internal camera implementation before returning. virtual void pre_shutdown(){} + virtual void post_shutdown(){} // Sent before/after a new camera goes up. // virtual void pre_new_source(const CameraInfo& device, Resolution resolution){} diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoFeed.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoFeed.h index b62fbb923..9ab611cc0 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoFeed.h +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoFeed.h @@ -51,7 +51,7 @@ struct VideoSnapshot{ // implementations. Unsupported implementations will never fire this callback. class VideoFrame; struct VideoFrameListener{ - virtual void on_frame(std::shared_ptr frame) = 0; + virtual void on_frame(std::shared_ptr frame) = 0; }; diff --git a/SerialPrograms/Source/NintendoSwitch/Framework/NintendoSwitch_SwitchSystemSession.cpp b/SerialPrograms/Source/NintendoSwitch/Framework/NintendoSwitch_SwitchSystemSession.cpp index 785c67c57..a53911efd 100644 --- a/SerialPrograms/Source/NintendoSwitch/Framework/NintendoSwitch_SwitchSystemSession.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Framework/NintendoSwitch_SwitchSystemSession.cpp @@ -59,7 +59,7 @@ SwitchSystemSession::SwitchSystemSession( m_overlay.add_stat(*m_cpu_utilization); m_overlay.add_stat(*m_main_thread_utilization); - m_history.start(m_audio.input_format()); + m_history.start(m_audio.input_format(), !option.m_camera.info.device_name().empty()); m_audio.add_state_listener(m_history); m_audio.add_stream_listener(m_history);