From ac53418c4cce340abac90c0cd28ff2560683ebbb Mon Sep 17 00:00:00 2001 From: Alexander Yee Date: Sat, 7 Dec 2024 21:09:07 -0800 Subject: [PATCH] Add stream history for real this time. (still buggy though) --- SerialPrograms/CMakeLists.txt | 3 + SerialPrograms/SerialPrograms.pro | 2 + .../AudioPipeline/AudioStream.h | 1 + .../ErrorReports/ErrorReports.cpp | 13 +- .../CommonFramework/GlobalSettingsPanel.cpp | 23 +- .../Source/CommonFramework/Globals.cpp | 6 +- .../Source/CommonFramework/Globals.h | 2 + .../Logging/FileWindowLogger.cpp | 4 +- .../Recording/StreamHistorySession.cpp | 18 +- .../Recording/StreamHistoryTracker_Null.h | 10 +- .../StreamHistoryTracker_ParallelStreams.h | 171 +++++++++++ .../Recording/StreamRecorder.cpp | 287 ++++++++++++++++++ .../Recording/StreamRecorder.h | 139 +++++++++ .../UI/NintendoSwitch_CommandRow.cpp | 12 + .../Framework/UI/NintendoSwitch_CommandRow.h | 2 + .../UI/NintendoSwitch_SwitchSystemWidget.cpp | 10 + 16 files changed, 673 insertions(+), 30 deletions(-) create mode 100644 SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_ParallelStreams.h create mode 100644 SerialPrograms/Source/CommonFramework/Recording/StreamRecorder.cpp create mode 100644 SerialPrograms/Source/CommonFramework/Recording/StreamRecorder.h diff --git a/SerialPrograms/CMakeLists.txt b/SerialPrograms/CMakeLists.txt index 06f0fcb4e..e74324364 100644 --- a/SerialPrograms/CMakeLists.txt +++ b/SerialPrograms/CMakeLists.txt @@ -512,8 +512,11 @@ file(GLOB MAIN_SOURCES Source/CommonFramework/Recording/StreamHistorySession.cpp Source/CommonFramework/Recording/StreamHistorySession.h Source/CommonFramework/Recording/StreamHistoryTracker_Null.h + Source/CommonFramework/Recording/StreamHistoryTracker_ParallelStreams.h Source/CommonFramework/Recording/StreamHistoryTracker_RecordOnTheFly.h Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h + Source/CommonFramework/Recording/StreamRecorder.cpp + Source/CommonFramework/Recording/StreamRecorder.h Source/CommonFramework/Resources/SpriteDatabase.cpp Source/CommonFramework/Resources/SpriteDatabase.h Source/CommonFramework/SetupSettings.cpp diff --git a/SerialPrograms/SerialPrograms.pro b/SerialPrograms/SerialPrograms.pro index 436964b5b..10a05008a 100644 --- a/SerialPrograms/SerialPrograms.pro +++ b/SerialPrograms/SerialPrograms.pro @@ -272,6 +272,7 @@ SOURCES += \ Source/CommonFramework/PersistentSettings.cpp \ Source/CommonFramework/ProgramSession.cpp \ Source/CommonFramework/Recording/StreamHistorySession.cpp \ + Source/CommonFramework/Recording/StreamRecorder.cpp \ Source/CommonFramework/Resources/SpriteDatabase.cpp \ Source/CommonFramework/SetupSettings.cpp \ Source/CommonFramework/Tools/BlackBorderCheck.cpp \ @@ -1365,6 +1366,7 @@ HEADERS += \ Source/CommonFramework/Recording/StreamHistorySession.h \ Source/CommonFramework/Recording/StreamHistoryTracker_RecordOnTheFly.h \ Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h \ + Source/CommonFramework/Recording/StreamRecorder.h \ Source/CommonFramework/Resources/SpriteDatabase.h \ Source/CommonFramework/SetupSettings.h \ Source/CommonFramework/Tools/BlackBorderCheck.h \ diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioStream.h b/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioStream.h index 28c2675d2..e5f79ab8f 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioStream.h +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioStream.h @@ -7,6 +7,7 @@ #ifndef PokemonAutomation_AudioPipeline_AudioSourceReader_H #define PokemonAutomation_AudioPipeline_AudioSourceReader_H +#include "Common/Cpp/Time.h" #include "Common/Cpp/StreamConverters.h" #include "Common/Cpp/Containers/AlignedVector.h" #include "AudioInfo.h" diff --git a/SerialPrograms/Source/CommonFramework/ErrorReports/ErrorReports.cpp b/SerialPrograms/Source/CommonFramework/ErrorReports/ErrorReports.cpp index 28e7bcbd1..fbfd0cb33 100644 --- a/SerialPrograms/Source/CommonFramework/ErrorReports/ErrorReports.cpp +++ b/SerialPrograms/Source/CommonFramework/ErrorReports/ErrorReports.cpp @@ -57,7 +57,7 @@ ErrorReportOption::ErrorReportOption() true ) , VIDEO( - "Include Video:
Include a video leading up to the error. (if possible)", + "Include Video:
Include a video leading up to the error. (if available)", LockMode::UNLOCK_WHILE_RUNNING, true ) @@ -80,9 +80,7 @@ ErrorReportOption::ErrorReportOption() PA_ADD_STATIC(DESCRIPTION); PA_ADD_OPTION(SEND_MODE); PA_ADD_OPTION(SCREENSHOT); - if (PreloadSettings::instance().DEVELOPER_MODE){ - PA_ADD_OPTION(VIDEO); - } + PA_ADD_OPTION(VIDEO); PA_ADD_OPTION(LOGS); PA_ADD_OPTION(DUMPS); if (PreloadSettings::instance().DEVELOPER_MODE){ @@ -179,7 +177,7 @@ SendableErrorReport::SendableErrorReport(std::string directory) const std::string* image_name = obj.get_string("Screenshot"); if (image_name){ try{ - m_image_owner = ImageRGB32(*image_name); + m_image_owner = ImageRGB32(m_directory + *image_name); m_image = m_image_owner; }catch (FileException&){} } @@ -238,9 +236,8 @@ void SendableErrorReport::save(Logger* logger) const{ report["Messages"] = std::move(messages); } if (m_image){ - std::string image_name = m_directory + "Screenshot.png"; - if (m_image.save(image_name)){ - report["Screenshot"] = std::move(image_name); + if (m_image.save(m_directory + "Screenshot.png")){ + report["Screenshot"] = "Screenshot.png"; } } if (!m_video_name.empty()){ diff --git a/SerialPrograms/Source/CommonFramework/GlobalSettingsPanel.cpp b/SerialPrograms/Source/CommonFramework/GlobalSettingsPanel.cpp index f095d6adf..d131632db 100644 --- a/SerialPrograms/Source/CommonFramework/GlobalSettingsPanel.cpp +++ b/SerialPrograms/Source/CommonFramework/GlobalSettingsPanel.cpp @@ -52,11 +52,18 @@ ResolutionOption::ResolutionOption( } StreamHistoryOption::StreamHistoryOption() - : GroupOption("Stream History", LockMode::LOCK_WHILE_RUNNING, true, false) + : GroupOption( + "Stream History", + LockMode::LOCK_WHILE_RUNNING, + true, + IS_BETA_VERSION + ) , DESCRIPTION( - "Keep a record of this many seconds of video+audio. This will allow video capture for unexpected events.
" - "Warning (Developer Only): The current implementation is inefficient and requires " - "10 GB of ram to store just 30 seconds of video. This feature is still a work-in-progress." + "Keep a record of this many seconds of video+audio. This will allow " + "video capture for unexpected events.
" + "Warning: The current implementation is inefficient " + "and may write a lot of data to disk. " + "This feature is still a work-in-progress." "" ) , VIDEO_HISTORY_SECONDS( @@ -285,10 +292,10 @@ GlobalSettings::GlobalSettings() #if QT_VERSION_MAJOR == 5 PA_ADD_OPTION(ENABLE_FRAME_SCREENSHOTS); #endif -#if QT_VERSION_MAJOR >= 6 - if (PreloadSettings::instance().DEVELOPER_MODE){ - PA_ADD_OPTION(STREAM_HISTORY); - } +#if (QT_VERSION_MAJOR == 6) && (QT_VERSION_MINOR >= 8) + PA_ADD_OPTION(STREAM_HISTORY); +#else + STREAM_HISTORY.set_enabled(false); #endif PA_ADD_OPTION(AUTO_RESET_AUDIO_SECONDS); diff --git a/SerialPrograms/Source/CommonFramework/Globals.cpp b/SerialPrograms/Source/CommonFramework/Globals.cpp index ddcb62d55..07f5ed576 100644 --- a/SerialPrograms/Source/CommonFramework/Globals.cpp +++ b/SerialPrograms/Source/CommonFramework/Globals.cpp @@ -25,7 +25,7 @@ namespace PokemonAutomation{ const bool IS_BETA_VERSION = true; const int PROGRAM_VERSION_MAJOR = 0; const int PROGRAM_VERSION_MINOR = 50; -const int PROGRAM_VERSION_PATCH = 9; +const int PROGRAM_VERSION_PATCH = 10; const std::string PROGRAM_VERSION_BASE = "v" + std::to_string(PROGRAM_VERSION_MAJOR) + @@ -68,6 +68,10 @@ const std::string COMPILER_VERSION = "Unknown Compiler"; #endif + +const size_t LOG_HISTORY_LINES = 2000; + + namespace{ QString get_application_base_dir_path(){ diff --git a/SerialPrograms/Source/CommonFramework/Globals.h b/SerialPrograms/Source/CommonFramework/Globals.h index 855a6b91b..139d8e6d5 100644 --- a/SerialPrograms/Source/CommonFramework/Globals.h +++ b/SerialPrograms/Source/CommonFramework/Globals.h @@ -33,6 +33,8 @@ extern const std::string COMPILER_VERSION; const auto SERIAL_REFRESH_RATE = std::chrono::milliseconds(1000); +extern const size_t LOG_HISTORY_LINES; + // Folder path (end with "/") to hold program setting files. const std::string& SETTINGS_PATH(); diff --git a/SerialPrograms/Source/CommonFramework/Logging/FileWindowLogger.cpp b/SerialPrograms/Source/CommonFramework/Logging/FileWindowLogger.cpp index 0c0b37fff..4d4b4bd39 100644 --- a/SerialPrograms/Source/CommonFramework/Logging/FileWindowLogger.cpp +++ b/SerialPrograms/Source/CommonFramework/Logging/FileWindowLogger.cpp @@ -7,7 +7,7 @@ #include #include #include -#include +#include "CommonFramework/Globals.h" #include "CommonFramework/Windows/DpiScaler.h" #include "CommonFramework/Windows/WindowTracker.h" #include "FileWindowLogger.h" @@ -46,7 +46,7 @@ FileWindowLogger::~FileWindowLogger(){ } FileWindowLogger::FileWindowLogger(const std::string& path) : m_file(QString::fromStdString(path)) - , m_max_queue_size(10000) + , m_max_queue_size(LOG_HISTORY_LINES) , m_stopping(false) , m_thread(&FileWindowLogger::thread_loop, this) { diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.cpp b/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.cpp index fa99a2c0f..d98b5c1e8 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.cpp +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.cpp @@ -11,8 +11,9 @@ #include "CommonFramework/VideoPipeline/Backends/VideoFrameQt.h" #if (QT_VERSION_MAJOR == 6) && (QT_VERSION_MINOR >= 8) -#include "StreamHistoryTracker_SaveFrames.h" +//#include "StreamHistoryTracker_SaveFrames.h" //#include "StreamHistoryTracker_RecordOnTheFly.h" +#include "StreamHistoryTracker_ParallelStreams.h" #else #include "StreamHistoryTracker_Null.h" #endif @@ -61,7 +62,7 @@ void StreamHistorySession::start(AudioChannelFormat format){ class HistorySaverThread : public QThread{ public: - HistorySaverThread(Logger& logger, const StreamHistoryTracker& tracker, const std::string& filename) + HistorySaverThread(Logger& logger, StreamHistoryTracker& tracker, const std::string& filename) : m_logger(logger) , m_tracker(tracker) , m_filename(filename) @@ -78,12 +79,13 @@ class HistorySaverThread : public QThread{ return m_success; } virtual void run() override{ - m_success = m_tracker.save(m_logger, m_filename); +// m_success = m_tracker.save(m_logger, m_filename); + m_success = m_tracker.save(m_filename); } private: Logger& m_logger; - const StreamHistoryTracker& m_tracker; + StreamHistoryTracker& m_tracker; const std::string& m_filename; bool m_success = false; }; @@ -143,19 +145,19 @@ void StreamHistorySession::initialize(){ switch (data.m_audio_format){ case AudioChannelFormat::NONE: expected_samples_per_frame = 0; - data.m_current.reset(new StreamHistoryTracker(0, 0, data.m_window)); + data.m_current.reset(new StreamHistoryTracker(data.m_logger, 0, 0, data.m_window)); return; case AudioChannelFormat::MONO_48000: - data.m_current.reset(new StreamHistoryTracker(1, 48000, data.m_window)); + data.m_current.reset(new StreamHistoryTracker(data.m_logger, 1, 48000, data.m_window)); return; case AudioChannelFormat::DUAL_44100: - data.m_current.reset(new StreamHistoryTracker(1, 44100, data.m_window)); + data.m_current.reset(new StreamHistoryTracker(data.m_logger, 1, 44100, data.m_window)); return; case AudioChannelFormat::DUAL_48000: case AudioChannelFormat::MONO_96000: case AudioChannelFormat::INTERLEAVE_LR_96000: case AudioChannelFormat::INTERLEAVE_RL_96000: - data.m_current.reset(new StreamHistoryTracker(2, 48000, data.m_window)); + data.m_current.reset(new StreamHistoryTracker(data.m_logger, 2, 48000, data.m_window)); return; default: throw InternalProgramError( diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_Null.h b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_Null.h index 45b160baf..e12f152fb 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_Null.h +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_Null.h @@ -19,14 +19,17 @@ namespace PokemonAutomation{ class StreamHistoryTracker{ public: StreamHistoryTracker( + Logger& logger, size_t audio_samples_per_frame, size_t audio_frames_per_second, std::chrono::seconds window - ){} + ) + : m_logger(logger) + {} void set_window(std::chrono::seconds window){} - bool save(Logger& logger, const std::string& filename) const{ - logger.log("Cannot save stream history: Not implemented.", COLOR_RED); + bool save(const std::string& filename) const{ + m_logger.log("Cannot save stream history: Not implemented.", COLOR_RED); return false; } @@ -35,6 +38,7 @@ class StreamHistoryTracker{ void on_frame(std::shared_ptr frame){} private: + Logger& m_logger; }; diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_ParallelStreams.h b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_ParallelStreams.h new file mode 100644 index 000000000..1937a8239 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_ParallelStreams.h @@ -0,0 +1,171 @@ +/* Stream History Tracker + * + * From: https://github.com/PokemonAutomation/Arduino-Source + * + * Implement by running two recordings in parallel. + * When each reaches 2X seconds, delete it start over. + * Both recordings are offset by X seconds - thus guaranteeing + * that the last X seconds are available. + * + */ + +#ifndef PokemonAutomation_StreamHistoryTracker_ParallelStreams_H +#define PokemonAutomation_StreamHistoryTracker_ParallelStreams_H + +#include +#include "Common/Cpp/AbstractLogger.h" +#include "Common/Cpp/Concurrency/SpinLock.h" +#include "CommonFramework/VideoPipeline/Backends/VideoFrameQt.h" +#include "StreamRecorder.h" + +// REMOVE +#include +using std::cout; +using std::endl; + +namespace PokemonAutomation{ + + + + + + + + + +class StreamHistoryTracker{ + static constexpr size_t PARALLEL_RECORDINGS = 2; + +public: + StreamHistoryTracker( + Logger& logger, + size_t audio_samples_per_frame, + size_t audio_frames_per_second, + std::chrono::seconds window + ) + : m_logger(logger) + , m_window(window) + , m_audio_samples_per_frame(audio_samples_per_frame) + , m_audio_frames_per_second(audio_frames_per_second) + { + update_streams(current_time()); + } + void set_window(std::chrono::seconds window){ + SpinLockGuard lg(m_lock); + m_window = window; + update_streams(current_time()); + } + + bool save(const std::string& filename){ + std::unique_ptr recording; + { + SpinLockGuard lg(m_lock); + if (m_recordings.empty()){ + m_logger.log("Cannot save stream history. Recording is not enabled.", COLOR_RED); + return false; + } + + m_logger.log("Saving stream history...", COLOR_BLUE); + + auto iter = m_recordings.begin(); + recording = std::move(iter->second); + m_recordings.erase(iter); + update_streams(current_time()); + } + return recording->stop_and_save(filename); + } + + + void on_samples(const float* samples, size_t frames){ + WallClock now = current_time(); + SpinLockGuard lg(m_lock); + for (auto& item : m_recordings){ + item.second->push_samples(now, samples, frames); + } + update_streams(now); + } + void on_frame(std::shared_ptr frame){ + WallClock now = current_time(); + SpinLockGuard lg(m_lock); + for (auto& item : m_recordings){ + item.second->push_frame(frame); + } + update_streams(now); + } + + +private: + void update_streams(WallClock current_time){ +// cout << "streams = " << m_recordings.size() << endl; + + // Must call under the lock. + WallClock threshold = current_time - m_window; + + // Clear any streams that are not needed any more. + // A stream is not needed anymore when there is a newer stream that + // is at least "m_window" long. + if (PARALLEL_RECORDINGS >= 2){ + while (m_recordings.size() >= PARALLEL_RECORDINGS){ + auto first = m_recordings.begin(); + auto second = first; + ++second; + if (second->first < threshold){ +// cout << "removing recording..." << endl; + m_recordings.erase(first); + }else{ + break; + } + } + } + + // If all recordings start in the future, reset everything. + if (!m_recordings.empty() && m_recordings.begin()->first > current_time){ + m_recordings.clear(); + } + + // Add recordings until we've reached the desired amount. + while (m_recordings.size() < PARALLEL_RECORDINGS){ +// cout << "adding recording = " << m_recordings.size() << endl; + // If no recordings are live, start it now. + // Otherwise, schedule to start "m_interval" after the start + // of the newest recording. + WallClock start_time = m_recordings.empty() + ? current_time + : m_recordings.rbegin()->first + m_window; + +// cout << std::chrono::duration_cast(start_time - current_time).count() / 1000. << endl; + + m_recordings.emplace( + std::piecewise_construct, + std::forward_as_tuple(start_time), + std::forward_as_tuple(new StreamRecording( + m_logger, std::chrono::seconds(1), + m_audio_samples_per_frame, + m_audio_frames_per_second, + start_time + )) + ); + } + + } + + +private: + Logger& m_logger; + mutable SpinLock m_lock; + std::chrono::seconds m_window; + const size_t m_audio_samples_per_frame; + const size_t m_audio_frames_per_second; + std::map> m_recordings; +}; + + + + + + + + + +} +#endif diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamRecorder.cpp b/SerialPrograms/Source/CommonFramework/Recording/StreamRecorder.cpp new file mode 100644 index 000000000..58c073f87 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamRecorder.cpp @@ -0,0 +1,287 @@ +/* Stream Recorder + * + * From: https://github.com/PokemonAutomation/Arduino-Source + * + */ + +#include +#if (QT_VERSION_MAJOR == 6) && (QT_VERSION_MINOR >= 8) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/PrettyPrint.h" +#include "Common/Cpp/Concurrency/SpinPause.h" +#include "CommonFramework/VideoPipeline/Backends/VideoFrameQt.h" +#include "StreamRecorder.h" + + +// This doesn't work yet. QMediaRecorder writes different bytes with +// setOutputDevice() vs. setOutputLocation(). +// So far I can't get setOutputDevice() to work. The bytes that it +// spits out is not a valid .mp4 file. + +//#define PA_STREAM_HISTORY_LOCAL_BUFFER + + +namespace PokemonAutomation{ + + +StreamRecording::StreamRecording( + Logger& logger, + std::chrono::milliseconds buffer_limit, + size_t audio_samples_per_frame, + size_t audio_frames_per_second, + WallClock start_time +) + : m_logger(logger) + , m_buffer_limit(buffer_limit) + , m_audio_samples_per_frame(audio_samples_per_frame) + , m_start_time(start_time) + , m_filename("TempFiles/" + now_to_filestring() + ".mp4") + , m_state(State::STARTING) + , m_last_drop(current_time()) +{ + m_audio_format.setChannelCount((int)audio_samples_per_frame); + m_audio_format.setChannelConfig(audio_samples_per_frame == 1 ? QAudioFormat::ChannelConfigMono : QAudioFormat::ChannelConfigStereo); + m_audio_format.setSampleRate((int)audio_frames_per_second); + m_audio_format.setSampleFormat(QAudioFormat::Float); + +#ifndef PA_STREAM_HISTORY_LOCAL_BUFFER + QDir().mkdir("TempFiles"); +#endif + start(); +} +StreamRecording::~StreamRecording(){ + { + std::lock_guard lg(m_lock); + m_state = State::STOPPING; +// cout << "signalling: ~StreamRecording()" << endl; + m_cv.notify_all(); + } + quit(); + wait(); +#ifndef PA_STREAM_HISTORY_LOCAL_BUFFER + QDir().remove(QString::fromStdString(m_filename)); +#endif +} + +bool StreamRecording::stop_and_save(const std::string& filename){ + { + std::lock_guard lg(m_lock); + m_state = State::STOPPING; +// cout << "signalling: stop_and_save()" << endl; + m_cv.notify_all(); + } + quit(); + wait(); + +#ifdef PA_STREAM_HISTORY_LOCAL_BUFFER + return m_write_buffer.write(m_logger, filename); +#else + bool ret = QDir().rename(QString::fromStdString(m_filename), QString::fromStdString(filename)); + m_filename.clear(); + return ret; +#endif +} + + +void StreamRecording::push_samples(WallClock timestamp, const float* data, size_t frames){ + WallClock now = current_time(); + if (now < m_start_time){ + return; + } + WallClock threshold = timestamp - m_buffer_limit; + + std::lock_guard lg(m_lock); + if (m_state != State::ACTIVE){ + return; + } + if (m_buffered_audio.empty() || m_buffered_audio.front()->timestamp > threshold){ + m_buffered_audio.emplace_back(std::make_shared( + timestamp, data, frames * m_audio_samples_per_frame + )); + m_cv.notify_all(); + return; + } + if (now - m_last_drop > std::chrono::seconds(5)){ + m_last_drop = now; + m_logger.log("Unable to keep up with audio recording. Dropping samples.", COLOR_RED); + } +} +void StreamRecording::push_frame(std::shared_ptr frame){ + WallClock now = current_time(); + if (now < m_start_time){ + return; + } +// cout << "push_frame()" << endl; + WallClock threshold = frame->timestamp - m_buffer_limit; + + std::lock_guard lg(m_lock); + if (m_state != State::ACTIVE){ + return; + } + if (m_buffered_frames.empty() || m_buffered_frames.front()->timestamp > threshold){ + m_buffered_frames.emplace_back(std::move(frame)); + m_cv.notify_all(); + return; + } + if (now - m_last_drop > std::chrono::seconds(5)){ + m_last_drop = now; + m_logger.log("Unable to keep up with video recording. Dropping samples.", COLOR_RED); + } +} + + + +void StreamRecording::internal_run(){ + QAudioBufferInput audio_input; + QVideoFrameInput video_input; + m_audio_input = &audio_input; + m_video_input = &video_input; + + QMediaCaptureSession session; + QMediaRecorder recorder; + session.setAudioBufferInput(&audio_input); + session.setVideoFrameInput(&video_input); + session.setRecorder(&recorder); + recorder.setMediaFormat(QMediaFormat::MPEG4); +// recorder.setQuality(QMediaRecorder::NormalQuality); + recorder.setQuality(QMediaRecorder::LowQuality); +// recorder.setQuality(QMediaRecorder::VeryLowQuality); + +#ifdef PA_STREAM_HISTORY_LOCAL_BUFFER + recorder.setOutputDevice(&m_write_buffer); +#else + QFileInfo file(QString::fromStdString(m_filename)); + recorder.setOutputLocation( + QUrl::fromLocalFile(file.absoluteFilePath()) + ); +#endif + +// cout << "EncodingMode = " << (int)recorder.encodingMode() << endl; + + recorder.connect( + &audio_input, &QAudioBufferInput::readyToSendAudioBuffer, + &recorder, [this](){ + std::lock_guard lg(m_lock); + m_cv.notify_all(); + } + ); + recorder.connect( + &video_input, &QVideoFrameInput::readyToSendVideoFrame, + &recorder, [this](){ + std::lock_guard lg(m_lock); + m_cv.notify_all(); + } + ); + recorder.connect( + &recorder, &QMediaRecorder::recorderStateChanged, + &recorder, [this](QMediaRecorder::RecorderState state){ + if (state == QMediaRecorder::StoppedState){ + std::lock_guard lg(m_lock); +// cout << "signalling: StoppedState" << endl; + m_state = State::STOPPING; + m_cv.notify_all(); + } + } + ); + +// cout << "starting recording" << endl; + recorder.record(); + + std::shared_ptr current_audio; + std::shared_ptr current_frame; + QAudioBuffer audio_buffer; + + while (true){ +// cout << "recording loop" << endl; + QCoreApplication::processEvents(); + + { + std::unique_lock lg(m_lock); + if (m_state == State::STOPPING){ + break; + } + m_state = State::ACTIVE; + + if (!current_audio && !m_buffered_audio.empty()){ + current_audio = std::move(m_buffered_audio.front()); + m_buffered_audio.pop_front(); + } + if (!current_frame && !m_buffered_frames.empty()){ + current_frame = std::move(m_buffered_frames.front()); + m_buffered_frames.pop_front(); + } + + if (!current_audio && !current_frame){ +// cout << "sleeping 0..." << endl; + m_cv.wait(lg); +// cout << "waking 0..." << endl; + } + } + + bool progress_made = false; + + if (current_audio){ + if (!audio_buffer.isValid()){ + const std::vector& samples = current_audio->samples; + QByteArray bytes((const char*)samples.data(), samples.size() * sizeof(float)); + audio_buffer = QAudioBuffer(bytes, m_audio_format); + } + if (audio_buffer.isValid() && audio_input.sendAudioBuffer(audio_buffer)){ + current_audio.reset(); + audio_buffer = QAudioBuffer(); + progress_made = true; + } + } + + if (current_frame && video_input.sendVideoFrame(current_frame->frame)){ + current_frame.reset(); + progress_made = true; + } + + if (!progress_made){ + std::unique_lock lg(m_lock); + if (m_state != State::ACTIVE){ + break; + } +// cout << "sleeping 1..." << endl; + m_cv.wait(lg); +// cout << "waking 1..." << endl; + } + } + + recorder.stop(); +// cout << "recorder.stop()" << endl; + + + while (recorder.recorderState() != QMediaRecorder::StoppedState){ +// cout << "StreamHistoryTracker: process" << endl; + QCoreApplication::processEvents(); + pause(); + } + +} +void StreamRecording::run(){ + try{ + internal_run(); + }catch (...){ + m_logger.log("Exception thrown out of stream recorder...", COLOR_RED); + std::lock_guard lg(m_lock); + m_state = State::STOPPING; + } +} + + + + + +} +#endif diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamRecorder.h b/SerialPrograms/Source/CommonFramework/Recording/StreamRecorder.h new file mode 100644 index 000000000..7ce918d19 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamRecorder.h @@ -0,0 +1,139 @@ +/* Stream Recorder + * + * From: https://github.com/PokemonAutomation/Arduino-Source + * + */ + +#ifndef PokemonAutomation_StreamRecorder_H +#define PokemonAutomation_StreamRecorder_H + +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Time.h" +#include "Common/Cpp/AbstractLogger.h" + +// REMOVE +#include +using std::cout; +using std::endl; + + +class QAudioBufferInput; +class QVideoFrameInput; + +namespace PokemonAutomation{ + +class VideoFrame; + + +struct AudioBlock{ + WallClock timestamp; + std::vector samples; + + AudioBlock(const AudioBlock&) = delete; + void operator=(const AudioBlock&) = delete; + + AudioBlock(WallClock p_timestamp, const float* p_samples, size_t p_count) + : timestamp(p_timestamp) + , samples(p_samples, p_samples + p_count) + {} +}; + + + +class WriteBuffer : public QIODevice{ +public: + ~WriteBuffer(){ + waitForBytesWritten(-1); + } + WriteBuffer(){ + setOpenMode(QIODeviceBase::WriteOnly); + } + + virtual qint64 readData(char* data, qint64 maxlen){ return 0; } + virtual qint64 writeData(const char* data, qint64 len){ + m_bytes += len; +// cout << "total = " << m_bytes << ", current = " << len << endl; + m_blocks.emplace_back(data, data + len); +// cout << "data = " << (unsigned)data[0] << endl; + return len; + } + + bool write(Logger& logger, const std::string& filename) const{ +// cout << "write()" << endl; + QFile file(QString::fromStdString(filename)); + if (!file.open(QFile::WriteOnly)){ + logger.log("Failed to write file: " + filename, COLOR_RED); + return false; + } + for (const std::vector& block : m_blocks){ + if ((size_t)file.write(block.data(), block.size()) != block.size()){ + logger.log("Failed to write file: " + filename, COLOR_RED); + return false; + } + } + file.close(); + return true; + } + +private: + uint64_t m_bytes = 0; + std::deque> m_blocks; +}; + + + +class StreamRecording : public QThread{ +public: + StreamRecording( + Logger& logger, + std::chrono::milliseconds buffer_limit, + size_t audio_samples_per_frame, + size_t audio_frames_per_second, + WallClock start_time + ); + ~StreamRecording(); + + void push_samples(WallClock timestamp, const float* data, size_t frames); + void push_frame(std::shared_ptr frame); + + bool stop_and_save(const std::string& filename); + +private: + void internal_run(); + virtual void run() override; + +private: + Logger& m_logger; + const std::chrono::milliseconds m_buffer_limit; + const size_t m_audio_samples_per_frame; + QAudioFormat m_audio_format; + const WallClock m_start_time; + std::string m_filename; + + std::mutex m_lock; + std::condition_variable m_cv; + + enum class State{ + STARTING, + ACTIVE, + STOPPING, + }; + + State m_state; + WallClock m_last_drop; + std::deque> m_buffered_audio; + std::deque> m_buffered_frames; + QAudioBufferInput* m_audio_input; + QVideoFrameInput* m_video_input; + WriteBuffer m_write_buffer; +}; + + + +} +#endif diff --git a/SerialPrograms/Source/NintendoSwitch/Framework/UI/NintendoSwitch_CommandRow.cpp b/SerialPrograms/Source/NintendoSwitch/Framework/UI/NintendoSwitch_CommandRow.cpp index 01f0ba273..198c562fe 100644 --- a/SerialPrograms/Source/NintendoSwitch/Framework/UI/NintendoSwitch_CommandRow.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Framework/UI/NintendoSwitch_CommandRow.cpp @@ -5,6 +5,7 @@ */ #include +#include "CommonFramework/GlobalSettingsPanel.h" #include "CommonFramework/Options/Environment/ThemeSelectorOption.h" #include "CommonFramework/VideoPipeline/UI/VideoOverlayWidget.h" #include "NintendoSwitch_CommandRow.h" @@ -104,6 +105,17 @@ CommandRow::CommandRow( this, [this](bool){ emit screenshot_requested(); } ); +#if (QT_VERSION_MAJOR == 6) && (QT_VERSION_MINOR >= 8) + if (GlobalSettings::instance().STREAM_HISTORY.enabled()){ + m_video_button = new QPushButton("Video Capture", this); + command_row->addWidget(m_video_button, 2); + connect( + m_video_button, &QPushButton::clicked, + this, [this](bool){ emit video_requested(); } + ); + } +#endif + m_session.add_listener(*this); } diff --git a/SerialPrograms/Source/NintendoSwitch/Framework/UI/NintendoSwitch_CommandRow.h b/SerialPrograms/Source/NintendoSwitch/Framework/UI/NintendoSwitch_CommandRow.h index d1c13d76c..ad3d3db6c 100644 --- a/SerialPrograms/Source/NintendoSwitch/Framework/UI/NintendoSwitch_CommandRow.h +++ b/SerialPrograms/Source/NintendoSwitch/Framework/UI/NintendoSwitch_CommandRow.h @@ -39,6 +39,7 @@ class CommandRow : public QWidget, public VirtualController, public VideoOverlay void load_profile(); void save_profile(); void screenshot_requested(); + void video_requested(); public: void set_focus(bool focused); @@ -66,6 +67,7 @@ class CommandRow : public QWidget, public VirtualController, public VideoOverlay QPushButton* m_load_profile_button; QPushButton* m_save_profile_button; QPushButton* m_screenshot_button; + QPushButton* m_video_button; bool m_last_known_focus; }; diff --git a/SerialPrograms/Source/NintendoSwitch/Framework/UI/NintendoSwitch_SwitchSystemWidget.cpp b/SerialPrograms/Source/NintendoSwitch/Framework/UI/NintendoSwitch_SwitchSystemWidget.cpp index 83d1d5f69..d974dd652 100644 --- a/SerialPrograms/Source/NintendoSwitch/Framework/UI/NintendoSwitch_SwitchSystemWidget.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Framework/UI/NintendoSwitch_SwitchSystemWidget.cpp @@ -151,6 +151,16 @@ SwitchSystemWidget::SwitchSystemWidget( }); } ); + connect( + m_command, &CommandRow::video_requested, + m_video_display, [this](){ + global_dispatcher.dispatch([this]{ + std::string filename = SCREENSHOTS_PATH() + "video-" + now_to_filestring() + ".mp4"; + m_session.logger().log("Saving screenshot to: " + filename, COLOR_PURPLE); + m_session.save_history(filename); + }); + } + ); }