diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.cpp b/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.cpp index a84893e70..91399b573 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.cpp +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.cpp @@ -66,13 +66,13 @@ void StreamHistorySession::start(AudioChannelFormat format, bool has_video){ } +#if 0 class HistorySaverThread : public QThread{ public: HistorySaverThread(StreamHistoryTracker& tracker, const std::string& filename) : m_tracker(tracker) , m_filename(filename) - { - } + {} ~HistorySaverThread(){ quit(); wait(); @@ -86,6 +86,7 @@ class HistorySaverThread : public QThread{ virtual void run() override{ // m_success = m_tracker.save(m_logger, m_filename); m_success = m_tracker.save(m_filename); + exec(); } private: @@ -93,8 +94,12 @@ class HistorySaverThread : public QThread{ const std::string& m_filename; bool m_success = false; }; +#endif bool StreamHistorySession::save(const std::string& filename) const{ + // This will be coming in from random threads. It will block until the save + // is finished or failed. + const Data& data = *m_data; // Get an owning reference to the current tracker. @@ -111,8 +116,10 @@ bool StreamHistorySession::save(const std::string& filename) const{ } // tracker->save(m_logger, filename); - HistorySaverThread saver(*tracker, filename); - return saver.save(); +// HistorySaverThread saver(*tracker, filename); +// return saver.save(); + + return tracker->save(filename); } void StreamHistorySession::on_samples(const float* samples, size_t frames){ Data& data = *m_data; diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_ParallelStreams.h b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_ParallelStreams.h index 1b053ff26..db7ab0920 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_ParallelStreams.h +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_ParallelStreams.h @@ -60,7 +60,7 @@ class StreamHistoryTracker{ } bool save(const std::string& filename){ - std::unique_ptr recording; + std::unique_ptr recording; { SpinLockGuard lg(m_lock); if (m_recordings.empty()){ @@ -141,7 +141,7 @@ class StreamHistoryTracker{ m_recordings.emplace( std::piecewise_construct, std::forward_as_tuple(start_time), - std::forward_as_tuple(new StreamRecording( + std::forward_as_tuple(new StreamRecording2( m_logger, std::chrono::milliseconds(500), start_time, m_audio_samples_per_frame, @@ -161,7 +161,7 @@ class StreamHistoryTracker{ const size_t m_audio_samples_per_frame; const size_t m_audio_frames_per_second; const bool m_has_video; - std::map> m_recordings; + std::map> m_recordings; }; diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamRecorder.cpp b/SerialPrograms/Source/CommonFramework/Recording/StreamRecorder.cpp index 4c401eb8d..ac201e265 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamRecorder.cpp +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamRecorder.cpp @@ -32,6 +32,12 @@ //#define PA_STREAM_HISTORY_LOCAL_BUFFER +// REMOVE +#include +using std::cout; +using std::endl; + + namespace PokemonAutomation{ @@ -96,6 +102,7 @@ StreamRecording::~StreamRecording(){ } bool StreamRecording::stop_and_save(const std::string& filename){ + auto scope_check = m_santizer.check_scope(); { std::lock_guard lg(m_lock); m_stopping = true; @@ -116,6 +123,7 @@ bool StreamRecording::stop_and_save(const std::string& filename){ void StreamRecording::push_samples(WallClock timestamp, const float* data, size_t frames){ + auto scope_check = m_santizer.check_scope(); #if 1 WallClock now = current_time(); if (m_audio_samples_per_frame == 0 || now < m_start_time){ @@ -154,6 +162,7 @@ void StreamRecording::push_samples(WallClock timestamp, const float* data, size_ #endif } void StreamRecording::push_frame(std::shared_ptr frame){ + auto scope_check = m_santizer.check_scope(); #if 1 WallClock now = current_time(); if (!m_has_video || now < m_start_time){ @@ -266,18 +275,6 @@ std::unique_ptr StreamRecording::initialize_video(){ }, Qt::DirectConnection ); - connect( - m_recorder, &QMediaRecorder::recorderStateChanged, - m_recorder, [this](QMediaRecorder::RecorderState state){ - if (state == QMediaRecorder::StoppedState){ - std::lock_guard lg(m_lock); -// cout << "signalling: StoppedState" << endl; - m_stopping = true; - m_cv.notify_all(); - } - }, - Qt::DirectConnection - ); return ret; } @@ -302,6 +299,18 @@ void StreamRecording::internal_run(){ m_recorder->setMediaFormat(QMediaFormat::MPEG4); + connect( + m_recorder, &QMediaRecorder::recorderStateChanged, + m_recorder, [this](QMediaRecorder::RecorderState state){ + if (state == QMediaRecorder::StoppedState){ + std::lock_guard lg(m_lock); +// cout << "signalling: StoppedState" << endl; + m_stopping = true; + m_cv.notify_all(); + } + }, + Qt::DirectConnection + ); #ifdef PA_STREAM_HISTORY_LOCAL_BUFFER @@ -398,6 +407,7 @@ void StreamRecording::internal_run(){ } void StreamRecording::run(){ + auto scope_check = m_santizer.check_scope(); try{ internal_run(); }catch (...){ @@ -411,5 +421,343 @@ void StreamRecording::run(){ +StreamRecording2::StreamRecording2( + Logger& logger, + std::chrono::milliseconds buffer_limit, + WallClock start_time, + size_t audio_samples_per_frame, + size_t audio_frames_per_second, + bool has_video +) + : m_logger(logger) + , m_buffer_limit(buffer_limit) + , m_start_time(start_time) + , m_audio_samples_per_frame(audio_samples_per_frame) + , m_has_video(has_video) + , m_filename(GlobalSettings::instance().TEMP_FOLDER) + , m_stopping(false) + , m_last_drop(current_time()) + , m_last_frame_time(std::numeric_limits::min()) +{ + logger.log( + "StreamRecording: Audio = " + std::to_string(audio_samples_per_frame) + + ", Video = " + std::to_string(has_video) + ); +// cout << "audio = " << audio_samples_per_frame << ", video = " << has_video << endl; + + if (audio_samples_per_frame == 0 && !has_video){ + return; + } + + if (audio_samples_per_frame > 0){ + m_audio_format.setChannelCount((int)audio_samples_per_frame); + m_audio_format.setChannelConfig(audio_samples_per_frame == 1 ? QAudioFormat::ChannelConfigMono : QAudioFormat::ChannelConfigStereo); + m_audio_format.setSampleRate((int)audio_frames_per_second); + m_audio_format.setSampleFormat(QAudioFormat::Float); + } + +#ifndef PA_STREAM_HISTORY_LOCAL_BUFFER + QDir().mkdir(QString::fromStdString(m_filename)); +#endif + if (has_video){ + m_filename += now_to_filestring() + ".mp4"; + }else{ + m_filename += now_to_filestring() + ".m4a"; + } + + m_session.setRecorder(&m_recorder); + + // Only initialize the streams we intend to use. + if (m_audio_samples_per_frame > 0){ + initialize_audio(); + } + if (m_has_video){ + initialize_video(); + } + + m_recorder.setMediaFormat(QMediaFormat::MPEG4); + + + +#ifdef PA_STREAM_HISTORY_LOCAL_BUFFER + m_recorder->setOutputDevice(&m_write_buffer); +#else + QFileInfo file(QString::fromStdString(m_filename)); + m_recorder.setOutputLocation( + QUrl::fromLocalFile(file.absoluteFilePath()) + ); +#endif + + m_recorder.record(); +} +StreamRecording2::~StreamRecording2(){ + stop(); +#ifndef PA_STREAM_HISTORY_LOCAL_BUFFER + QDir().remove(QString::fromStdString(m_filename)); +#endif +} + +void StreamRecording2::stop(){ + auto scope_check = m_santizer.check_scope(); + + { + std::unique_lock lg(m_lock); + if (m_stopping){ + return; + } + m_stopping = true; + } + + QEventLoop loop; + m_recorder.connect( + &m_recorder, &QMediaRecorder::recorderStateChanged, + &m_recorder, [&](QMediaRecorder::RecorderState state){ + if (state == QMediaRecorder::StoppedState){ + loop.quit(); + } + } + ); + + emit m_recorder.stop(); + + if (m_recorder.recorderState() != QMediaRecorder::StoppedState){ + loop.exec(); + } +} + + +void StreamRecording2::initialize_audio(){ + m_audio_input.reset(new QAudioBufferInput()); + m_session.setAudioBufferInput(m_audio_input.get()); + + m_recorder.connect( + m_audio_input.get(), &QAudioBufferInput::readyToSendAudioBuffer, + &m_recorder, [this](){ +// cout << "readyToSendAudioBuffer()" << endl; +// std::lock_guard lg(m_lock); +// m_cv.notify_all(); + process(); + }, + Qt::DirectConnection + ); +} +void StreamRecording2::initialize_video(){ + m_video_input.reset(new QVideoFrameInput()); + m_session.setVideoFrameInput(m_video_input.get()); + + const StreamHistoryOption& settings = GlobalSettings::instance().STREAM_HISTORY; + + switch (settings.RESOLUTION){ + case StreamHistoryOption::Resolution::MATCH_INPUT: + break; + case StreamHistoryOption::Resolution::FORCE_720p: + m_recorder.setVideoResolution(1280, 720); + break; + case StreamHistoryOption::Resolution::FORCE_1080p: + m_recorder.setVideoResolution(1920, 1080); + break; + } + + switch (settings.ENCODING_MODE){ + case StreamHistoryOption::EncodingMode::FIXED_QUALITY: + switch (settings.VIDEO_QUALITY){ + case StreamHistoryOption::VideoQuality::VERY_LOW: + m_recorder.setQuality(QMediaRecorder::VeryLowQuality); + break; + case StreamHistoryOption::VideoQuality::LOW: + m_recorder.setQuality(QMediaRecorder::LowQuality); + break; + case StreamHistoryOption::VideoQuality::NORMAL: + m_recorder.setQuality(QMediaRecorder::NormalQuality); + break; + case StreamHistoryOption::VideoQuality::HIGH: + m_recorder.setQuality(QMediaRecorder::HighQuality); + break; + case StreamHistoryOption::VideoQuality::VERY_HIGH: + m_recorder.setQuality(QMediaRecorder::VeryHighQuality); + break; + } + break; + case StreamHistoryOption::EncodingMode::FIXED_BITRATE: + m_recorder.setVideoBitRate(settings.VIDEO_BITRATE * 1000); + m_recorder.setEncodingMode(QMediaRecorder::AverageBitRateEncoding); + break; + } + + m_recorder.connect( + m_video_input.get(), &QVideoFrameInput::readyToSendVideoFrame, + &m_recorder, [this](){ +// cout << "readyToSendVideoFrame()" << endl; +// std::lock_guard lg(m_lock); +// m_cv.notify_all(); + process(); + }, + Qt::DirectConnection + ); +} + + + + +void StreamRecording2::push_samples(WallClock timestamp, const float* data, size_t frames){ + auto scope_check = m_santizer.check_scope(); +#if 1 + WallClock now = current_time(); + if (m_audio_samples_per_frame == 0 || now < m_start_time){ + return; + } + WallClock threshold = timestamp - m_buffer_limit; + + { + std::lock_guard lg(m_lock); + if (m_stopping){ + return; + } + + do{ + if (m_buffered_audio.empty()){ + break; + } + + // Too much has been buffered. Drop the block. + if (m_buffered_audio.front().timestamp < threshold){ + // Throttle the prints. + if (now - m_last_drop > std::chrono::seconds(5)){ + m_last_drop = now; + m_logger.log("Unable to keep up with audio recording. Dropping samples.", COLOR_RED); + } + return; + } + + }while (false); + + // Enqueue the sample block. + m_buffered_audio.emplace_back( + timestamp, + data, frames * m_audio_samples_per_frame + ); + } + + emit m_audio_input->readyToSendAudioBuffer(); +#endif +} +void StreamRecording2::push_frame(std::shared_ptr frame){ + auto scope_check = m_santizer.check_scope(); +#if 1 + WallClock now = current_time(); + if (!m_has_video || now < m_start_time){ + return; + } +// cout << "push_frame(): " << frame->frame.startTime() << " - " << frame->frame.endTime() << endl; + WallClock threshold = frame->timestamp - m_buffer_limit; + + { + std::lock_guard lg(m_lock); + if (m_stopping){ + return; + } + + qint64 frame_time = frame->frame.startTime(); + do{ + if (m_buffered_frames.empty()){ + break; + } + + // Too much has been buffered. Drop the frame. + if (m_buffered_frames.front()->timestamp < threshold){ + // Throttle the prints. + if (now - m_last_drop > std::chrono::seconds(5)){ + m_last_drop = now; + m_logger.log("Unable to keep up with video recording. Dropping samples.", COLOR_RED); + } + return; + } + + // Non-increasing timestamp. Drop possible duplicate frame. + if (frame_time <= m_last_frame_time){ + return; + } + }while (false); + + // Enqueue the frame. + m_buffered_frames.emplace_back(std::move(frame)); + m_last_frame_time = frame_time; + } + + emit m_video_input->readyToSendVideoFrame(); +#endif +} + + +bool StreamRecording2::stop_and_save(const std::string& filename){ + stop(); + +#ifdef PA_STREAM_HISTORY_LOCAL_BUFFER + return m_write_buffer.write(m_logger, filename); +#else + bool ret = QDir().rename(QString::fromStdString(m_filename), QString::fromStdString(filename)); + m_filename.clear(); + return ret; +#endif +} + +void StreamRecording2::process(){ + auto scope_check = m_santizer.check_scope(); + +// cout << "StreamRecording2::process()" << endl; + + bool progress_made; + do{ + { + std::unique_lock lg(m_lock); + if (m_stopping){ + cout << "exit" << endl; // REMOVE + break; + } + + if (!m_current_audio.is_valid() && !m_buffered_audio.empty()){ + m_current_audio = std::move(m_buffered_audio.front()); + m_buffered_audio.pop_front(); + } + if (!m_current_frame && !m_buffered_frames.empty()){ + m_current_frame = std::move(m_buffered_frames.front()); + m_buffered_frames.pop_front(); + } + + if (!m_current_audio.is_valid() && !m_current_frame){ + return; + } + } + + progress_made = false; + + if (m_current_audio.is_valid()){ + if (!m_audio_buffer.isValid()){ + const std::vector& samples = m_current_audio.samples; + QByteArray bytes((const char*)samples.data(), samples.size() * sizeof(float)); + m_audio_buffer = QAudioBuffer(bytes, m_audio_format); + } + if (m_audio_buffer.isValid() && m_audio_input->sendAudioBuffer(m_audio_buffer)){ + m_current_audio.clear(); + m_audio_buffer = QAudioBuffer(); + progress_made = true; + } + } +// cout << "Before: " << m_video_input << endl; + if (m_current_frame && m_video_input->sendVideoFrame(m_current_frame->frame)){ +// cout << "push frame: " << current_frame->frame.startTime() << endl; + m_current_frame.reset(); + progress_made = true; + } + + }while (progress_made); +} + + + + + + + } #endif diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamRecorder.h b/SerialPrograms/Source/CommonFramework/Recording/StreamRecorder.h index 8b44d4118..287f092aa 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamRecorder.h +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamRecorder.h @@ -12,7 +12,11 @@ #include #include #include +#include #include +#include +#include +#include "Common/Cpp/LifetimeSanitizer.h" #include "Common/Cpp/Time.h" #include "Common/Cpp/AbstractLogger.h" #include "CommonFramework/VideoPipeline/Backends/VideoFrameQt.h" @@ -149,9 +153,74 @@ class StreamRecording : public QThread{ QVideoFrameInput* m_video_input = nullptr; QMediaCaptureSession* m_session = nullptr; QMediaRecorder* m_recorder = nullptr; + + LifetimeSanitizer m_santizer; +}; + + + + +class StreamRecording2{ +public: + StreamRecording2( + Logger& logger, + std::chrono::milliseconds buffer_limit, + WallClock start_time, + size_t audio_samples_per_frame, + size_t audio_frames_per_second, + bool has_video + ); + ~StreamRecording2(); + + void push_samples(WallClock timestamp, const float* data, size_t frames); + void push_frame(std::shared_ptr frame); + + void stop(); + bool stop_and_save(const std::string& filename); + +private: + void initialize_audio(); + void initialize_video(); + void process(); + + +private: + Logger& m_logger; + const std::chrono::milliseconds m_buffer_limit; + const WallClock m_start_time; + const size_t m_audio_samples_per_frame; + const bool m_has_video; + + QAudioFormat m_audio_format; + std::string m_filename; + + std::mutex m_lock; +// std::condition_variable m_cv; + + bool m_stopping = false; + WallClock m_last_drop; + + std::deque m_buffered_audio; + + qint64 m_last_frame_time; + std::deque> m_buffered_frames; + + WriteBuffer m_write_buffer; + + std::unique_ptr m_audio_input; + std::unique_ptr m_video_input; + QMediaCaptureSession m_session; + QMediaRecorder m_recorder; + + AudioBlock m_current_audio; + std::shared_ptr m_current_frame; + QAudioBuffer m_audio_buffer; + + LifetimeSanitizer m_santizer; }; + } #endif diff --git a/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp b/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp index ad6994dfe..5baad2725 100644 --- a/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp +++ b/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp @@ -122,6 +122,9 @@ #include "PokemonSwSh/Inference/PokemonSwSh_IvJudgeReader.h" //#include "CommonFramework/Environment/SystemSleep.h" #include "CommonFramework/ErrorReports/ErrorReports.h" +#include "PokemonLA/Inference/Map/PokemonLA_OutbreakReader.h" +#include "PokemonSV/Programs/Farming/PokemonSV_AuctionFarmer.h" +#include "PokemonLA/Inference/Objects/PokemonLA_MMOQuestionMarkDetector.h" @@ -267,6 +270,32 @@ void TestProgram::program(MultiSwitchProgramEnvironment& env, CancellableScope& BotBaseContext context(scope, console.botbase()); VideoOverlaySet overlays(overlay); + +// LifetimeSanitizer::terminate_with_dump(); + +// PokemonSV::AuctionFarmer farmer; +// farmer.check_offers(env); +// std::terminate(); + +#if 0 + ImageRGB32 image("screenshot-20241210-110029984325.png"); +// auto image = feed.snapshot(); + + MMOQuestionMarkDetector question_mark_detector(logger); + question_mark_detector.detect_MMO_on_hisui_map(image); +#endif + + +#if 0 + PokemonLA::OutbreakReader reader(logger, Language::English, overlay); + reader.make_overlays(overlays); + + ImageRGB32 image("screenshot-20241124-135028529403.png"); +#endif + +// reader.read(feed.snapshot()); + + // PokemonLA::ButtonDetector detector(logger, PokemonLA::ButtonType::ButtonA,); // while (true){