Skip to content

Commit

Permalink
Improve update menu tolerance for starting game.
Browse files Browse the repository at this point in the history
  • Loading branch information
@Mysticial
Mysticial committed Oct 12, 2023
1 parent 5206eb9 commit aa4e5f9
Show file tree
Hide file tree
Showing 5 changed files with 166 additions and 33 deletions.
2 changes: 0 additions & 2 deletions SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramComputer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -236,8 +236,6 @@ void TestProgramComputer::program(ProgramEnvironment& env, CancellableScope& sco
// using namespace NintendoSwitch::PokemonSwSh::MaxLairInternal;




ImageRGB32 image("screenshot-20231005-203932147068.png");
SummaryStatsReader reader;
// cout << detector.detect(image) << endl;
Expand Down
20 changes: 19 additions & 1 deletion SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@
#include <filesystem>
#include "PokemonSwSh/MaxLair/Inference/PokemonSwSh_MaxLair_Detect_Lobby.h"
#include "PokemonSV/Inference/PokemonSV_StatHexagonReader.h"
#include "NintendoSwitch/Programs/NintendoSwitch_GameEntry.h"



Expand Down Expand Up @@ -237,12 +238,29 @@ void TestProgram::program(MultiSwitchProgramEnvironment& env, CancellableScope&
VideoOverlaySet overlays(overlay);


start_game_from_home(
console, context,
true, 0, 0,
10
);

#if 0
// UpdateMenuWatcher update_menu(false);
CheckOnlineDetector update_menu(false);
update_menu.make_overlays(overlays);

auto snapshot = console.video().snapshot();
cout << update_menu.detect(snapshot) << endl;
#endif

#if 0

// ImageRGB32 image("screenshot-20231003-202430049819.png");
auto snapshot = console.video().snapshot();

PokemonSummaryDetector detector;
cout << detector.detect(snapshot) << endl;

#endif

#if 0
SummaryStatsReader reader;
Expand Down
113 changes: 98 additions & 15 deletions SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_DetectHome.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,19 +104,20 @@ bool HomeDetector::detect(const ImageViewRGB32& screen) const{



StartGameUserSelectDetector::StartGameUserSelectDetector()
: m_bottom_row(0.10, 0.92, 0.10, 0.05)
StartGameUserSelectDetector::StartGameUserSelectDetector(Color color)
: m_color(color)
, m_bottom_row(0.10, 0.92, 0.10, 0.05)
, m_bottom_icons(0.70, 0.92, 0.28, 0.05)
, m_top_row(0.50, 0.47, 0.45, 0.05)
, m_mid_row(0.10, 0.55, 0.80, 0.02)
, m_user_slot(0.45, 0.55, 0.10, 0.35)
{}
void StartGameUserSelectDetector::make_overlays(VideoOverlaySet& items) const{
items.add(COLOR_RED, m_bottom_row);
items.add(COLOR_RED, m_bottom_icons);
items.add(COLOR_RED, m_top_row);
items.add(COLOR_RED, m_mid_row);
items.add(COLOR_RED, m_user_slot);
items.add(m_color, m_bottom_row);
items.add(m_color, m_bottom_icons);
items.add(m_color, m_top_row);
items.add(m_color, m_mid_row);
items.add(m_color, m_user_slot);
}
bool StartGameUserSelectDetector::detect(const ImageViewRGB32& screen) const{
ImageStats stats_bottom_row = image_stats(extract_box_reference(screen, m_bottom_row));
Expand Down Expand Up @@ -173,8 +174,9 @@ bool StartGameUserSelectDetector::detect(const ImageViewRGB32& screen) const{



UpdateMenuDetector::UpdateMenuDetector(bool invert)
: m_invert(invert)
UpdateMenuDetector::UpdateMenuDetector(Color color, bool invert)
: m_color(color)
, m_invert(invert)
, m_box_top(0.25, 0.26, 0.50, 0.02)
, m_box_mid(0.25, 0.52, 0.50, 0.02)
, m_top(0.10, 0.15, 0.80, 0.03)
Expand All @@ -183,12 +185,12 @@ UpdateMenuDetector::UpdateMenuDetector(bool invert)
, m_bottom_buttons(0.70, 0.92, 0.28, 0.05)
{}
void UpdateMenuDetector::make_overlays(VideoOverlaySet& items) const{
items.add(COLOR_RED, m_box_top);
items.add(COLOR_RED, m_box_mid);
items.add(COLOR_RED, m_top);
items.add(COLOR_RED, m_left);
items.add(COLOR_RED, m_bottom_solid);
items.add(COLOR_RED, m_bottom_buttons);
items.add(m_color, m_box_top);
items.add(m_color, m_box_mid);
items.add(m_color, m_top);
items.add(m_color, m_left);
items.add(m_color, m_bottom_solid);
items.add(m_color, m_bottom_buttons);
}
bool UpdateMenuDetector::detect(const ImageViewRGB32& screen) const{
ImageStats stats_box_top = image_stats(extract_box_reference(screen, m_box_top));
Expand Down Expand Up @@ -259,6 +261,87 @@ bool UpdateMenuDetector::detect(const ImageViewRGB32& screen) const{



CheckOnlineDetector::CheckOnlineDetector(Color color, bool invert)
: m_color(color)
, m_invert(invert)
, m_box_top(0.25, 0.32, 0.50, 0.02)
, m_box_mid(0.25, 0.57, 0.50, 0.02)
, m_top(0.10, 0.15, 0.80, 0.03)
, m_left(0.08, 0.25, 0.10, 0.38)
, m_bottom_solid(0.10, 0.84, 0.80, 0.04)
, m_bottom_buttons(0.70, 0.92, 0.28, 0.05)
{}
void CheckOnlineDetector::make_overlays(VideoOverlaySet& items) const{
items.add(m_color, m_box_top);
items.add(m_color, m_box_mid);
items.add(m_color, m_top);
items.add(m_color, m_left);
items.add(m_color, m_bottom_solid);
items.add(m_color, m_bottom_buttons);
}
bool CheckOnlineDetector::detect(const ImageViewRGB32& screen) const{
ImageStats stats_box_top = image_stats(extract_box_reference(screen, m_box_top));
// cout << stats_box_top.average << stats_box_top.stddev << endl;
bool white;
if (stats_box_top.average.sum() < 300){
white = false;
}else if (stats_box_top.average.sum() > 500){
white = true;
}else{
return m_invert;
}
if (stats_box_top.stddev.sum() > 10){
return m_invert;
}

// cout << "white: " << white << endl;

ImageStats stats_box_mid = image_stats(extract_box_reference(screen, m_box_mid));
if (stats_box_mid.stddev.sum() > 10){
return m_invert;
}
if (euclidean_distance(stats_box_top.average, stats_box_mid.average) > 10){
return m_invert;
}

ImageStats stats_left = image_stats(extract_box_reference(screen, m_left));
// cout << stats_left.stddev << endl;
if (stats_left.stddev.sum() < 30){
// cout << "zxcv" << endl;
return m_invert;
}

ImageStats stats_top = image_stats(extract_box_reference(screen, m_top));
// cout << stats_top.average << stats_top.stddev << endl;

ImageStats bottom_solid = image_stats(extract_box_reference(screen, m_bottom_solid));
// cout << bottom_solid.average << bottom_solid.stddev << endl;

if (euclidean_distance(stats_top.average, bottom_solid.average) > 10){
// cout << "qwer" << endl;
return m_invert;
}

if (white){
if (!is_grey(stats_top, 100, 300) || !is_grey(bottom_solid, 100, 300)){
// cout << "asdf" << endl;
return m_invert;
}
}else{
if (!is_grey(stats_top, 0, 100) || !is_grey(bottom_solid, 0, 100)){
// cout << "zxcv" << endl;
return m_invert;
}
}

ImageStats stats_bottom_buttons = image_stats(extract_box_reference(screen, m_bottom_buttons));
// cout << stats_bottom_buttons.average << stats_bottom_buttons.stddev << endl;
if (stats_bottom_buttons.stddev.sum() < 30){
return m_invert;
}

return !m_invert;
}



Expand Down
41 changes: 32 additions & 9 deletions SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_DetectHome.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#ifndef PokemonAutomation_NintendoSwitch_DetectHome_H
#define PokemonAutomation_NintendoSwitch_DetectHome_H

#include "Common/Cpp/Color.h"
#include "CommonFramework/ImageTools/ImageBoxes.h"
#include "CommonFramework/InferenceInfra/VisualInferenceCallback.h"
#include "CommonFramework/Inference/VisualDetector.h"
Expand Down Expand Up @@ -41,12 +42,13 @@ class HomeWatcher : public DetectorToFinder<HomeDetector>{

class StartGameUserSelectDetector : public StaticScreenDetector{
public:
StartGameUserSelectDetector();
StartGameUserSelectDetector(Color color = COLOR_RED);

virtual void make_overlays(VideoOverlaySet& items) const override;
virtual bool detect(const ImageViewRGB32& screen) const override;

private:
Color m_color;
ImageFloatBox m_bottom_row;
ImageFloatBox m_bottom_icons;
ImageFloatBox m_top_row;
Expand All @@ -55,21 +57,22 @@ class StartGameUserSelectDetector : public StaticScreenDetector{
};
class StartGameUserSelectWatcher : public DetectorToFinder<StartGameUserSelectDetector>{
public:
StartGameUserSelectWatcher()
: DetectorToFinder("StartGameUserSelectWatcher", std::chrono::milliseconds(250))
StartGameUserSelectWatcher(Color color = COLOR_RED)
: DetectorToFinder("StartGameUserSelectWatcher", std::chrono::milliseconds(250), color)
{}
};


// Detect the Switch system update screen when you are about to enter a game from Switch Home screen
// Detect the Switch system update screen when you are about to enter a game from Switch Home screen
class UpdateMenuDetector : public StaticScreenDetector{
public:
UpdateMenuDetector(bool invert = false);
UpdateMenuDetector(Color color = COLOR_RED, bool invert = false);

virtual void make_overlays(VideoOverlaySet& items) const override;
virtual bool detect(const ImageViewRGB32& screen) const override;

private:
Color m_color;
bool m_invert;
ImageFloatBox m_box_top;
ImageFloatBox m_box_mid;
Expand All @@ -80,17 +83,37 @@ class UpdateMenuDetector : public StaticScreenDetector{
};
class UpdateMenuWatcher : public DetectorToFinder<UpdateMenuDetector>{
public:
UpdateMenuWatcher(bool invert)
: DetectorToFinder("UpdateMenuWatcher", std::chrono::milliseconds(250), invert)
UpdateMenuWatcher(Color color = COLOR_RED, bool invert = false)
: DetectorToFinder("UpdateMenuWatcher", std::chrono::milliseconds(250), color, invert)
{}
};



// Detect the "Checking if the software can be played..." menu.
class CheckOnlineDetector : public StaticScreenDetector{
public:
CheckOnlineDetector(Color color = COLOR_RED, bool invert = false);

virtual void make_overlays(VideoOverlaySet& items) const override;
virtual bool detect(const ImageViewRGB32& screen) const override;



private:
Color m_color;
bool m_invert;
ImageFloatBox m_box_top;
ImageFloatBox m_box_mid;
ImageFloatBox m_top;
ImageFloatBox m_left;
ImageFloatBox m_bottom_solid;
ImageFloatBox m_bottom_buttons;
};
class CheckOnlineWatcher : public DetectorToFinder<CheckOnlineDetector>{
public:
CheckOnlineWatcher(Color color = COLOR_RED, bool invert = false)
: DetectorToFinder("CheckOnlineWatcher", std::chrono::milliseconds(250), color, invert)
{}
};


}
Expand Down
23 changes: 17 additions & 6 deletions SerialPrograms/Source/NintendoSwitch/Programs/NintendoSwitch_GameEntry.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ void resume_game_from_home(

while (true){
{
UpdateMenuWatcher update_detector(false);
UpdateMenuWatcher update_detector;
int ret = wait_until(
console, context,
std::chrono::milliseconds(1000),
Expand Down Expand Up @@ -131,14 +131,21 @@ void start_game_from_home_with_inference(

while (true){
HomeWatcher home(std::chrono::milliseconds(2000));
StartGameUserSelectWatcher user_select;
UpdateMenuWatcher update_menu(false);
BlackScreenWatcher black_screen;
StartGameUserSelectWatcher user_select(COLOR_GREEN);
UpdateMenuWatcher update_menu(COLOR_PURPLE);
CheckOnlineWatcher check_online(COLOR_CYAN);
BlackScreenWatcher black_screen(COLOR_BLUE);
context.wait_for_all_requests();
int ret = wait_until(
console, context,
std::chrono::seconds(10),
{ home, user_select, update_menu, black_screen }
{
home,
user_select,
update_menu,
check_online,
black_screen,
}
);

// Wait for screen to stabilize.
Expand All @@ -153,13 +160,17 @@ void start_game_from_home_with_inference(
console.log("Detected user-select screen.");
move_to_user(context, user_slot);
pbf_press_button(context, BUTTON_A, 10, start_game_wait);
return;
break;
case 2:
console.log("Detected update menu.", COLOR_RED);
pbf_press_dpad(context, DPAD_UP, 5, 0);
pbf_press_button(context, BUTTON_A, 20, 105);
break;
case 3:
console.log("Detected check online.", COLOR_RED);
context.wait_for(std::chrono::seconds(1));
break;
case 4:
console.log("Detected black screen. Game started...");
return;
default:
Expand Down

0 comments on commit aa4e5f9

Please sign in to comment.