diff --git a/SerialPrograms/CMakeLists.txt b/SerialPrograms/CMakeLists.txt index 4dc12c9ca..ba84aa182 100644 --- a/SerialPrograms/CMakeLists.txt +++ b/SerialPrograms/CMakeLists.txt @@ -1332,6 +1332,8 @@ file(GLOB MAIN_SOURCES Source/PokemonSV/Inference/Overworld/PokemonSV_LetsGoKillDetector.h Source/PokemonSV/Inference/Overworld/PokemonSV_NoMinimapDetector.cpp Source/PokemonSV/Inference/Overworld/PokemonSV_NoMinimapDetector.h + Source/PokemonSV/Inference/Overworld/PokemonSV_OliveDetector.cpp + Source/PokemonSV/Inference/Overworld/PokemonSV_OliveDetector.h Source/PokemonSV/Inference/Overworld/PokemonSV_OverworldDetector.cpp Source/PokemonSV/Inference/Overworld/PokemonSV_OverworldDetector.h Source/PokemonSV/Inference/Overworld/PokemonSV_StationaryOverworldWatcher.cpp diff --git a/SerialPrograms/SerialPrograms.pro b/SerialPrograms/SerialPrograms.pro index f31552362..b60b57694 100644 --- a/SerialPrograms/SerialPrograms.pro +++ b/SerialPrograms/SerialPrograms.pro @@ -666,6 +666,7 @@ SOURCES += \ Source/PokemonSV/Inference/Overworld/PokemonSV_LetsGoHpReader.cpp \ Source/PokemonSV/Inference/Overworld/PokemonSV_LetsGoKillDetector.cpp \ Source/PokemonSV/Inference/Overworld/PokemonSV_NoMinimapDetector.cpp \ + Source/PokemonSV/Inference/Overworld/PokemonSV_OliveDetector.cpp \ Source/PokemonSV/Inference/Overworld/PokemonSV_OverworldDetector.cpp \ Source/PokemonSV/Inference/Overworld/PokemonSV_StationaryOverworldWatcher.cpp \ Source/PokemonSV/Inference/Picnics/PokemonSV_PicnicDetector.cpp \ @@ -1766,6 +1767,7 @@ HEADERS += \ Source/PokemonSV/Inference/Overworld/PokemonSV_LetsGoHpReader.h \ Source/PokemonSV/Inference/Overworld/PokemonSV_LetsGoKillDetector.h \ Source/PokemonSV/Inference/Overworld/PokemonSV_NoMinimapDetector.h \ + Source/PokemonSV/Inference/Overworld/PokemonSV_OliveDetector.h \ Source/PokemonSV/Inference/Overworld/PokemonSV_OverworldDetector.h \ Source/PokemonSV/Inference/Overworld/PokemonSV_StationaryOverworldWatcher.h \ Source/PokemonSV/Inference/Picnics/PokemonSV_PicnicDetector.h \ diff --git a/SerialPrograms/Source/PokemonSV/Inference/Overworld/PokemonSV_OliveDetector.cpp b/SerialPrograms/Source/PokemonSV/Inference/Overworld/PokemonSV_OliveDetector.cpp new file mode 100644 index 000000000..b8293d7a3 --- /dev/null +++ b/SerialPrograms/Source/PokemonSV/Inference/Overworld/PokemonSV_OliveDetector.cpp @@ -0,0 +1,386 @@ +/* Olive Detector + * + * From: https://github.com/PokemonAutomation/Arduino-Source + * + */ +#include "Kernels/Waterfill/Kernels_Waterfill_Session.h" +#include "CommonFramework/Globals.h" +#include "CommonFramework/Exceptions/OperationFailedException.h" +#include "CommonFramework/Exceptions/OliveActionFailedException.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "CommonFramework/ImageMatch/ImageDiff.h" +#include "CommonFramework/ImageMatch/ExactImageMatcher.h" +#include "CommonFramework/ImageTools/BinaryImage_FilterRgb32.h" +#include "CommonFramework/ImageTools/WaterfillUtilities.h" +#include "CommonFramework/ImageMatch/WaterfillTemplateMatcher.h" +#include "CommonFramework/ImageTools/ImageFilter.h" +#include "CommonFramework/Tools/ConsoleHandle.h" +#include "NintendoSwitch/Programs/NintendoSwitch_SnapshotDumper.h" +#include "PokemonSV/Inference/Overworld/PokemonSV_DirectionDetector.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "PokemonSV_OliveDetector.h" + +#include +using std::cout; +using std::endl; + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonSV{ + + +class OliveMatcher : public ImageMatch::WaterfillTemplateMatcher{ +public: + OliveMatcher() : WaterfillTemplateMatcher( + "PokemonSV/Olive.png", Color(0,0,0), Color(255, 255, 255), 5 + ){ + m_aspect_ratio_lower = 0.35; + m_aspect_ratio_upper = 3; + m_area_ratio_lower = 0.5; + m_area_ratio_upper = 1.5; + } + + static const ImageMatch::WaterfillTemplateMatcher& instance(){ + static OliveMatcher matcher; + return matcher; + } +}; + + +OliveDetector::OliveDetector(ConsoleHandle& console, Color color) + : m_overlays(console.overlay()) + , m_color(color) +{} +void OliveDetector::make_overlays(VideoOverlaySet& items) const{ +} + + +std::pair OliveDetector::olive_location(ConsoleHandle& console, BotBaseContext& context, ImageFloatBox box){ + context.wait_for_all_requests(); + ImageFloatBox location = get_olive_floatbox(console, context, 30, box); + double x = location.x + (location.width / 2); + double y = location.y + (location.height / 2); + + return std::make_pair(x, y); +} + +std::pair box_center(ImageFloatBox& box){ + double x = box.x + (box.width / 2); + double y = box.y + (box.height / 2); + + return std::make_pair(x, y); +} + +ImageFloatBox OliveDetector::get_olive_floatbox(const ImageViewRGB32& screen, BotBaseContext& context, uint8_t rgb_gap, ImageFloatBox box){ + const std::vector> filters = { + {combine_rgb(0, 10, 0), combine_rgb(255, 255, 255)}, + }; + + ImageRGB32 green_only = filter_green(screen, Color(0xff000000), rgb_gap); + + const double min_object_size = 1000; + const double rmsd_threshold = 100; + + const double screen_rel_size = (screen.height() / 1080.0); + const size_t min_size = size_t(screen_rel_size * screen_rel_size * min_object_size); + + size_t largest_size = 0; + ImageFloatBox largest_green(0, 0, 0, 0); + ImageViewRGB32 cropped = extract_box_reference(green_only, box); + ImagePixelBox pixel_search_area = floatbox_to_pixelbox(screen.width(), screen.height(), box); + match_template_by_waterfill( + cropped, + OliveMatcher::instance(), + filters, + {min_size, SIZE_MAX}, + rmsd_threshold, + [&](Kernels::Waterfill::WaterfillObject& object) -> bool { + size_t object_area = object.area; + if (object_area > largest_size){ + largest_size = object_area; + // cout << largest_size << endl; + ImagePixelBox found_box( + object.min_x + pixel_search_area.min_x, object.min_y + pixel_search_area.min_y, + object.max_x + pixel_search_area.min_x, object.max_y + pixel_search_area.min_y); + largest_green = pixelbox_to_floatbox(screen.width(), screen.height(), found_box); + } + + return false; + } + ); + + m_overlays.clear(); + m_overlays.add(m_color, largest_green); + + return largest_green; +} + +ImageFloatBox OliveDetector::get_olive_floatbox(ConsoleHandle& console, BotBaseContext& context, uint8_t rgb_gap, ImageFloatBox box){ + size_t MAX_ATTEMPTS = 2; + for (size_t i = 0; i < MAX_ATTEMPTS; i++){ + context.wait_for_all_requests(); + auto snapshot = console.video().snapshot(); + const ImageViewRGB32& screen = snapshot; + ImageFloatBox olive_box = get_olive_floatbox(screen, context, rgb_gap, box); + if (olive_box.x == 0 && olive_box.y == 0){ + // Olive not detected. try again. may have been obscured by the player's head/hat + context.wait_for(Milliseconds(2000)); + continue; + } + return olive_box; + } + + // dump_snapshot(console); + throw OliveActionFailedException( + ErrorReport::SEND_ERROR_REPORT, console, + "get_olive_floatbox(): Olive not detected.", + true, + OliveFail::NO_OLIVE_DETECTED + ); +} + +ImageFloatBox OliveDetector::align_to_olive( + const ProgramInfo& info, + ConsoleHandle& console, + BotBaseContext& context, + double direction_facing, + uint8_t rgb_gap, + ImageFloatBox area_to_check +){ + size_t MAX_ATTEMPTS = 10; + // size_t olive_unchanged_count = 0; + ImageFloatBox olive_box; + DirectionDetector direction; + uint16_t scale_factor = 130; + int16_t prev_push_direction = 0; + for (size_t i = 0; i < MAX_ATTEMPTS; i++){ + direction.change_direction(info, console, context, direction_facing); + pbf_move_left_joystick(context, 128, 0, 5, 20); + + olive_box = get_olive_floatbox(console, context, rgb_gap, area_to_check); + + std::pair olive = box_center(olive_box); + double olive_x = olive.first; + double olive_y = olive.second; + double olive_area = olive_box.width * olive_box.height; + + double diff_from_center = olive_x - 0.5; + console.log("olive_x: " + std::to_string(olive_x) + ", olive_y: " + std::to_string(olive_y) + ", olive_area: " + std::to_string(olive_area)); + if (std::abs(diff_from_center) < 0.01){ + return olive_box; + } + + + int16_t push_direction = (diff_from_center > 0) ? 1 : -1; + if (olive_y > 0.50 && scale_factor > 100){ + scale_factor = 100; + } + if (olive_box.height > 0.35 && scale_factor > 50){ + scale_factor = 50; + } + if (push_direction * -1 == prev_push_direction){ + // if you overshot the olive, and are now pushing in the opposite direction + // then reduce the push_duration. + scale_factor = int16_t(scale_factor * 0.5); + } + + // cout << "olive_height" << olive_box.height << endl; + // if (olive_box.height > 0.4){ + // scale_factor = 50; + // } + + + uint16_t push_duration = std::max(uint16_t((std::abs(diff_from_center) + 0.02) * scale_factor / (olive_y)), uint16_t(15)); + double push_magnitude = 128; // std::max(double(128 / (i + 1)), double(20)); // push less with each iteration/attempt + uint8_t push_x = uint8_t(std::max(std::min(int(128 + (push_direction * push_magnitude)), 255), 0)); + console.log("scale_factor: " + std::to_string(scale_factor)); + console.log("push x: " + std::to_string(push_x) + ", push duration: " + std::to_string(push_duration)); + // pbf_wait(context, 100); + uint16_t wait_ticks = 50; + if (std::abs(diff_from_center) < 0.05){ + wait_ticks = 100; + } + pbf_move_left_joystick(context, push_x, 128, push_duration, wait_ticks); + prev_push_direction = push_direction; + + + // // check if we're making progress towards centering on the olive. + // ImageFloatBox olive_box_2 = get_olive_floatbox(console, context, rgb_gap, area_to_check); + // double box_1_area = olive_box.width * olive_box.height; + // double box_2_area = olive_box_2.width * olive_box_2.height; + // double area_diff = std::abs(box_1_area - box_2_area); + // double x_diff = std::abs(olive_box.x - olive_box_2.x); + // double y_diff = std::abs(olive_box.y - olive_box_2.y); + + // if (area_diff < 0.02 && x_diff < 0.01 && y_diff < 0.01){ + // olive_unchanged_count++; + // // not moving closer to the olive. either wall/fence is in the way, or we are right next the olive. + // console.log("Can't align to Olive. Try moving backwards and try again."); + // pbf_move_left_joystick(context, 128, 255, 75, 100); // walk backwards + // if (olive_unchanged_count == 2){ + // throw OliveActionFailedException( + // ErrorReport::SEND_ERROR_REPORT, console, + // "align_to_olive(): Failed to align to olive.", + // true, + // OliveFail::FAILED_ALIGN_TO_OLIVE + // ); + // } + // }else{ + // olive_unchanged_count = 0; + // } + } + return olive_box; + + // don't throw an exception, since sometimes the program has trouble detecting the olive's exact location with the white logo on the olive. + // so we rely on maxing out the attempts to move on. + // throw OliveActionFailedException( + // ErrorReport::SEND_ERROR_REPORT, console, + // "align_to_olive(): Failed to align to olive.", + // true, + // OliveFail::FAILED_ALIGN_TO_OLIVE + // ); +} + + +// todo: detect and handle case where olive is stuck. +// todo: detect and handle case where olive is slightly to the left, and so we need to move on to the next phase +uint16_t OliveDetector::push_olive_forward( + const ProgramInfo& info, + ConsoleHandle& console, + BotBaseContext& context, + double direction_facing, + uint16_t total_forward_distance, + uint16_t push_olive, + uint8_t rgb_gap, + ImageFloatBox area_to_check +){ + console.log("push_olive_forward(): Total distance: " + std::to_string(total_forward_distance)); + // uint16_t initial_push_olive = push_olive; + uint16_t ticks_walked = 0; + size_t MAX_ATTEMPTS = 10; + for (size_t i = 0; i < MAX_ATTEMPTS; i++){ + align_to_olive(info, console, context, direction_facing, rgb_gap, area_to_check); + ticks_walked += walk_up_to_olive(info, console, context, direction_facing, rgb_gap, area_to_check); + + + if (ticks_walked >= total_forward_distance){ + console.log("Distance walked: " + std::to_string(ticks_walked) + "/" + std::to_string(total_forward_distance)); + return ticks_walked; + } + + align_to_olive(info, console, context, direction_facing, rgb_gap, area_to_check); + // check location of olive before and after push + // if olive is approximately in the same location, then the olive is stuck. try moving backward and running forward again. + ImageFloatBox olive_box_1 = get_olive_floatbox(console, context, rgb_gap, area_to_check); + for (size_t j = 0; j < 3; j++){ + console.log("Distance walked: " + std::to_string(ticks_walked) + "/" + std::to_string(total_forward_distance)); + console.log("Push the olive."); + pbf_move_left_joystick(context, 128, 0, push_olive, 7 * TICKS_PER_SECOND); + + ticks_walked += push_olive; + ImageFloatBox olive_box_2 = get_olive_floatbox(console, context, rgb_gap, area_to_check); + double box_1_area = olive_box_1.width * olive_box_1.height; + double box_2_area = olive_box_2.width * olive_box_2.height; + double area_diff = std::abs(box_1_area - box_2_area); + double x_diff = std::abs(olive_box_1.x - olive_box_2.x); + double y_diff = std::abs(olive_box_1.y - olive_box_2.y); + + if (area_diff < 0.02 && x_diff < 0.05 && y_diff < 0.05){ + console.log("Olive is stuck? Move backwards and try pushing again."); + console.log("Olive 1: area: " + std::to_string(box_1_area) + ", x: " + std::to_string(olive_box_1.x) + ", y: " + std::to_string(olive_box_1.y)); + console.log("Olive 2: area: " + std::to_string(box_2_area) + ", x: " + std::to_string(olive_box_2.x) + ", y: " + std::to_string(olive_box_2.y)); + pbf_move_left_joystick(context, 128, 255, 75, 100); // walk backwards + ticks_walked -= push_olive; + push_olive = 200; // run forward more on the next push + + if (j == 2){ + throw OliveActionFailedException( + ErrorReport::SEND_ERROR_REPORT, console, + "push_olive_forward(): Olive stuck.", + true, + OliveFail::OLIVE_STUCK + ); + } + }else{ + break; + } + } + + if (ticks_walked > total_forward_distance){ + console.log("Distance walked: " + std::to_string(ticks_walked) + "/" + std::to_string(total_forward_distance)); + return ticks_walked; + } + + console.log("Distance walked: " + std::to_string(ticks_walked) + "/" + std::to_string(total_forward_distance)); + + } + + throw OliveActionFailedException( + ErrorReport::SEND_ERROR_REPORT, console, + "push_olive_forward(): Something went wrong. Failed to walk the Olive forward as expected.", + true, + OliveFail::FAILED_PUSH_OLIVE_TOTAL_DISTANCE + ); + +} + +uint16_t OliveDetector::walk_up_to_olive( + const ProgramInfo& info, + ConsoleHandle& console, + BotBaseContext& context, + double direction_facing, + uint8_t rgb_gap, + ImageFloatBox area_to_check +){ + uint16_t ticks_walked = 0; + size_t MAX_ATTEMPTS = 20; + for (size_t i = 0; i < MAX_ATTEMPTS; i++){ + ImageFloatBox olive_box = get_olive_floatbox(console, context, rgb_gap, area_to_check); + std::pair olive = box_center(olive_box); + // double olive_x = olive.first; + double olive_y = olive.second; + + uint16_t scale_factor = 2000; + uint16_t push_duration = std::max(uint16_t((0.57 - olive_y)*(0.57 - olive_y) * scale_factor), uint16_t(20)); + console.log("walk_up_to_olive(): olive_y: " + std::to_string(olive_y)); + if (olive_y > 0.515){ + return ticks_walked; + } + console.log("push duration: " + std::to_string(push_duration)); + // when push durations are low, the player moves less than expected + // once above 45, you walk about as much as expected + double walking_factor = 1; + if (push_duration <= 20){ + walking_factor = 0.40; + }else if (push_duration <= 25){ + walking_factor = 0.53; + }else if (push_duration <= 30){ + walking_factor = 0.69; + }else if (push_duration <= 35){ + walking_factor = 0.83; + }else if (push_duration <= 40){ + walking_factor = 0.91; + } + ticks_walked += uint16_t(push_duration * walking_factor); + + uint16_t wait_ticks = 50; + if (olive_y > 0.4){ + wait_ticks = 100; + } + pbf_move_left_joystick(context, 128, 0, push_duration, wait_ticks); + } + + throw OliveActionFailedException( + ErrorReport::SEND_ERROR_REPORT, console, + "walk_up_to_olive(): Something went wrong. Failed to walk up to the Olive", + true, + OliveFail::FAILED_WALK_TO_OLIVE + ); + +} + + + +} +} +} diff --git a/SerialPrograms/Source/PokemonSV/Inference/Overworld/PokemonSV_OliveDetector.h b/SerialPrograms/Source/PokemonSV/Inference/Overworld/PokemonSV_OliveDetector.h new file mode 100644 index 000000000..4c88bc7bb --- /dev/null +++ b/SerialPrograms/Source/PokemonSV/Inference/Overworld/PokemonSV_OliveDetector.h @@ -0,0 +1,86 @@ +/* Olive Detector + * + * From: https://github.com/PokemonAutomation/Arduino-Source + * + */ + +#ifndef PokemonAutomation_PokemonSV_OliveDetector_H +#define PokemonAutomation_PokemonSV_OliveDetector_H + +#include "Common/Cpp/Color.h" +#include "ClientSource/Connection/BotBase.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonSV{ + +class OliveDetector { +public: + OliveDetector(ConsoleHandle& console, Color color = COLOR_RED); + + void make_overlays(VideoOverlaySet& items) const; + + std::pair olive_location(ConsoleHandle& console, BotBaseContext& context, ImageFloatBox box = {0, 0.15, 1, 0.7}); + + ImageFloatBox get_olive_floatbox( + const ImageViewRGB32& screen, + BotBaseContext& context, + uint8_t rgb_gap, + ImageFloatBox box + ); + + // return ImageFloatBox of the of the Olive, based on the largest blob of green + ImageFloatBox get_olive_floatbox( + ConsoleHandle& console, + BotBaseContext& context, + uint8_t rgb_gap, + ImageFloatBox box + ); + + ImageFloatBox align_to_olive( + const ProgramInfo& info, + ConsoleHandle& console, + BotBaseContext& context, + double direction_facing, + uint8_t rgb_gap = 20, + ImageFloatBox area_to_check = {0, 0.3, 1.0, 0.40} + ); + + // push the olive forward. + // move forward a certain number of ticks, as per total_forward_distance + // always face a certain direction, as per direction_facing + // return number of ticks walked + uint16_t push_olive_forward( + const ProgramInfo& info, + ConsoleHandle& console, + BotBaseContext& context, + double direction_facing, + uint16_t total_forward_distance, + uint16_t push_olive = 75, + uint8_t rgb_gap = 20, + ImageFloatBox area_to_check = {0, 0.3, 1.0, 0.40} // {0, 0.15, 1, 0.7} + ); + + uint16_t walk_up_to_olive( + const ProgramInfo& info, + ConsoleHandle& console, + BotBaseContext& context, + double direction_facing, + uint8_t rgb_gap = 20, + ImageFloatBox area_to_check = {0, 0.3, 1.0, 0.40} + ); + + +protected: + VideoOverlaySet m_overlays; + const Color m_color; +}; + + + +} +} +} +#endif