Skip to content

Commit

Permalink
change Material detector box member fields into an array of boxes
Browse files Browse the repository at this point in the history
  • Loading branch information
@jw098
jw098 committed Jul 2, 2024
1 parent 54a378d commit 7cc2fd4
Show file tree
Hide file tree
Showing 4 changed files with 53 additions and 209 deletions.
6 changes: 3 additions & 3 deletions SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -288,11 +288,11 @@ void TestProgram::program(MultiSwitchProgramEnvironment& env, CancellableScope&
cout << (int)detector.detect_jobs(logger, env.inference_dispatcher(), screen) << endl;
#endif

#if 1
#if 0
ItemPrinterMaterialDetector detector(COLOR_RED, LANGUAGE);
detector.make_overlays(overlays);
detector.find_happiny_dust_row_num(env.inference_dispatcher(), console, context);
cout << (int)detector.detect_material_quantity(env.inference_dispatcher(), console, context, 2) << endl;
cout << (int)detector.find_happiny_dust_row_index(env.inference_dispatcher(), console, context) << endl;
// cout << (int)detector.detect_material_quantity(env.inference_dispatcher(), console, context, 2) << endl;
#endif

#if 0
Expand Down
205 changes: 39 additions & 166 deletions ...Programs/Source/PokemonSV/Inference/ItemPrinter/PokemonSV_ItemPrinterMaterialDetector.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,69 +53,31 @@ OCR::StringMatchResult MaterialNameReader::read_substring(
ItemPrinterMaterialDetector::ItemPrinterMaterialDetector(Color color, Language language)
: m_color(color)
, m_language(language)
, m_box_1_mat_value(0.39, 0.176, 0.025, 0.050)
, m_box_1_mat_quantity(0.485, 0.176, 0.037, 0.050)
, m_box_1_mat_name(0.090, 0.176, 0.275, 0.050)
, m_box_2_mat_value(0.39, 0.251, 0.025, 0.050)
, m_box_2_mat_quantity(0.485, 0.251, 0.037, 0.050)
, m_box_2_mat_name(0.090, 0.251, 0.275, 0.050)
, m_box_3_mat_value(0.39, 0.328, 0.025, 0.050)
, m_box_3_mat_quantity(0.485, 0.328, 0.037, 0.050)
, m_box_3_mat_name(0.090, 0.328, 0.275, 0.050)
, m_box_4_mat_value(0.39, 0.400, 0.025, 0.050)
, m_box_4_mat_quantity(0.485, 0.400, 0.037, 0.050)
, m_box_4_mat_name(0.090, 0.400, 0.275, 0.050)
, m_box_5_mat_value(0.39, 0.472, 0.025, 0.050)
, m_box_5_mat_quantity(0.485, 0.472, 0.037, 0.050)
, m_box_5_mat_name(0.090, 0.472, 0.275, 0.050)
, m_box_6_mat_value(0.39, 0.546, 0.025, 0.050)
, m_box_6_mat_quantity(0.485, 0.546, 0.037, 0.050)
, m_box_6_mat_name(0.090, 0.546, 0.275, 0.050)
, m_box_7_mat_value(0.39, 0.623, 0.025, 0.050)
, m_box_7_mat_quantity(0.485, 0.623, 0.037, 0.050)
, m_box_7_mat_name(0.090, 0.623, 0.275, 0.050)
, m_box_8_mat_value(0.39, 0.698, 0.025, 0.050)
, m_box_8_mat_quantity(0.485, 0.698, 0.037, 0.050)
, m_box_8_mat_name(0.090, 0.698, 0.275, 0.050)
, m_box_9_mat_value(0.39, 0.771, 0.025, 0.050)
, m_box_9_mat_quantity(0.485, 0.771, 0.037, 0.050)
, m_box_9_mat_name(0.090, 0.771, 0.275, 0.050)
, m_box_10_mat_value(0.39, 0.843, 0.025, 0.050)
, m_box_10_mat_quantity(0.485, 0.843, 0.037, 0.050)
, m_box_10_mat_name(0.090, 0.843, 0.275, 0.050)
, m_box_mat_value(Material_Boxes(ImageFloatBox(0.39, 0.176758, 0.025, 0.050)))
, m_box_mat_quantity(Material_Boxes(ImageFloatBox(0.485, 0.176758, 0.037, 0.050)))
, m_box_mat_name(Material_Boxes(ImageFloatBox(0.090, 0.176758, 0.275, 0.050)))
{}

std::array<ImageFloatBox, 10> ItemPrinterMaterialDetector::Material_Boxes(ImageFloatBox initial_box){
std::array<ImageFloatBox, 10> material_boxes;
double x = initial_box.x;
double width = initial_box.width;
double height = initial_box.height;
double initial_y = initial_box.y;
double y_spacing = 0.074219;
for (size_t i = 0; i < 10; i++){
double y = initial_y + i*y_spacing;
material_boxes[i] = ImageFloatBox(x, y, width, height);
}
return material_boxes;
}

void ItemPrinterMaterialDetector::make_overlays(VideoOverlaySet& items) const{
items.add(m_color, m_box_1_mat_value);
items.add(m_color, m_box_1_mat_quantity);
items.add(m_color, m_box_1_mat_name);
items.add(m_color, m_box_2_mat_value);
items.add(m_color, m_box_2_mat_quantity);
items.add(m_color, m_box_2_mat_name);
items.add(m_color, m_box_3_mat_value);
items.add(m_color, m_box_3_mat_quantity);
items.add(m_color, m_box_3_mat_name);
items.add(m_color, m_box_4_mat_value);
items.add(m_color, m_box_4_mat_quantity);
items.add(m_color, m_box_4_mat_name);
items.add(m_color, m_box_5_mat_value);
items.add(m_color, m_box_5_mat_quantity);
items.add(m_color, m_box_5_mat_name);
items.add(m_color, m_box_6_mat_value);
items.add(m_color, m_box_6_mat_quantity);
items.add(m_color, m_box_6_mat_name);
items.add(m_color, m_box_7_mat_value);
items.add(m_color, m_box_7_mat_quantity);
items.add(m_color, m_box_7_mat_name);
items.add(m_color, m_box_8_mat_value);
items.add(m_color, m_box_8_mat_quantity);
items.add(m_color, m_box_8_mat_name);
items.add(m_color, m_box_9_mat_value);
items.add(m_color, m_box_9_mat_quantity);
items.add(m_color, m_box_9_mat_name);
items.add(m_color, m_box_10_mat_value);
items.add(m_color, m_box_10_mat_quantity);
items.add(m_color, m_box_10_mat_name);
for (size_t i = 0; i < 10; i++){
items.add(m_color, m_box_mat_value[i]);
items.add(m_color, m_box_mat_quantity[i]);
items.add(m_color, m_box_mat_name[i]);
}
}

int16_t ItemPrinterMaterialDetector::read_number(
Expand Down Expand Up @@ -220,20 +182,20 @@ int16_t ItemPrinterMaterialDetector::read_number_black_or_white_text(
// return row number where Happiny dust is located on screen
// keep pressing DPAD_RIGHT until Happiny dust is on screen
// check each row on the screen for Happiny Dust
int8_t ItemPrinterMaterialDetector::find_happiny_dust_row_num(
int8_t ItemPrinterMaterialDetector::find_happiny_dust_row_index(
AsyncDispatcher& dispatcher,
ConsoleHandle& console, BotBaseContext& context
) const{
int8_t value_68_row_num;
int8_t value_68_row_index;
for (size_t c = 0; c < 10; c++){
context.wait_for_all_requests();
value_68_row_num = find_material_value_row_num(dispatcher, console, context, 68);
if (value_68_row_num != -1
&& detect_material_name(console, context, value_68_row_num) == "happiny-dust"
value_68_row_index = find_material_value_row_index(dispatcher, console, context, 68);
if (value_68_row_index != -1
&& detect_material_name(console, context, value_68_row_index) == "happiny-dust"
){
// found screen and row number with Happiny dust.
// std::cout << "Happiny dust found. Row number: " << std::to_string(value_68_row_num) << std::endl;
return value_68_row_num;
// std::cout << "Happiny dust found. Row number: " << std::to_string(value_68_row_index) << std::endl;
return value_68_row_index;
}
// keep searching for Happiny dust
pbf_press_dpad(context, DPAD_RIGHT, 20, 30);
Expand All @@ -247,17 +209,17 @@ int8_t ItemPrinterMaterialDetector::find_happiny_dust_row_num(

}

// detects the material name at the given row_num
// detects the material name at the given row_index
// MaterialNameReader has a very limited dictionary,
// so it can only reliably read the material names with 68% value
// (i.e. Ditto Goo, Happiny Dust, Magby Hair, Beldum Claw)
std::string ItemPrinterMaterialDetector::detect_material_name(
ConsoleHandle& console,
BotBaseContext& context,
int8_t row_num
int8_t row_index
) const{
VideoSnapshot snapshot = console.video().snapshot();
ImageFloatBox material_name_box = get_material_name_box(row_num);
ImageFloatBox material_name_box = m_box_mat_name[row_index];
ImageViewRGB32 material_name_image = extract_box_reference(snapshot, material_name_box);
const auto ocr_result = MaterialNameReader::instance().read_substring(
console, m_language,
Expand Down Expand Up @@ -287,21 +249,18 @@ std::string ItemPrinterMaterialDetector::detect_material_name(
}

// return row number that matches given material_value. else return -1
int8_t ItemPrinterMaterialDetector::find_material_value_row_num(
int8_t ItemPrinterMaterialDetector::find_material_value_row_index(
AsyncDispatcher& dispatcher,
ConsoleHandle& console,
BotBaseContext& context,
int16_t material_value
) const{
context.wait_for_all_requests();
VideoSnapshot snapshot = console.video().snapshot();
for (size_t c = 0; c < 10; c++){
context.wait_for_all_requests();
int8_t row_num = (int8_t)c+1;
ImageFloatBox material_value_box = get_material_value_box(row_num);

int16_t value = read_number(console, dispatcher, snapshot, material_value_box);
for (int8_t i = 0; i < 10; i++){
int16_t value = read_number(console, dispatcher, snapshot, m_box_mat_value[i]);
if (value == material_value){
return row_num;
return i;
}
}

Expand All @@ -314,100 +273,14 @@ int16_t ItemPrinterMaterialDetector::detect_material_quantity(
AsyncDispatcher& dispatcher,
ConsoleHandle& console,
BotBaseContext& context,
int8_t row_num
int8_t row_index
) const{
context.wait_for_all_requests();
VideoSnapshot snapshot = console.video().snapshot();
int16_t value = read_number(console, dispatcher, snapshot, get_material_quantity_box(row_num));
int16_t value = read_number(console, dispatcher, snapshot, m_box_mat_quantity[row_index]);
return value;
}

// return image float box for the material quantity at the given row_num
ImageFloatBox ItemPrinterMaterialDetector::get_material_quantity_box(int8_t row_num) const{
switch(row_num){
case 1:
return m_box_1_mat_quantity;
case 2:
return m_box_2_mat_quantity;
case 3:
return m_box_3_mat_quantity;
case 4:
return m_box_4_mat_quantity;
case 5:
return m_box_5_mat_quantity;
case 6:
return m_box_6_mat_quantity;
case 7:
return m_box_7_mat_quantity;
case 8:
return m_box_8_mat_quantity;
case 9:
return m_box_9_mat_quantity;
case 10:
return m_box_10_mat_quantity;
default:
return m_box_1_mat_quantity;
}
}

// return image float box for the material value at the given row_num
ImageFloatBox ItemPrinterMaterialDetector::get_material_value_box(int8_t row_num) const{
switch(row_num){
case 1:
return m_box_1_mat_value;
case 2:
return m_box_2_mat_value;
case 3:
return m_box_3_mat_value;
case 4:
return m_box_4_mat_value;
case 5:
return m_box_5_mat_value;
case 6:
return m_box_6_mat_value;
case 7:
return m_box_7_mat_value;
case 8:
return m_box_8_mat_value;
case 9:
return m_box_9_mat_value;
case 10:
return m_box_10_mat_value;
default:
return m_box_1_mat_value;
}
}

// return image float box for the material name at the given row_num
ImageFloatBox ItemPrinterMaterialDetector::get_material_name_box(int8_t row_num) const{
switch(row_num){
case 1:
return m_box_1_mat_name;
case 2:
return m_box_2_mat_name;
case 3:
return m_box_3_mat_name;
case 4:
return m_box_4_mat_name;
case 5:
return m_box_5_mat_name;
case 6:
return m_box_6_mat_name;
case 7:
return m_box_7_mat_name;
case 8:
return m_box_8_mat_name;
case 9:
return m_box_9_mat_name;
case 10:
return m_box_10_mat_name;
default:
return m_box_1_mat_name;
}
}






Expand Down
49 changes: 10 additions & 39 deletions ...alPrograms/Source/PokemonSV/Inference/ItemPrinter/PokemonSV_ItemPrinterMaterialDetector.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
#include "CommonFramework/ImageTools/ImageBoxes.h"
#include "CommonFramework/Inference/VisualDetector.h"
#include "CommonFramework/OCR/OCR_SmallDictionaryMatcher.h"
#include <array>

namespace PokemonAutomation{
class Logger;
Expand Down Expand Up @@ -46,12 +47,14 @@ class ItemPrinterMaterialDetector{

void make_overlays(VideoOverlaySet& items) const;

int8_t find_happiny_dust_row_num(
std::array<ImageFloatBox, 10> Material_Boxes(ImageFloatBox initial_box);

int8_t find_happiny_dust_row_index(
AsyncDispatcher& dispatcher,
ConsoleHandle& console, BotBaseContext& context
) const;

int8_t find_material_value_row_num(
int8_t find_material_value_row_index(
AsyncDispatcher& dispatcher,
ConsoleHandle& console,
BotBaseContext& context,
Expand All @@ -62,13 +65,13 @@ class ItemPrinterMaterialDetector{
AsyncDispatcher& dispatcher,
ConsoleHandle& console,
BotBaseContext& context,
int8_t row_num
int8_t row_index
) const;

std::string detect_material_name(
ConsoleHandle& console,
BotBaseContext& context,
int8_t row_num
int8_t row_index
) const;


Expand All @@ -84,45 +87,13 @@ class ItemPrinterMaterialDetector{
bool is_white_text
) const;

ImageFloatBox get_material_quantity_box(int8_t row_num) const;

ImageFloatBox get_material_value_box(int8_t row_num) const;

ImageFloatBox get_material_name_box(int8_t row_num) const;

private:
Color m_color;
Language m_language;
ImageFloatBox m_box_1_mat_value;
ImageFloatBox m_box_1_mat_quantity;
ImageFloatBox m_box_1_mat_name;
ImageFloatBox m_box_2_mat_value;
ImageFloatBox m_box_2_mat_quantity;
ImageFloatBox m_box_2_mat_name;
ImageFloatBox m_box_3_mat_value;
ImageFloatBox m_box_3_mat_quantity;
ImageFloatBox m_box_3_mat_name;
ImageFloatBox m_box_4_mat_value;
ImageFloatBox m_box_4_mat_quantity;
ImageFloatBox m_box_4_mat_name;
ImageFloatBox m_box_5_mat_value;
ImageFloatBox m_box_5_mat_quantity;
ImageFloatBox m_box_5_mat_name;
ImageFloatBox m_box_6_mat_value;
ImageFloatBox m_box_6_mat_quantity;
ImageFloatBox m_box_6_mat_name;
ImageFloatBox m_box_7_mat_value;
ImageFloatBox m_box_7_mat_quantity;
ImageFloatBox m_box_7_mat_name;
ImageFloatBox m_box_8_mat_value;
ImageFloatBox m_box_8_mat_quantity;
ImageFloatBox m_box_8_mat_name;
ImageFloatBox m_box_9_mat_value;
ImageFloatBox m_box_9_mat_quantity;
ImageFloatBox m_box_9_mat_name;
ImageFloatBox m_box_10_mat_value;
ImageFloatBox m_box_10_mat_quantity;
ImageFloatBox m_box_10_mat_name;
std::array<ImageFloatBox, 10> m_box_mat_value;
std::array<ImageFloatBox, 10> m_box_mat_quantity;
std::array<ImageFloatBox, 10> m_box_mat_name;
};


Expand Down
2 changes: 1 addition & 1 deletion SerialPrograms/Source/PokemonSV/Programs/ItemPrinter/PokemonSV_ItemPrinterRNG.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -765,7 +765,7 @@ uint32_t ItemPrinterRNG::check_num_happiny_dust(
env.log("Detected material selection.");
ItemPrinterMaterialDetector detector(COLOR_RED, LANGUAGE);

int8_t happiny_dust_row_num = detector.find_happiny_dust_row_num(env.inference_dispatcher(), env.console, context);
int8_t happiny_dust_row_num = detector.find_happiny_dust_row_index(env.inference_dispatcher(), env.console, context);
num_happiny_dust = detector.detect_material_quantity(env.inference_dispatcher(), env.console, context, happiny_dust_row_num);
pbf_mash_button(context, BUTTON_B, 100);
return num_happiny_dust;
Expand Down

0 comments on commit 7cc2fd4

Please sign in to comment.