From f5afb33c974a7fd0b233b65af8e615cdd797ab94 Mon Sep 17 00:00:00 2001 From: Developer-Butters Date: Fri, 10 Jul 2026 16:08:22 -0500 Subject: [PATCH 1/2] weather detector refactor Refactored the weather detector to use waterfill instead of direct object comparison. --- .../DevPrograms/TestProgramSwitch.cpp | 66 ++++- .../Inference/PokemonLZA_WeatherDetector.cpp | 252 ++++++++++++------ .../Inference/PokemonLZA_WeatherDetector.h | 23 +- 3 files changed, 233 insertions(+), 108 deletions(-) diff --git a/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp b/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp index 64c3094af4..1bf41d74c1 100644 --- a/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp +++ b/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp @@ -14,6 +14,7 @@ #include "Common/Cpp/Containers/FixedLimitVector.tpp" #include "Common/Cpp/Concurrency/BusyPeriodicRunner.h" #include "CommonFramework/Exceptions/OperationFailedException.h" +#include "CommonFramework/GlobalSettingsPanel.h" #include "CommonTools/Async/InferenceRoutines.h" #include "PokemonLA/Inference/PokemonLA_MountDetector.h" #include "Pokemon/Pokemon_Strings.h" @@ -173,12 +174,12 @@ #include "PokemonFRLG/Inference/PokemonFRLG_BattleSelectionArrowDetector.h" #include "Controllers/RumbleListener.h" #include "PokemonSwSh/Inference/PokemonSwSh_SelectionArrowFinder.h" +#include "PokemonLZA/Inference/PokemonLZA_WeatherDetector.h" #include "PokemonSwSh/Inference/PokemonSwSh_MainMenuDetector.h" #include "PokemonSwSh/Programs/PokemonSwSh_MenuNavigation.h" #include "PokemonLGPE/Inference/Battles/PokemonLGPE_BattleArrowDetector.h" - #include #include @@ -342,6 +343,69 @@ void TestProgram::program(MultiSwitchProgramEnvironment& env, CancellableScope& VideoOverlaySet overlays(overlay); +#if 0 + YCommIconDetector detector(COLOR_RED, true); + detector.make_overlays(overlays); + + auto snapshot = feed.snapshot(); + cout << detector.detect(snapshot) << endl; +#endif + auto snapshot = feed.snapshot(); +#if 1 + + using namespace PokemonLZA; + + const bool old_image_template_matching = PreloadSettings::debug().IMAGE_TEMPLATE_MATCHING; + PreloadSettings::debug().IMAGE_TEMPLATE_MATCHING = true; + cout << "Weather detector waterfill debug enabled (dump + stats)." << endl; + + const struct { + WeatherIconType type; + const char* name; + } weather[] = { + {WeatherIconType::Clear, "Clear"}, + {WeatherIconType::Sunny, "Sunny"}, + {WeatherIconType::Rain, "Rain"}, + {WeatherIconType::Cloudy, "Cloudy"}, + {WeatherIconType::Foggy, "Foggy"}, + {WeatherIconType::Rainbow, "Rainbow"}, + }; + bool weather_detected[sizeof(weather)/sizeof(weather[0])] = {}; + + for (size_t i = 0; i < sizeof(weather)/sizeof(weather[0]); i++) { + const auto& entry = weather[i]; + WeatherIconDetector detector(entry.type, &overlay); + + detector.make_overlays(overlays); + + bool detected = detector.detect(snapshot); + weather_detected[i] = detected; + + cout << entry.name + << ": " + << (detected ? "MATCH" : "NO MATCH") + << endl; + } + + PreloadSettings::debug().IMAGE_TEMPLATE_MATCHING = old_image_template_matching; + cout << "Weather detector waterfill debug restored." << endl; + + cout << endl; + cout << "Weather Summary:" << endl; + for (size_t i = 0; i < sizeof(weather)/sizeof(weather[0]); i++) { + cout << weather[i].name + << ": " + << (weather_detected[i] ? "MATCH" : "NO MATCH") + << endl; + } + + scope.wait_for(std::chrono::seconds(30)); +#endif + + + +// context->issue_gyro_accel_x(&scope, 1000ms, 1000ms, 0ms, 123); + #if 0 size_t min_area = 100; std::string path = "test.png"; diff --git a/SerialPrograms/Source/PokemonLZA/Inference/PokemonLZA_WeatherDetector.cpp b/SerialPrograms/Source/PokemonLZA/Inference/PokemonLZA_WeatherDetector.cpp index 62da2d5227..8066b3103d 100644 --- a/SerialPrograms/Source/PokemonLZA/Inference/PokemonLZA_WeatherDetector.cpp +++ b/SerialPrograms/Source/PokemonLZA/Inference/PokemonLZA_WeatherDetector.cpp @@ -1,83 +1,119 @@ #include "CommonFramework/Globals.h" #include "CommonFramework/ImageTools/ImageDiff.h" -#include "CommonTools/Images/ImageFilter.h" +#include "CommonTools/Images/WaterfillUtilities.h" +#include "CommonTools/ImageMatch/WaterfillTemplateMatcher.h" #include "PokemonLZA_WeatherDetector.h" +#include +#include +#include +#include namespace PokemonAutomation { namespace NintendoSwitch { namespace PokemonLZA { +namespace{ +const ImageFloatBox WEATHER_ICON_ROI(0.880000, 0.010000, 0.035800, 0.068000); -//----------------------------------------------------- -// Weather Info Table (two images per weather) -//----------------------------------------------------- +struct SupplementalTemplateCheck{ + const char* path; + ImageFloatBox box; + Color color; + double rmsd_threshold; +}; -static const WeatherTemplateInfo WEATHER_TABLE[] = { +const std::vector& supplemental_template_checks(WeatherIconType type){ + static const std::vector NONE = {}; + + static const std::vector RAIN = { + {"PokemonLZA/Weather/rain_cloud.png", ImageFloatBox(0.8865, 0.0265, 0.0210, 0.0270), COLOR_GREEN, 90.0}, + {"PokemonLZA/Weather/rain_drop.png", ImageFloatBox(0.8885, 0.0525, 0.0050, 0.0150), COLOR_BLUE, 90.0}, + }; + static const std::vector CLOUDY = { + {"PokemonLZA/Weather/cloudy_cloud.png", ImageFloatBox(0.8895, 0.0430, 0.0210, 0.0265), COLOR_GREEN, 90.0}, + {"PokemonLZA/Weather/cloudy_drop.png", ImageFloatBox(0.8915, 0.0255, 0.0050, 0.0100), COLOR_BLUE, 90.0}, + }; + static const std::vector RAINBOW = { + {"PokemonLZA/Weather/rainbow_cloud.png", ImageFloatBox(0.8840, 0.0465, 0.0140, 0.0165), COLOR_GREEN, 90.0}, + {"PokemonLZA/Weather/rainbow_arch.png", ImageFloatBox(0.8930, 0.0420, 0.0120, 0.0100), COLOR_BLUE, 90.0}, + }; + static const std::vector FOGGY = { + {"PokemonLZA/Weather/foggy_tray_1.png", ImageFloatBox(0.8893, 0.0487, 0.0218, 0.0080), COLOR_GREEN, 90.0}, + {"PokemonLZA/Weather/foggy_tray_2.png", ImageFloatBox(0.8880, 0.0555, 0.0225, 0.0080), COLOR_BLUE, 90.0}, + }; + + switch (type){ + case WeatherIconType::Rain: + return RAIN; + case WeatherIconType::Cloudy: + return CLOUDY; + case WeatherIconType::Rainbow: + return RAINBOW; + case WeatherIconType::Foggy: + return FOGGY; + default: + return NONE; + } +} - // Clear - { - "PokemonLZA/Weather/sun_drop.png", - "PokemonLZA/Weather/clear_core.png", - ImageFloatBox(0.8945, 0.0570, 0.0050, 0.0100), - ImageFloatBox(0.8910, 0.0360, 0.0110, 0.0210), - COLOR_RED, - COLOR_GREEN - }, - - // Sunny - { - "PokemonLZA/Weather/sun_drop.png", - "PokemonLZA/Weather/sunny_core.png", - ImageFloatBox(0.8945, 0.0570, 0.0050, 0.0100), - ImageFloatBox(0.8910, 0.0360, 0.0110, 0.0210), - COLOR_RED, - COLOR_GREEN - }, - - // Rain +class WeatherFullMatcher : public ImageMatch::WaterfillTemplateMatcher{ +public: + WeatherFullMatcher(const char* path, double max_rmsd) + : WaterfillTemplateMatcher(path, Color(0xff707070), Color(0xffffffff), 50) + , m_max_rmsd(max_rmsd) { - "PokemonLZA/Weather/rain_cloud.png", - "PokemonLZA/Weather/rain_drop.png", - ImageFloatBox(0.8865, 0.0265, 0.0210, 0.0270), - ImageFloatBox(0.8885, 0.0525, 0.0050, 0.0150), - COLOR_RED, - COLOR_GREEN - }, - - // Cloudy - { - "PokemonLZA/Weather/cloudy_cloud.png", - "PokemonLZA/Weather/cloudy_drop.png", - ImageFloatBox(0.8895, 0.0430, 0.0210, 0.0265), - ImageFloatBox(0.8915, 0.0255, 0.0050, 0.0100), - COLOR_RED, - COLOR_GREEN - }, - - // Foggy - { - "PokemonLZA/Weather/foggy_tray_1.png", - "PokemonLZA/Weather/foggy_tray_2.png", - ImageFloatBox(0.8893, 0.0487, 0.0218, 0.0080), - ImageFloatBox(0.8880, 0.0555, 0.0225, 0.0080), - COLOR_RED, - COLOR_GREEN - }, - - // Rainbow - { - "PokemonLZA/Weather/rainbow_cloud.png", - "PokemonLZA/Weather/rainbow_arch.png", - ImageFloatBox(0.8840, 0.0465, 0.0140, 0.0165), - ImageFloatBox(0.8930, 0.0420, 0.0120, 0.0100), - COLOR_RED, - COLOR_GREEN - }, -}; + m_aspect_ratio_lower = 0.60; + m_aspect_ratio_upper = 1.40; + m_area_ratio_lower = 0.55; + m_area_ratio_upper = 1.45; + } + + static const WeatherFullMatcher& clear(){ + static const WeatherFullMatcher matcher("PokemonLZA/Weather/clear_full.png", 100.0); + return matcher; + } + static const WeatherFullMatcher& sunny(){ + static const WeatherFullMatcher matcher("PokemonLZA/Weather/sunny_full.png", 100.0); + return matcher; + } + static const WeatherFullMatcher& rain(){ + static const WeatherFullMatcher matcher("PokemonLZA/Weather/rain_full.png", 100.0); + return matcher; + } + static const WeatherFullMatcher& cloudy(){ + static const WeatherFullMatcher matcher("PokemonLZA/Weather/cloudy_full.png", 100.0); + return matcher; + } + static const WeatherFullMatcher& foggy(){ + static const WeatherFullMatcher matcher("PokemonLZA/Weather/foggy_full.png", 100.0); + return matcher; + } + static const WeatherFullMatcher& rainbow(){ + static const WeatherFullMatcher matcher("PokemonLZA/Weather/rainbow_full.png", 100.0); + return matcher; + } + double m_max_rmsd; +}; -const WeatherTemplateInfo& weather_template_info(WeatherIconType icon){ - return WEATHER_TABLE[(int)icon]; +const WeatherFullMatcher& weather_full_matcher(WeatherIconType type){ + switch (type){ + case WeatherIconType::Clear: + return WeatherFullMatcher::clear(); + case WeatherIconType::Sunny: + return WeatherFullMatcher::sunny(); + case WeatherIconType::Rain: + return WeatherFullMatcher::rain(); + case WeatherIconType::Cloudy: + return WeatherFullMatcher::cloudy(); + case WeatherIconType::Foggy: + return WeatherFullMatcher::foggy(); + case WeatherIconType::Rainbow: + return WeatherFullMatcher::rainbow(); + default: + throw std::runtime_error("No weather full matcher for requested WeatherIconType"); + } +} } //----------------------------------------------------- @@ -86,37 +122,83 @@ const WeatherTemplateInfo& weather_template_info(WeatherIconType icon){ WeatherIconDetector::WeatherIconDetector(WeatherIconType type, VideoOverlay* overlay) { - m_info = &weather_template_info(type); - + m_type = type; if (overlay){ - m_overlay1.emplace(*overlay, m_info->box1, m_info->color1); - m_overlay2.emplace(*overlay, m_info->box2, m_info->color2); + m_overlay1.emplace(*overlay, WEATHER_ICON_ROI, COLOR_RED); } } void WeatherIconDetector::make_overlays(VideoOverlaySet& items) const { - items.add(m_info->color1, m_info->box1); - items.add(m_info->color2, m_info->box2); + items.add(COLOR_RED, WEATHER_ICON_ROI); + for (const auto& check : supplemental_template_checks(m_type)){ + items.add(check.color, check.box); + } } bool WeatherIconDetector::detect(const ImageViewRGB32& screen){ - - // Extract screen patches - ImageRGB32 c1 = extract_box_reference(screen, m_info->box1).copy(); - ImageRGB32 c2 = extract_box_reference(screen, m_info->box2).copy(); - - // Load templates - ImageRGB32 t1(RESOURCE_PATH() + m_info->path1); - ImageRGB32 t2(RESOURCE_PATH() + m_info->path2); - - if (t1.width() == 0 || t2.width() == 0){ + const WeatherFullMatcher& matcher = weather_full_matcher(m_type); + + const double scale = screen.height() / 1080.0; + const size_t min_area = (size_t)(scale * scale * 120.0); + + static const std::vector> FILTERS = { + {0xff707070, 0xffffffff}, + }; + + const bool full_match = match_template_by_waterfill( + screen.size(), + extract_box_reference(screen, WEATHER_ICON_ROI), + matcher, + FILTERS, + {min_area, SIZE_MAX}, + matcher.m_max_rmsd, + [](Kernels::Waterfill::WaterfillObject& object) -> bool { + (void)object; + return true; + } + ); + + if (!full_match){ return false; } - double rms1 = ImageMatch::pixel_RMSD(c1, t1); - double rms2 = ImageMatch::pixel_RMSD(c2, t2); + for (const auto& check : supplemental_template_checks(m_type)){ + ImageViewRGB32 candidate = extract_box_reference(screen, check.box); + ImageRGB32 templ(RESOURCE_PATH() + check.path); + if (templ.width() == 0 || templ.height() == 0){ + return false; + } + + auto compute_rmsd = [&](const ImageViewRGB32& image) -> double{ + return image.width() == templ.width() && image.height() == templ.height() + ? ImageMatch::pixel_RMSD(image, templ) + : ImageMatch::pixel_RMSD(image, templ.scale_to(image.width(), image.height())); + }; + + double rmsd = compute_rmsd(candidate); + + if (screen.height() < 1080){ + const int search_radius = std::min(candidate.width(), candidate.height()) <= 12 ? 2 : 1; + for (int dy = -search_radius; dy <= search_radius; dy++){ + for (int dx = -search_radius; dx <= search_radius; dx++){ + if (dx == 0 && dy == 0){ + continue; + } + ImageViewRGB32 shifted = extract_box_reference(screen, check.box, dx, dy); + if (shifted.width() != candidate.width() || shifted.height() != candidate.height()){ + continue; + } + rmsd = std::min(rmsd, compute_rmsd(shifted)); + } + } + } + + if (rmsd >= check.rmsd_threshold){ + return false; + } + } - return rms1 < 90 && rms2 < 90; + return true; } diff --git a/SerialPrograms/Source/PokemonLZA/Inference/PokemonLZA_WeatherDetector.h b/SerialPrograms/Source/PokemonLZA/Inference/PokemonLZA_WeatherDetector.h index 0d9bd10e89..30c9a8826c 100644 --- a/SerialPrograms/Source/PokemonLZA/Inference/PokemonLZA_WeatherDetector.h +++ b/SerialPrograms/Source/PokemonLZA/Inference/PokemonLZA_WeatherDetector.h @@ -2,11 +2,8 @@ #define PokemonAutomation_PokemonLZA_WeatherDetector_H #include -#include "Common/Cpp/Color.h" -#include "CommonFramework/ImageTools/ImageBoxes.h" #include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" #include "CommonTools/VisualDetector.h" -#include "CommonTools/ImageMatch/WaterfillTemplateMatcher.h" namespace PokemonAutomation { namespace NintendoSwitch { @@ -23,23 +20,6 @@ enum class WeatherIconType { Unknown, }; - -//----------------------------------------------------- -// Template Info Struct -//----------------------------------------------------- -struct WeatherTemplateInfo{ - const char* path1; // template for box1 - const char* path2; // template for box2 - ImageFloatBox box1; - ImageFloatBox box2; - Color color1; - Color color2; -}; - - -const WeatherTemplateInfo& weather_template_info(WeatherIconType icon); - - //----------------------------------------------------- // Detector //----------------------------------------------------- @@ -52,10 +32,9 @@ class WeatherIconDetector : public StaticScreenDetector { virtual bool detect(const ImageViewRGB32& screen) override; private: - const WeatherTemplateInfo* m_info; + WeatherIconType m_type; std::optional m_overlay1; - std::optional m_overlay2; }; From 479680ca102a014f76b1445b59523bb7ec727673 Mon Sep 17 00:00:00 2001 From: Developer-Butters Date: Mon, 3 Aug 2026 06:37:32 -0500 Subject: [PATCH 2/2] updated tests and weather detector added new testing infrastructure and made some improvements to RMSDs of the weather detector based on the results of the new testing infrastructure. --- .../Inference/PokemonLZA_WeatherDetector.cpp | 301 +++++++++++++++++- .../Inference/PokemonLZA_WeatherDetector.h | 4 + .../Source/PokemonLZA/PokemonLZA_Tests.cpp | 2 + 3 files changed, 300 insertions(+), 7 deletions(-) diff --git a/SerialPrograms/Source/PokemonLZA/Inference/PokemonLZA_WeatherDetector.cpp b/SerialPrograms/Source/PokemonLZA/Inference/PokemonLZA_WeatherDetector.cpp index 8066b3103d..012e02ca37 100644 --- a/SerialPrograms/Source/PokemonLZA/Inference/PokemonLZA_WeatherDetector.cpp +++ b/SerialPrograms/Source/PokemonLZA/Inference/PokemonLZA_WeatherDetector.cpp @@ -1,10 +1,18 @@ +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" #include "CommonFramework/Globals.h" #include "CommonFramework/ImageTools/ImageDiff.h" #include "CommonTools/Images/WaterfillUtilities.h" #include "CommonTools/ImageMatch/WaterfillTemplateMatcher.h" +#include "Tests/TestUtils.h" +#include "CommonFramework/VideoPipeline/VideoOverlay.h" #include "PokemonLZA_WeatherDetector.h" +#include #include +#include +#include +#include #include +#include #include #include @@ -30,8 +38,8 @@ const std::vector& supplemental_template_checks(Weath {"PokemonLZA/Weather/rain_drop.png", ImageFloatBox(0.8885, 0.0525, 0.0050, 0.0150), COLOR_BLUE, 90.0}, }; static const std::vector CLOUDY = { - {"PokemonLZA/Weather/cloudy_cloud.png", ImageFloatBox(0.8895, 0.0430, 0.0210, 0.0265), COLOR_GREEN, 90.0}, - {"PokemonLZA/Weather/cloudy_drop.png", ImageFloatBox(0.8915, 0.0255, 0.0050, 0.0100), COLOR_BLUE, 90.0}, + {"PokemonLZA/Weather/cloudy_cloud.png", ImageFloatBox(0.8895, 0.0430, 0.0210, 0.0265), COLOR_GREEN, 100.0}, + {"PokemonLZA/Weather/cloudy_drop.png", ImageFloatBox(0.8915, 0.0255, 0.0050, 0.0100), COLOR_BLUE, 100.0}, }; static const std::vector RAINBOW = { {"PokemonLZA/Weather/rainbow_cloud.png", ImageFloatBox(0.8840, 0.0465, 0.0140, 0.0165), COLOR_GREEN, 90.0}, @@ -56,6 +64,131 @@ const std::vector& supplemental_template_checks(Weath } } +const char* weather_full_template_path(WeatherIconType type){ + switch (type){ + case WeatherIconType::Clear: return "PokemonLZA/Weather/clear_full.png"; + case WeatherIconType::Sunny: return "PokemonLZA/Weather/sunny_full.png"; + case WeatherIconType::Rain: return "PokemonLZA/Weather/rain_full.png"; + case WeatherIconType::Cloudy: return "PokemonLZA/Weather/cloudy_full.png"; + case WeatherIconType::Foggy: return "PokemonLZA/Weather/foggy_full.png"; + case WeatherIconType::Rainbow: return "PokemonLZA/Weather/rainbow_full.png"; + default: return nullptr; + } +} + +const ImageRGB32& weather_full_template_image(WeatherIconType type){ + static const ImageRGB32 CLEAR(RESOURCE_PATH() + std::string(weather_full_template_path(WeatherIconType::Clear))); + static const ImageRGB32 SUNNY(RESOURCE_PATH() + std::string(weather_full_template_path(WeatherIconType::Sunny))); + static const ImageRGB32 RAIN(RESOURCE_PATH() + std::string(weather_full_template_path(WeatherIconType::Rain))); + static const ImageRGB32 CLOUDY(RESOURCE_PATH() + std::string(weather_full_template_path(WeatherIconType::Cloudy))); + static const ImageRGB32 FOGGY(RESOURCE_PATH() + std::string(weather_full_template_path(WeatherIconType::Foggy))); + static const ImageRGB32 RAINBOW(RESOURCE_PATH() + std::string(weather_full_template_path(WeatherIconType::Rainbow))); + + switch (type){ + case WeatherIconType::Clear: return CLEAR; + case WeatherIconType::Sunny: return SUNNY; + case WeatherIconType::Rain: return RAIN; + case WeatherIconType::Cloudy: return CLOUDY; + case WeatherIconType::Foggy: return FOGGY; + case WeatherIconType::Rainbow: return RAINBOW; + default: return CLEAR; + } +} + +double full_template_rmsd(const ImageViewRGB32& screen, WeatherIconType type){ + ImageViewRGB32 candidate = extract_box_reference(screen, WEATHER_ICON_ROI); + const ImageRGB32& templ = weather_full_template_image(type); + if (templ.width() == 0 || templ.height() == 0 || candidate.width() == 0 || candidate.height() == 0){ + return std::numeric_limits::infinity(); + } + return candidate.width() == templ.width() && candidate.height() == templ.height() + ? ImageMatch::pixel_RMSD(candidate, templ) + : ImageMatch::pixel_RMSD(candidate, templ.scale_to(candidate.width(), candidate.height())); +} +} + +const std::array& weather_test_types(){ + static const std::array TYPES = { + WeatherIconType::Clear, + WeatherIconType::Sunny, + WeatherIconType::Rain, + WeatherIconType::Cloudy, + WeatherIconType::Foggy, + WeatherIconType::Rainbow, + }; + return TYPES; +} + +std::string weather_name(WeatherIconType type){ + switch (type){ + case WeatherIconType::Clear: return "Clear"; + case WeatherIconType::Sunny: return "Sunny"; + case WeatherIconType::Rain: return "Rain"; + case WeatherIconType::Cloudy: return "Cloudy"; + case WeatherIconType::Foggy: return "Foggy"; + case WeatherIconType::Rainbow: return "Rainbow"; + default: return "Unknown"; + } +} + +std::string weather_detection_status_string(const std::vector>& statuses){ + std::ostringstream ss; + ss << "Detection map: "; + for (size_t c = 0; c < statuses.size(); c++){ + if (c > 0){ + ss << ", "; + } + ss << weather_name(statuses[c].first) << "=" << (statuses[c].second ? "true" : "false"); + } + return ss.str(); +} + +WeatherIconType weather_from_filename(const std::string& image){ + std::string lower = image; + std::transform(lower.begin(), lower.end(), lower.begin(), [](unsigned char c){ return (char)std::tolower(c); }); + + if (lower.find("rainbow") != std::string::npos){ + return WeatherIconType::Rainbow; + } + if (lower.find("sunny") != std::string::npos){ + return WeatherIconType::Sunny; + } + if (lower.find("cloud") != std::string::npos){ + return WeatherIconType::Cloudy; + } + if (lower.find("clear") != std::string::npos){ + return WeatherIconType::Clear; + } + if (lower.find("fog") != std::string::npos){ + return WeatherIconType::Foggy; + } + if (lower.find("rain") != std::string::npos){ + return WeatherIconType::Rain; + } + return WeatherIconType::Unknown; +} + +bool expected_result_from_filename(const std::string& image){ + std::string lower = image; + std::transform(lower.begin(), lower.end(), lower.begin(), [](unsigned char c){ return (char)std::tolower(c); }); + + if (lower.find("_false") != std::string::npos){ + return false; + } + return true; +} + +std::string weather_list_string(const std::vector& weathers){ + std::ostringstream ss; + for (size_t c = 0; c < weathers.size(); c++){ + if (c > 0){ + ss << ", "; + } + ss << weather_name(weathers[c]); + } + return ss.str(); +} + class WeatherFullMatcher : public ImageMatch::WaterfillTemplateMatcher{ public: WeatherFullMatcher(const char* path, double max_rmsd) @@ -69,11 +202,11 @@ class WeatherFullMatcher : public ImageMatch::WaterfillTemplateMatcher{ } static const WeatherFullMatcher& clear(){ - static const WeatherFullMatcher matcher("PokemonLZA/Weather/clear_full.png", 100.0); + static const WeatherFullMatcher matcher("PokemonLZA/Weather/clear_full.png", 115.0); return matcher; } static const WeatherFullMatcher& sunny(){ - static const WeatherFullMatcher matcher("PokemonLZA/Weather/sunny_full.png", 100.0); + static const WeatherFullMatcher matcher("PokemonLZA/Weather/sunny_full.png", 120.0); return matcher; } static const WeatherFullMatcher& rain(){ @@ -81,7 +214,7 @@ class WeatherFullMatcher : public ImageMatch::WaterfillTemplateMatcher{ return matcher; } static const WeatherFullMatcher& cloudy(){ - static const WeatherFullMatcher matcher("PokemonLZA/Weather/cloudy_full.png", 100.0); + static const WeatherFullMatcher matcher("PokemonLZA/Weather/cloudy_full.png", 110.0); return matcher; } static const WeatherFullMatcher& foggy(){ @@ -114,8 +247,6 @@ const WeatherFullMatcher& weather_full_matcher(WeatherIconType type){ throw std::runtime_error("No weather full matcher for requested WeatherIconType"); } } -} - //----------------------------------------------------- // Detector //----------------------------------------------------- @@ -162,6 +293,32 @@ bool WeatherIconDetector::detect(const ImageViewRGB32& screen){ return false; } + if (m_type == WeatherIconType::Clear || m_type == WeatherIconType::Sunny || m_type == WeatherIconType::Cloudy){ + const double clear_rmsd = full_template_rmsd(screen, WeatherIconType::Clear); + const double sunny_rmsd = full_template_rmsd(screen, WeatherIconType::Sunny); + const double cloudy_rmsd = full_template_rmsd(screen, WeatherIconType::Cloudy); + + switch (m_type){ + case WeatherIconType::Clear: + if (clear_rmsd + 4.0 >= sunny_rmsd || clear_rmsd > cloudy_rmsd + 5.0){ + return false; + } + break; + case WeatherIconType::Sunny: + if (sunny_rmsd > clear_rmsd + 12.0 || sunny_rmsd > cloudy_rmsd + 6.0){ + return false; + } + break; + case WeatherIconType::Cloudy: + if (cloudy_rmsd > sunny_rmsd || cloudy_rmsd > clear_rmsd + 5.0){ + return false; + } + break; + default: + break; + } + } + for (const auto& check : supplemental_template_checks(m_type)){ ImageViewRGB32 candidate = extract_box_reference(screen, check.box); ImageRGB32 templ(RESOURCE_PATH() + check.path); @@ -202,6 +359,136 @@ bool WeatherIconDetector::detect(const ImageViewRGB32& screen){ } + +class Test_WeatherIconDetector : public UnitTest{ +public: + Test_WeatherIconDetector( + const std::string& image, + WeatherIconType expected_weather, + bool expected_result + ) + : UnitTest("PokemonPLZA::WeatherIconDetector - " + image + " [" + weather_name(expected_weather) + "]") + , m_image(UNIT_TEST_RESOURCE_PATH() + image) + , m_expected_weather(expected_weather) + , m_expected_result(expected_result) + {} + + virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ + (void)scope; + + DummyVideoOverlay overlay; + ImageRGB32 image(m_image); + if (image.width() == 0 || image.height() == 0){ + return "Failed to load test image: " + m_image; + } + + std::vector detected_weathers; + std::vector> statuses; + for (WeatherIconType type : weather_test_types()){ + WeatherIconDetector detector(type, &overlay); + const bool matched = detector.detect(image); + statuses.emplace_back(type, matched); + if (matched){ + detected_weathers.push_back(type); + } + } + const std::string status_string = weather_detection_status_string(statuses); + logger.log(status_string, COLOR_BLUE); + + if (!m_expected_result){ + bool expected_found = false; + for (WeatherIconType weather : detected_weathers){ + if (weather == m_expected_weather){ + expected_found = true; + break; + } + } + TEST_RESULT_COMPONENT_EQUAL(expected_found, false, "expected weather present"); + if (expected_found){ + return "Expected weather should be absent, but " + weather_name(m_expected_weather) + " was detected. " + status_string; + } + return !expected_found; + } + + TEST_RESULT_COMPONENT_EQUAL(detected_weathers.size(), (size_t)1, "num detected weather types"); + + if (detected_weathers.empty()){ + if (m_expected_weather != WeatherIconType::Unknown){ + return "Expected " + weather_name(m_expected_weather) + ", but detected no weather. " + status_string; + } + return "Detected no weather. " + status_string; + } + + if (detected_weathers.size() > 1){ + return "Detected multiple weather types: " + weather_list_string(detected_weathers) + ". " + status_string; + } + + const WeatherIconType detected = detected_weathers[0]; + + if (m_expected_weather != WeatherIconType::Unknown){ + if (detected != m_expected_weather){ + return "Expected " + weather_name(m_expected_weather) + ", but detected " + weather_name(detected) + ". " + status_string; + } + TEST_RESULT_COMPONENT_EQUAL_STR(weather_name(detected), weather_name(m_expected_weather), "weather type"); + return true; + } + + return true; + } + +private: + std::string m_image; + WeatherIconType m_expected_weather; + bool m_expected_result; +}; + + + +void add_tests_WeatherDetector(UnitTestDatabase& database){ + auto add = [&](const char* filename){ + const std::string image = "PokemonLZA/WeatherDetector/" + std::string(filename); + database.add(image, weather_from_filename(filename), expected_result_from_filename(filename)); + }; + + add("clear_1_True.png"); + add("clear_2_True.png"); + add("clear_dark_1_True.png"); + add("clear_dark_2_True.png"); + add("clear_dark_map_True.png"); + add("clear_overview_True.png"); + add("clear_wild_zone_1_2_True.png"); + add("clear_wild_zone_20_1_True.png"); + add("cloudy_1_True.png"); + add("cloudy_hyperspace_rogue_mega_arena_True.png"); + add("cloudy_zoomed_1_True.png"); + add("cloudy_zoomed_2_True.png"); + add("fog_1_True.png"); + add("fog_wild_zone_1_True.png"); + add("fog_wild_zone_20_True.png"); + add("french_clear_place_centrale_True.png"); + add("french_clear_wild_zone_20_2_True.png"); + add("french_cloudy_wild_zone_20_True.png"); + add("french_rainbow_wild_zone_20_True.png"); + add("french_rain_wild_zone_20_True.png"); + add("french_sunny_wild_zone_20_1_True.png"); + add("japanese_clear_1_True.jpg"); + add("japanese_clear_2_True.jpg"); + add("japanese_sunny_True.png"); + add("japanese_sunny_wild_zone_17_True.png"); + add("rainbow_1_True.png"); + add("rainbow_True.png"); + add("rain_1_True.png"); + add("rain_hyperspace_wild_zone_True.png"); + add("rain_wild_zone_1_True.png"); + add("rain_wild_zone_6_True.png"); + add("sunny_3_True.png"); + add("sunny_wild_zone_1_True.png"); + add("sunny_wild_zone_8_True.png"); + add("sunny_zoomed_1_True.jpg"); + add("sunny_zoomed_2_True.jpg"); +} + + } } } diff --git a/SerialPrograms/Source/PokemonLZA/Inference/PokemonLZA_WeatherDetector.h b/SerialPrograms/Source/PokemonLZA/Inference/PokemonLZA_WeatherDetector.h index 30c9a8826c..2daec48337 100644 --- a/SerialPrograms/Source/PokemonLZA/Inference/PokemonLZA_WeatherDetector.h +++ b/SerialPrograms/Source/PokemonLZA/Inference/PokemonLZA_WeatherDetector.h @@ -2,6 +2,7 @@ #define PokemonAutomation_PokemonLZA_WeatherDetector_H #include +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" #include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" #include "CommonTools/VisualDetector.h" @@ -38,6 +39,9 @@ class WeatherIconDetector : public StaticScreenDetector { }; +void add_tests_WeatherDetector(UnitTestDatabase& database); + + } } } diff --git a/SerialPrograms/Source/PokemonLZA/PokemonLZA_Tests.cpp b/SerialPrograms/Source/PokemonLZA/PokemonLZA_Tests.cpp index 3974da53ef..16c6490e28 100644 --- a/SerialPrograms/Source/PokemonLZA/PokemonLZA_Tests.cpp +++ b/SerialPrograms/Source/PokemonLZA/PokemonLZA_Tests.cpp @@ -6,6 +6,7 @@ #include "Common/Cpp/TestRunners/UnitTestDatabase.h" #include "Inference/PokemonLZA_DialogDetector.h" +#include "Inference/PokemonLZA_WeatherDetector.h" namespace PokemonAutomation{ namespace NintendoSwitch{ @@ -15,6 +16,7 @@ namespace PokemonLZA{ void add_tests(UnitTestDatabase& database){ add_tests_DialogDetector(database); + add_tests_WeatherDetector(database); }