diff --git a/cmake/preload/toolchains/set_flags.cmake b/cmake/preload/toolchains/set_flags.cmake
index bd6e24070..7f208db3b 100644
--- a/cmake/preload/toolchains/set_flags.cmake
+++ b/cmake/preload/toolchains/set_flags.cmake
@@ -2,6 +2,7 @@
 get_property(IS_IN_TRY_COMPILE GLOBAL PROPERTY IN_TRY_COMPILE)
 foreach(LANG IN ITEMS C CXX ASM)
     set(CMAKE_${LANG}_FLAGS_INIT "${ARM_TOOLCHAIN_COMMON_FLAGS}")
+    unset(CMAKE_${LANG}_FLAGS_DEBUG CACHE)
     if (PICO_DEOPTIMIZED_DEBUG)
         set(CMAKE_${LANG}_FLAGS_DEBUG_INIT "-O0")
     else()