diff --git a/components/esp_system/port/soc/esp32c5/clk.c b/components/esp_system/port/soc/esp32c5/clk.c
index 2c14e538b7..e5a3fecabf 100644
--- a/components/esp_system/port/soc/esp32c5/clk.c
+++ b/components/esp_system/port/soc/esp32c5/clk.c
@@ -96,7 +96,9 @@ __attribute__((weak)) void esp_clk_init(void)
     rtc_clk_fast_src_set(SOC_RTC_FAST_CLK_SRC_RC_FAST);
 #elif CONFIG_RTC_FAST_CLK_SRC_XTAL
     rtc_clk_fast_src_set(SOC_RTC_FAST_CLK_SRC_XTAL);
-    esp_sleep_sub_mode_config(ESP_SLEEP_RTC_FAST_USE_XTAL_MODE, true);
+    if (esp_sleep_sub_mode_dump_config(NULL)[ESP_SLEEP_RTC_FAST_USE_XTAL_MODE] == 0) {
+        esp_sleep_sub_mode_config(ESP_SLEEP_RTC_FAST_USE_XTAL_MODE, true);
+    }
 #else
 #error "No RTC fast clock source configured"
 #endif
diff --git a/components/esp_system/port/soc/esp32p4/clk.c b/components/esp_system/port/soc/esp32p4/clk.c
index 591a6a45a4..6acf5a8f87 100644
--- a/components/esp_system/port/soc/esp32p4/clk.c
+++ b/components/esp_system/port/soc/esp32p4/clk.c
@@ -94,7 +94,9 @@ __attribute__((weak)) void esp_clk_init(void)
     rtc_clk_fast_src_set(SOC_RTC_FAST_CLK_SRC_RC_FAST);
 #elif CONFIG_RTC_FAST_CLK_SRC_XTAL
     rtc_clk_fast_src_set(SOC_RTC_FAST_CLK_SRC_XTAL);
-    esp_sleep_sub_mode_config(ESP_SLEEP_RTC_FAST_USE_XTAL_MODE, true);
+    if (esp_sleep_sub_mode_dump_config(NULL)[ESP_SLEEP_RTC_FAST_USE_XTAL_MODE] == 0) {
+        esp_sleep_sub_mode_config(ESP_SLEEP_RTC_FAST_USE_XTAL_MODE, true);
+    }
 #else
 #error "No RTC fast clock source configured"
 #endif