feat(nvs_flash): Added Kconfig option contolling NVS heap allocation source

NVS configuration is extended with Kconfig option controlling RAM area for
NVS heap allocation. Either Internal RAM or SPIRAM can be chosen.
Tests were extended to check memory consumption from Internal and SPIRAM pool with respect
to the Kconfig option chosen.
Documentation was extended with notes related to NVS behavior in various situations.

components/nvs_flash/test_apps/main/CMakeLists.txt

@@ -1,6 +1,6 @@
 idf_component_register(SRC_DIRS "."
                        PRIV_REQUIRES cmock test_utils nvs_flash nvs_sec_provider
-                                     bootloader_support spi_flash
+                                     bootloader_support spi_flash esp_psram
                        EMBED_TXTFILES encryption_keys.bin partition_encrypted.bin
                                       partition_encrypted_hmac.bin sample.bin
                        WHOLE_ARCHIVE)

components/nvs_flash/test_apps/main/app_main.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Unlicense OR CC0-1.0
  */
@@ -12,6 +12,50 @@
 #endif
 #include "memory_checks.h"
 
+#include "esp_heap_caps.h"
+#include "time.h"
+
+// recorded heap free sizes (MALLOC_CAP_INTERNAL and MALLOC_CAP_SPIRAM)
+static size_t recorded_internal_heap_free_size = 0;
+static size_t recorded_spiram_heap_free_size = 0;
+
+// stores heap free sizes for internal and spiram pools
+void record_heap_free_sizes(void)
+{
+    recorded_internal_heap_free_size = heap_caps_get_free_size(MALLOC_CAP_INTERNAL);
+    recorded_spiram_heap_free_size = heap_caps_get_free_size(MALLOC_CAP_SPIRAM);
+}
+
+// returns difference between actual heap free size and recorded heap free size
+// parameter nvs_active_pool controls whether active or inactive heap will be examined
+// if CONFIG_NVS_ALLOCATE_CACHE_IN_SPIRAM is not set, active pool is MALLOC_CAP_INTERNAL and inactive is MALLOC_CAP_SPIRAM
+// if CONFIG_NVS_ALLOCATE_CACHE_IN_SPIRAM is set, active pool is MALLOC_CAP_SPIRAM and inactive is MALLOC_CAP_INTERNAL
+int32_t get_heap_free_difference(const bool nvs_active_pool)
+{
+    int32_t recorded_heap_free_size = 0;
+    int32_t actual_heap_free_size = 0;
+
+    bool evaluate_spiram = false;
+#ifdef CONFIG_NVS_ALLOCATE_CACHE_IN_SPIRAM
+    // here active means spiram
+    evaluate_spiram = nvs_active_pool;
+#else
+    // here active means internal
+    evaluate_spiram = !nvs_active_pool;
+#endif
+
+    if(evaluate_spiram) {
+        recorded_heap_free_size = recorded_spiram_heap_free_size;
+        actual_heap_free_size = heap_caps_get_free_size(MALLOC_CAP_SPIRAM);
+    }
+    else {
+        recorded_heap_free_size = recorded_internal_heap_free_size;
+        actual_heap_free_size = heap_caps_get_free_size(MALLOC_CAP_INTERNAL);
+    }
+
+    return actual_heap_free_size - recorded_heap_free_size;
+}
+
 /* setUp runs before every test */
 void setUp(void)
 {
@@ -47,7 +91,6 @@ void tearDown(void)
 
     test_utils_finish_and_evaluate_leaks(test_utils_get_leak_level(ESP_LEAK_TYPE_WARNING, ESP_COMP_LEAK_ALL),
             test_utils_get_leak_level(ESP_LEAK_TYPE_CRITICAL, ESP_COMP_LEAK_ALL));
-
 }
 
 static void test_task(void *pvParameters)

components/nvs_flash/test_apps/main/test_nvs.c

@@ -26,6 +26,9 @@
 #include "unity.h"
 #include "memory_checks.h"
 
+#include "esp_heap_caps.h"
+#include "esp_random.h"
+
 #ifdef CONFIG_NVS_ENCRYPTION
 #include "mbedtls/aes.h"
 #endif
@@ -34,8 +37,74 @@
 #include "esp_hmac.h"
 #endif
 
+extern void record_heap_free_sizes(void);
+extern int32_t get_heap_free_difference(const bool nvs_active_pool);
+
 static const char* TAG = "test_nvs";
 
+TEST_CASE("Kconfig option controls heap capability allocator for NVS", "[nvs_ram]")
+{
+    // number of keys used for test
+    const size_t max_key = 400;
+
+    char key_name[sizeof("keyXXXXX ")];
+    int32_t out_val = 0;
+    nvs_handle_t handle;
+
+    esp_err_t err = nvs_flash_init();
+    if (err == ESP_ERR_NVS_NO_FREE_PAGES || err == ESP_ERR_NVS_NEW_VERSION_FOUND) {
+        ESP_LOGW(TAG, "nvs_flash_init failed (0x%x), erasing partition and retrying", err);
+        ESP_ERROR_CHECK(nvs_flash_erase());
+        err = nvs_flash_init();
+    }
+    ESP_ERROR_CHECK( err );
+
+    TEST_ESP_OK(nvs_open("test_namespace1", NVS_READWRITE, &handle));
+    TEST_ESP_OK(nvs_erase_all(handle));
+    record_heap_free_sizes();
+
+    for(size_t i=0; i<max_key; i++) {
+        // prepare key name
+        sprintf(key_name, "key%05u", i);
+
+        TEST_ESP_OK(nvs_set_i32(handle, key_name, 666));
+        TEST_ESP_OK(nvs_commit(handle));
+    }
+
+    // after writing records, active pool should decrease while inactive should stay same
+    TEST_ASSERT_LESS_THAN_INT32(0, get_heap_free_difference(true));
+    TEST_ASSERT_EQUAL_INT32(0, get_heap_free_difference(false));
+    record_heap_free_sizes();
+
+    for(size_t i=0; i<max_key; i++) {
+        // prepare random key name
+        uint32_t key_num = esp_random();
+        key_num = key_num % max_key;
+        sprintf(key_name, "key%05lu", (uint32_t) key_num);
+
+        TEST_ESP_OK(nvs_get_i32(handle, key_name, &out_val));
+    }
+
+    // after reading records, no changes on heap are expected
+    TEST_ASSERT_EQUAL_INT32(0, get_heap_free_difference(true));
+    TEST_ASSERT_EQUAL_INT32(0, get_heap_free_difference(false));
+    record_heap_free_sizes();
+
+    TEST_ESP_OK(nvs_erase_all(handle));
+
+    // after erasing records, active pool should increase while inactive should stay same
+    TEST_ASSERT_GREATER_THAN_INT32(0, get_heap_free_difference(true));
+    TEST_ASSERT_EQUAL_INT32(0, get_heap_free_difference(false));
+    record_heap_free_sizes();
+
+    nvs_close(handle);
+    TEST_ESP_OK(nvs_flash_deinit());
+
+    // after deinit, active pool should increase by space occupied by the page management while inactive should only slightly increase
+    TEST_ASSERT_GREATER_THAN_INT32(0, get_heap_free_difference(true));
+    TEST_ASSERT_GREATER_OR_EQUAL_INT32(0, get_heap_free_difference(false));
+}
+
 TEST_CASE("Partition name no longer than 16 characters", "[nvs]")
 {
     const char *TOO_LONG_NAME = "0123456789abcdefg";

components/nvs_flash/test_apps/pytest_nvs_flash.py

@@ -1,6 +1,5 @@
 # SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
 import pytest
 from pytest_embedded_idf.dut import IdfDut
 
@@ -39,3 +38,9 @@ def test_nvs_flash_encr_flash_enc(dut: IdfDut) -> None:
     # Erase the nvs_key partition
     dut.serial.erase_partition('nvs_key')
     dut.run_all_single_board_cases()
+
+
+@pytest.mark.esp32
+@pytest.mark.psram
+def test_nvs_flash_ram(dut: IdfDut) -> None:
+    dut.run_all_single_board_cases(group='nvs_ram')

components/nvs_flash/test_apps/sdkconfig.ci.spiram

@@ -0,0 +1,7 @@
+# Restricting to ESP32
+CONFIG_IDF_TARGET="esp32"
+
+CONFIG_SPIRAM=y
+CONFIG_SPIRAM_USE_CAPS_ALLOC=y
+CONFIG_INT_WDT_TIMEOUT_MS=3000
+CONFIG_NVS_ALLOCATE_CACHE_IN_SPIRAM=y