Merge branch 'feat/add-heap-walker-api' into 'master'

feat(heap): Add walker to the heap component

Closes IDF-9189

See merge request espressif/esp-idf!29047

components/heap/Kconfig

@@ -119,15 +119,6 @@ menu "Heap memory debugging"
             features will be added and bugs will be fixed in the IDF source
             but cannot be synced to ROM.
 
-    config HEAP_TLSF_CHECK_PATCH
-        bool "Patch the tlsf_check_pool() for ROM HEAP TLSF implementation"
-        depends on HEAP_TLSF_USE_ROM_IMPL && IDF_TARGET_ESP32C2 && ESP32C2_REV_MIN_FULL < 200
-        default y
-        help
-            ROM does not contain the patch of tlsf_check_pool() allowing perform
-            the integrity checking on used blocks. The patch to allow such check
-            needs to be applied.
-
     config HEAP_PLACE_FUNCTION_INTO_FLASH
         bool "Force the entire heap component to be placed in flash memory"
         depends on !HEAP_TLSF_USE_ROM_IMPL

components/heap/heap_caps.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -15,6 +15,10 @@
 #include "heap_private.h"
 #include "esp_system.h"
 
+#if CONFIG_HEAP_TLSF_USE_ROM_IMPL
+#include "esp_rom_multi_heap.h"
+#endif
+
 #ifdef CONFIG_HEAP_USE_HOOKS
 #define CALL_HOOK(hook, ...) {      \
     if (hook != NULL) {             \
@@ -860,3 +864,46 @@ void *heap_caps_aligned_calloc(size_t alignment, size_t n, size_t size, uint32_t
 
     return ptr;
 }
+
+typedef struct walker_data {
+        void *opaque_ptr;
+        heap_caps_walker_cb_t cb_func;
+        heap_t *heap;
+} walker_data_t;
+
+__attribute__((noinline)) static bool heap_caps_walker(void* block_ptr, size_t block_size, int block_used, void *user_data)
+{
+    walker_data_t *walker_data = (walker_data_t*)user_data;
+
+    walker_heap_into_t heap_info = {
+        (intptr_t)walker_data->heap->start,
+        (intptr_t)walker_data->heap->end
+    };
+    walker_block_info_t block_info = {
+        block_ptr,
+        block_size,
+        (bool)block_used
+    };
+
+    return walker_data->cb_func(heap_info, block_info, walker_data->opaque_ptr);
+}
+
+void heap_caps_walk(uint32_t caps, heap_caps_walker_cb_t walker_func, void *user_data)
+{
+    assert(walker_func != NULL);
+
+    bool all_heaps = caps & MALLOC_CAP_INVALID;
+    heap_t *heap;
+    SLIST_FOREACH(heap, &registered_heaps, next) {
+        if (heap->heap != NULL
+            && (all_heaps || (get_all_caps(heap) & caps) == caps)) {
+            walker_data_t walker_data = {user_data, walker_func, heap};
+            multi_heap_walk(heap->heap, heap_caps_walker, &walker_data);
+        }
+    }
+}
+
+void heap_caps_walk_all(heap_caps_walker_cb_t walker_func, void *user_data)
+{
+    heap_caps_walk(MALLOC_CAP_INVALID, walker_func, user_data);
+}

components/heap/include/esp_heap_caps.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2019-2023 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2019-2024 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -444,6 +444,56 @@ void heap_caps_dump_all(void);
  */
 size_t heap_caps_get_allocated_size( void *ptr );
 
+/**
+ * @brief Structure used to store heap related data passed to
+ * the walker callback function
+ */
+typedef struct walker_heap_info {
+    intptr_t start; ///< Start address of the heap in which the block is located
+    intptr_t end; ///< End address of the heap in which the block is located
+} walker_heap_into_t;
+
+/**
+ * @brief Structure used to store block related data passed to
+ * the walker callback function
+ */
+typedef struct walker_block_info {
+    void *ptr; ///< Pointer to the block data
+    size_t size; ///< The size of the block
+    bool used; ///< Block status. True: used, False: free
+} walker_block_info_t;
+
+/**
+ * @brief Function callback used to get information of memory block
+ * during calls to heap_caps_walk or heap_caps_walk_all
+ *
+ * @param heap_info See walker_heap_into_t
+ * @param block_info See walker_block_info_t
+ * @param user_data Opaque pointer to user defined data
+ *
+ * @return True to proceed with the heap traversal
+ *         False to stop the traversal of the current heap and continue
+ *         with the traversal of the next heap (if any)
+ */
+typedef bool (*heap_caps_walker_cb_t)(walker_heap_into_t heap_info, walker_block_info_t block_info, void *user_data);
+
+/**
+ * @brief Function called to walk through the heaps with the given set of capabilities
+ *
+ * @param caps The set of capabilities assigned to the heaps to walk through
+ * @param walker_func Callback called for each block of the heaps being traversed
+ * @param user_data Opaque pointer to user defined data
+ */
+void heap_caps_walk(uint32_t caps, heap_caps_walker_cb_t walker_func, void *user_data);
+
+/**
+ * @brief Function called to walk through all heaps defined by the heap component
+ *
+ * @param walker_func Callback called for each block of the heaps being traversed
+ * @param user_data Opaque pointer to user defined data
+ */
+void heap_caps_walk_all(heap_caps_walker_cb_t walker_func, void *user_data);
+
 #ifdef __cplusplus
 }
 #endif

components/heap/include/multi_heap.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -207,6 +207,29 @@ size_t multi_heap_reset_minimum_free_bytes(multi_heap_handle_t heap);
  */
 void multi_heap_restore_minimum_free_bytes(multi_heap_handle_t heap, const size_t new_minimum_free_bytes_value);
 
+/**
+ * @brief Callback called when walking the given heap blocks of memory
+ *
+ * @param block_ptr Pointer to the block data
+ * @param block_size The size of the block
+ * @param block_used Block status. 0: free, 1: allocated
+ * @param user_data Opaque pointer to user defined data
+ *
+ * @return True if the walker is expected to continue the heap traversal
+ *         False if the walker is expected to stop the traversal of the heap
+ */
+typedef bool (*multi_heap_walker_cb_t)(void *block_ptr, size_t block_size, int block_used, void *user_data);
+
+/**
+ * @brief Call the tlsf_walk_pool function of the heap given as parameter with
+ * the walker function passed as parameter
+ *
+ * @param heap The heap to traverse
+ * @param walker_func The walker to trigger on each block of the heap
+ * @param user_data Opaque pointer to user defined data
+ */
+void multi_heap_walk(multi_heap_handle_t heap, multi_heap_walker_cb_t walker_func, void *user_data);
+
 #ifdef __cplusplus
 }
 #endif

components/heap/multi_heap.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -355,13 +355,11 @@ bool multi_heap_check(multi_heap_handle_t heap, bool print_errors)
     return valid;
 }
 
-__attribute__((noinline)) static void multi_heap_dump_tlsf(void* ptr, size_t size, int used, void* user)
+__attribute__((noinline)) static bool multi_heap_dump_tlsf(void *ptr, size_t size, int used, void *user)
 {
     (void)user;
-    MULTI_HEAP_STDERR_PRINTF("Block %p data, size: %d bytes, Free: %s \n",
-                            (void *)ptr,
-                            size,
-                            used ? "No" : "Yes");
+    MULTI_HEAP_STDERR_PRINTF("Block %p data, size: %d bytes, Free: %s \n", (void *)ptr, size, used ? "No" : "Yes");
+    return true;
 }
 
 void multi_heap_dump(multi_heap_handle_t heap)
@@ -392,7 +390,7 @@ size_t multi_heap_minimum_free_size_impl(multi_heap_handle_t heap)
     return heap->minimum_free_bytes;
 }
 
-__attribute__((noinline)) static void multi_heap_get_info_tlsf(void* ptr, size_t size, int used, void* user)
+__attribute__((noinline)) static bool multi_heap_get_info_tlsf(void* ptr, size_t size, int used, void* user)
 {
     multi_heap_info_t *info = user;
 
@@ -407,6 +405,7 @@ __attribute__((noinline)) static void multi_heap_get_info_tlsf(void* ptr, size_t
     }
 
     info->total_blocks++;
+    return true;
 }
 
 void multi_heap_get_info_impl(multi_heap_handle_t heap, multi_heap_info_t *info)
@@ -431,6 +430,15 @@ void multi_heap_get_info_impl(multi_heap_handle_t heap, multi_heap_info_t *info)
     multi_heap_internal_unlock(heap);
 }
 
+void multi_heap_walk(multi_heap_handle_t heap, multi_heap_walker_cb_t walker_func, void *user_data)
+{
+    assert(heap != NULL);
+
+    multi_heap_internal_lock(heap);
+    tlsf_walk_pool(tlsf_get_pool(heap->heap_data), walker_func, user_data);
+    multi_heap_internal_unlock(heap);
+}
+
 #endif // CONFIG_HEAP_TLSF_USE_ROM_IMPL
 
 size_t multi_heap_reset_minimum_free_bytes(multi_heap_handle_t heap)

components/heap/test_apps/heap_tests/main/CMakeLists.txt

@@ -8,7 +8,8 @@ set(src_test "test_heap_main.c"
              "test_malloc.c"
              "test_realloc.c"
              "test_runtime_heap_reg.c"
-             "test_task_tracking.c")
+             "test_task_tracking.c"
+             "test_walker.c")
 
 idf_component_register(SRCS ${src_test}
                        INCLUDE_DIRS "."

components/heap/test_apps/heap_tests/main/test_walker.c

@@ -0,0 +1,101 @@
+/*
+ * SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Unlicense OR CC0-1.0
+ */
+#include "unity.h"
+#include "stdio.h"
+
+#include "esp_heap_caps.h"
+#include "esp_rom_sys.h"
+#define PASS_CODE 0x9876BAAB
+#define ALLOC_SIZE 16
+
+static void calculate_block_metadata_size(size_t *header, size_t *footer)
+{
+    *header = 0;
+    *footer = 0;
+#if !CONFIG_HEAP_POISONING_DISABLED
+    *header += 8; // sizeof(poison_head_t)
+    *footer += 4; // sizeof(poison_tail_t)
+#endif
+#if CONFIG_HEAP_TASK_TRACKING
+    *header += 4; // sizeof(TaskHandle_t)
+#endif
+}
+
+static bool block_found = false;
+static bool heap_corrupted = false;
+static bool heap_walker(walker_heap_into_t heap_info, walker_block_info_t block_info, void* user_data)
+{
+    if ((intptr_t)heap_info.end - (intptr_t)block_info.ptr < block_info.size)
+    {
+        heap_corrupted = true;
+        return false;
+    }
+
+    TEST_ASSERT(*(uint32_t*)user_data == PASS_CODE);
+    TEST_ASSERT_NOT_NULL(block_info.ptr);
+
+    size_t metadata_size_head = 0;
+    size_t metadata_size_tail = 0;
+    calculate_block_metadata_size(&metadata_size_head, &metadata_size_tail);
+
+    /* look for the first 4 bytes pass code to identify the memory allocated in the test */
+    const uint32_t pass_code = *((uint32_t*)block_info.ptr + (metadata_size_head / sizeof(uint32_t)));
+    if (pass_code == PASS_CODE) {
+        TEST_ASSERT(block_info.size == ALLOC_SIZE + metadata_size_head + metadata_size_tail);
+        TEST_ASSERT_TRUE(block_info.used);
+        block_found = true;
+    }
+
+    return true;
+}
+
+/* This test assures the proper functioning of heap_caps_walk and heap_caps_walk_all
+ * when a corruption occurs in a random heap. The callback which detects the corruption
+ * returns false to the walker which should result in the interruption of the heap traversal
+ * by the walker instead of a crash.
+ */
+TEST_CASE("heap walker", "[heap]")
+{
+    /* Allocate memory using the MALLOC_CAP_DEFAULT capability */
+    void *default_ptr = heap_caps_malloc(ALLOC_SIZE, MALLOC_CAP_DEFAULT);
+    TEST_ASSERT_NOT_NULL(default_ptr);
+
+    /* Write the pass code in the first word of the allocated memory */
+    *((uint32_t*)default_ptr) = (uint32_t)PASS_CODE;
+
+    /* call the heap_caps_walker to make sure the hook function is triggered
+     * and check that the allocated memory is found while walking the heap */
+    uint32_t user_code = PASS_CODE;
+    heap_caps_walk_all(heap_walker, &user_code);
+    TEST_ASSERT_TRUE(block_found);
+}
+
+/* This test assures the proper functioning of heap_caps_walk and heap_caps_walk_all
+ */
+TEST_CASE("heap walker corrupted heap detection", "[heap]")
+{
+    /* Allocate memory using the MALLOC_CAP_DEFAULT capability */
+    void *default_ptr = heap_caps_malloc(ALLOC_SIZE, MALLOC_CAP_DEFAULT);
+    TEST_ASSERT_NOT_NULL(default_ptr);
+
+    size_t metadata_size_head = 0;
+    size_t metadata_size_tail = 0;
+    calculate_block_metadata_size(&metadata_size_head, &metadata_size_tail);
+    (void)metadata_size_tail;
+
+    /* corrupting the size field of the block metadata with a size bigger
+     * than the heap itself */
+    *((uint32_t*)default_ptr - (metadata_size_head / 4) - 1) = 0xFF000000;
+
+    /* Write the pass code in the first word of the allocated memory */
+    *((uint32_t*)default_ptr) = (uint32_t)PASS_CODE;
+
+    /* call the heap_caps_walker to make sure the hook function is triggered
+     * and check that the allocated memory is found while walking the heap */
+    uint32_t user_code = PASS_CODE;
+    heap_caps_walk_all(heap_walker, &user_code);
+    TEST_ASSERT_TRUE(heap_corrupted);
+}