Add logic to make external RAM usable with malloc()

components/esp32/Kconfig

@@ -47,7 +47,7 @@ config SPIRAM_BOOT_INIT
 
 choice SPIRAM_USE
     prompt "SPI RAM access method"
-    default SPIRAM_USE_MEMMAP
+    default SPIRAM_USE_MALLOC
     help
         The SPI RAM can be accessed in multiple methods: by just having it available as an unmanaged
         memory region in the ESP32 memory map, by integrating it in the ESP32s heap as 'special' memory
@@ -59,8 +59,7 @@ config SPIRAM_USE_MEMMAP
 config SPIRAM_USE_CAPS_ALLOC
     bool "Make RAM allocatable using heap_caps_malloc(..., MALLOC_CAP_SPIRAM)"
 config SPIRAM_USE_MALLOC
-    bool "Make RAM allocatable using malloc as well"
-    depends on TO_BE_DONE
+    bool "Make RAM allocatable using malloc() as well"
 endchoice
 
 choice SPIRAM_TYPE
@@ -118,6 +117,46 @@ config SPIRAM_CACHE_WORKAROUND
         with the workaround and located in flash instead.
 
 
+config SPIRAM_MALLOC_ALWAYSINTERNAL
+    int "Maximum malloc() size, in bytes, to always put in internal memory"
+    depends on SPIRAM_USE_MALLOC
+    default 16384
+    range 0 131072
+    help
+        If malloc() is capable of also allocating SPI-connected ram, its allocation strategy will prefer to allocate chunks less
+        than this size in internal memory, while allocations larger than this will be done from external RAM.
+        If allocation from the preferred region fails, an attempt is made to allocate from the non-preferred
+        region instead, so malloc() will not suddenly fail when either internal or external memory is full.
+
+config SPIRAM_MALLOC_RESERVE_INTERNAL
+    int "Reserve this amount of bytes for data that specifically needs to be in DMA or internal memory"
+    depends on SPIRAM_USE_MALLOC
+    default 32768
+    range 0 131072
+    help
+        Because the external/internal RAM allocation strategy is not always perfect, it sometimes may happen
+        that the internal memory is entirely filled up. This causes allocations that are specifically done in
+        internal memory, for example the stack for new tasks or memory to service DMA or have memory that's 
+        also available when SPI cache is down, to fail. This option reserves a pool specifically for requests 
+        like that; the memory in this pool is not given out when a normal malloc() is called.
+        
+        Set this to 0 to disable this feature.
+        
+        Note that because FreeRTOS stacks are forced to internal memory, they will also use this memory pool;
+        be sure to keep this in mind when adjusting this value.
+
+config SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY
+    bool "Allow external memory as an argument to xTaskCreateStatic"
+    default n
+    depends on SPIRAM_USE_MALLOC
+    help
+        Because some bits of the ESP32 code environment cannot be recompiled with the cache workaround, normally
+        tasks cannot be safely run with their stack residing in external memory; for this reason xTaskCreate and
+        friends always allocate stack in internal memory and xTaskCreateStatic will check if the memory passed
+        to it is in internal memory. If you have a task that needs a large amount of stack and does not call on
+        ROM code in any way (no direct calls, but also no Bluetooth/WiFi), you can try to disable this and use
+        xTaskCreateStatic to create the tasks stack in external memory.
+
 endmenu
 
 config MEMMAP_TRACEMEM
@@ -776,6 +815,7 @@ config ESP32_WIFI_STATIC_TX_BUFFER
     bool "STATIC"
 config ESP32_WIFI_DYNAMIC_TX_BUFFER
     bool "DYNAMIC"
+    depends on !SPIRAM_USE_MALLOC
 endchoice
 
 config ESP32_WIFI_TX_BUFFER_TYPE

components/esp32/cpu_start.c

@@ -262,6 +262,16 @@ void start_cpu0_default(void)
         ESP_EARLY_LOGE(TAG, "External RAM could not be added to heap!");
         abort();
     }
+#if CONFIG_SPIRAM_MALLOC_RESERVE_INTERNAL
+    r=esp_spiram_reserve_dma_pool(CONFIG_SPIRAM_MALLOC_RESERVE_INTERNAL);
+    if (r != ESP_OK) {
+        ESP_EARLY_LOGE(TAG, "Could not reserve internal/DMA pool!");
+        abort();
+    }
+#endif
+#if CONFIG_SPIRAM_USE_MALLOC
+    heap_caps_malloc_extmem_enable(CONFIG_SPIRAM_MALLOC_ALWAYSINTERNAL);
+#endif
 #endif
 
 //Enable trace memory and immediately start trace.

components/esp32/include/esp_spiram.h

@@ -64,4 +64,16 @@ void esp_spiram_writeback_cache();
 
 
 
+/**
+ * @brief Reserve a pool of internal memory for specific DMA/internal allocations
+ *
+ * @param size Size of reserved pool in bytes
+ *
+ * @return
+ *          - ESP_OK on success
+ *          - ESP_ERR_NO_MEM when no memory available for pool
+ */
+esp_err_t esp_spiram_reserve_dma_pool(size_t size);
+
+
 #endif

components/esp32/intr_alloc.c

@@ -222,7 +222,7 @@ static vector_desc_t *get_desc_for_int(int intno, int cpu)
 {
     vector_desc_t *vd=find_desc_for_int(intno, cpu);
     if (vd==NULL) {
-        vector_desc_t *newvd=malloc(sizeof(vector_desc_t));
+        vector_desc_t *newvd=heap_caps_malloc(sizeof(vector_desc_t), MALLOC_CAP_INTERNAL|MALLOC_CAP_8BIT);
         if (newvd==NULL) return NULL;
         memset(newvd, 0, sizeof(vector_desc_t));
         newvd->intno=intno;
@@ -574,7 +574,7 @@ esp_err_t esp_intr_alloc_intrstatus(int source, int flags, uint32_t intrstatusre
     if (source==ETS_INTERNAL_PROFILING_INTR_SOURCE) force=ETS_INTERNAL_PROFILING_INTR_NO;
 
     //Allocate a return handle. If we end up not needing it, we'll free it later on.
-    ret=malloc(sizeof(intr_handle_data_t));
+    ret=heap_caps_malloc(sizeof(intr_handle_data_t), MALLOC_CAP_INTERNAL|MALLOC_CAP_8BIT);
     if (ret==NULL) return ESP_ERR_NO_MEM;
 
     portENTER_CRITICAL(&spinlock);

components/esp32/spiram.c

@@ -124,11 +124,24 @@ esp_err_t esp_spiram_init()
 
 esp_err_t esp_spiram_add_to_heapalloc()
 {
+    ESP_EARLY_LOGI(TAG, "Adding pool of %dK of external SPI memory to heap allocator", CONFIG_SPIRAM_SIZE/1024);
     //Add entire external RAM region to heap allocator. Heap allocator knows the capabilities of this type of memory, so there's
     //no need to explicitly specify them.
     return heap_caps_add_region((intptr_t)SOC_EXTRAM_DATA_LOW, (intptr_t)SOC_EXTRAM_DATA_LOW + CONFIG_SPIRAM_SIZE-1);
 }
 
+
+static uint8_t *dma_heap;
+
+esp_err_t esp_spiram_reserve_dma_pool(size_t size) {
+    if (size==0) return ESP_OK; //no-op
+    ESP_EARLY_LOGI(TAG, "Reserving pool of %dK of internal memory for DMA/internal allocations", size/1024);
+    dma_heap=heap_caps_malloc(size, MALLOC_CAP_DMA|MALLOC_CAP_INTERNAL);
+    if (!dma_heap) return ESP_ERR_NO_MEM;
+    uint32_t caps[]={MALLOC_CAP_DMA|MALLOC_CAP_INTERNAL, 0, MALLOC_CAP_8BIT|MALLOC_CAP_32BIT};
+    return heap_caps_add_region_with_caps(caps, dma_heap, dma_heap+size-1);
+}
+
 size_t esp_spiram_get_size()
 {
     return CONFIG_SPIRAM_SIZE;

components/esp32/system_api.c

@@ -360,12 +360,12 @@ void system_restore(void)
 
 uint32_t esp_get_free_heap_size( void )
 {
-    return heap_caps_get_free_size( MALLOC_CAP_8BIT );
+    return heap_caps_get_free_size( MALLOC_CAP_DEFAULT );
 }
 
 uint32_t esp_get_minimum_free_heap_size( void )
 {
-    return heap_caps_get_minimum_free_size( MALLOC_CAP_8BIT );
+    return heap_caps_get_minimum_free_size( MALLOC_CAP_DEFAULT );
 }
 
 uint32_t system_get_free_heap_size(void) __attribute__((alias("esp_get_free_heap_size")));