alex/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2025-08-07 20:00:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(esp_http_server): Modified the calculation of buf_len

Browse Source 
Modified the calculation of buf_len, so that scratch buffer allocates
the memory according to requirement and not extra (except the last read chunk)
This commit is contained in:
hrushikesh.bhosale
2025-01-28 15:47:30 +05:30
parent 97b6043435
commit 9846584def
19 changed files with 120 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
components/esp_http_server/Kconfig
View File

@@ -2,15 +2,19 @@ menu "HTTP Server"
config HTTPD_MAX_REQ_HDR_LEN
int "Max HTTP Request Header Length"
default 2000
int "HTTP Request Header Length limit"
default 1024
range 128 65536
help
This sets the maximum supported size of headers section in HTTP request packet to be processed by the
server
This sets the default limit for the HTTP request header length. The limit can be
configured at run time by setting max_req_hdr_len member of httpd_config_t structure.
The memory allocated will depend on the actual header length. Hence keeping a sufficiently
large max header length is recommended.
config HTTPD_MAX_URI_LEN
int "Max HTTP URI Length"
default 2000
default 512
help
This sets the maximum supported size of HTTP request URI to be processed by the server

25
components/esp_http_server/include/esp_http_server.h
View File

@@ -53,10 +53,10 @@ initializer that should be kept in sync
#define HTTPD_DEFAULT_CONFIG() { \
.task_priority = tskIDLE_PRIORITY+5, \
.stack_size = 4096, \
.hdr_buf_size_limit = HTTPD_MAX_REQ_HDR_LEN, \
.uri_buf_size_limit = HTTPD_MAX_URI_LEN, \
.core_id = tskNO_AFFINITY, \
.task_caps = (MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT), \
.max_req_hdr_len = CONFIG_HTTPD_MAX_REQ_HDR_LEN, \
.max_uri_len = CONFIG_HTTPD_MAX_URI_LEN, \
.server_port = 80, \
.ctrl_port = ESP_HTTPD_DEF_CTRL_PORT, \
.max_open_sockets = 7, \
@@ -176,11 +176,12 @@ typedef struct httpd_config {
uint32_t task_caps; /*!< The memory capabilities to use when allocating the HTTP server task's stack */
/**
* Size limits for the header and URI buffers respectively. These are just limits, actually allocation would be depend upon actual size of URI/header.
* These are set to the values defined in the Kconfig file.
* Size limits for the header and URI buffers respectively.
* These are just limits, allocation would depend upon actual size of URI/header.
*/
uint16_t hdr_buf_size_limit; /*!< Size limit for the header buffer */
uint16_t uri_buf_size_limit; /*!< Size limit for the URI buffer */
size_t max_req_hdr_len; /*!< Size limit for the header buffer (By default this value is set to CONFIG_HTTPD_MAX_REQ_HDR_LEN, overwrite is possible) */
size_t max_uri_len; /*!< Size limit for the URI buffer By default this value is set to CONFIG_HTTPD_MAX_URI_LEN, overwrite is possible) */
/**
* TCP Port number for receiving and transmitting HTTP traffic
*/
@@ -366,19 +367,13 @@ esp_err_t httpd_stop(httpd_handle_t handle);
* @{
*/
/* Max supported HTTP request header length */
#define HTTPD_MAX_REQ_HDR_LEN CONFIG_HTTPD_MAX_REQ_HDR_LEN
/* Max supported HTTP request URI length */
#define HTTPD_MAX_URI_LEN CONFIG_HTTPD_MAX_URI_LEN
/**
* @brief HTTP Request Data Structure
*/
typedef struct httpd_req {
httpd_handle_t handle; /*!< Handle to server instance */
int method; /*!< The type of HTTP request, -1 if unsupported method, HTTP_ANY for wildcard method to support every method */
const char uri[HTTPD_MAX_URI_LEN + 1]; /*!< The URI of this request (1 byte extra for null termination) */
const char uri[CONFIG_HTTPD_MAX_URI_LEN + 1]; /*!< The URI of this request (1 byte extra for null termination) */
size_t content_len; /*!< Length of the request body */
void *aux; /*!< Internally used members */
@@ -619,10 +614,10 @@ typedef enum {
/* Incoming payload is too large */
HTTPD_413_CONTENT_TOO_LARGE,
/* URI length greater than CONFIG_HTTPD_MAX_URI_LEN */
/* URI length greater than CONFIG_CONFIG_HTTPD_MAX_URI_LEN */
HTTPD_414_URI_TOO_LONG,
/* Headers section larger than CONFIG_HTTPD_MAX_REQ_HDR_LEN */
/* Headers section larger than CONFIG_CONFIG_HTTPD_MAX_REQ_HDR_LEN */
HTTPD_431_REQ_HDR_FIELDS_TOO_LARGE,
/* Used internally for retrieving the total count of errors */

11
components/esp_http_server/src/esp_httpd_priv.h
View File

@@ -31,8 +31,7 @@ extern "C" {
#endif
/* Size of request data block/chunk (not to be confused with chunked encoded data)
* that is received and parsed in one turn of the parsing process. This should not
* exceed the scratch buffer size and should at least be 8 bytes */
* that is received and parsed in one turn of the parsing process. */
#define PARSER_BLOCK_SIZE 128
/* Formats a log string to prepend context function name */
@@ -86,10 +85,10 @@ struct sock_db {
struct httpd_req_aux {
struct sock_db *sd; /*!< Pointer to socket database */
char *scratch; /*!< Temporary buffer for our operations (1 byte extra for null termination) */
uint16_t scratch_size_limit; /*!< Scratch buffer size limit */
uint16_t scratch_cur_size; /*!< Scratch buffer cur size */
uint16_t hdr_buf_size_limit; /*!< Header buffer size limit */
uint16_t uri_buf_size_limit; /*!< URI buffer size limit */
size_t scratch_size_limit; /*!< Scratch buffer size limit (By default this value is set to CONFIG_HTTPD_MAX_REQ_HDR_LEN, overwrite is possible) */
size_t scratch_cur_size; /*!< Scratch buffer cur size (By default this value is set to CONFIG_HTTPD_MAX_URI_LEN, overwrite is possible) */
size_t max_req_hdr_len; /*!< Header buffer size limit */
size_t max_uri_len; /*!< URI buffer size limit */
size_t remaining_len; /*!< Amount of data remaining to be fetched */
char *status; /*!< HTTP response's status code */
char *content_type; /*!< HTTP response's content type */

49
components/esp_http_server/src/httpd_parse.c
View File

@@ -114,6 +114,7 @@ static esp_err_t cb_url(http_parser *parser,
parser_data_t *parser_data = (parser_data_t *) parser->data;
httpd_req_t *req = parser_data->req;
struct httpd_req_aux *raux = req->aux;
if (parser_data->status == PARSING_IDLE) {
ESP_LOGD(TAG, LOG_FMT("message begin"));
@@ -131,9 +132,9 @@ static esp_err_t cb_url(http_parser *parser,
ESP_LOGD(TAG, LOG_FMT("processing url = %.*s"), (int)length, at);
/* Update length of URL string */
if ((parser_data->last.length += length) > raux->uri_buf_size_limit) {
if ((parser_data->last.length += length) > raux->max_uri_len) {
ESP_LOGW(TAG, LOG_FMT("URI length (%"NEWLIB_NANO_COMPAT_FORMAT") greater than supported (%d)"),
NEWLIB_NANO_COMPAT_CAST(parser_data->last.length), raux->uri_buf_size_limit);
NEWLIB_NANO_COMPAT_CAST(parser_data->last.length), raux->max_uri_len);
parser_data->error = HTTPD_414_URI_TOO_LONG;
parser_data->status = PARSING_FAILED;
return ESP_FAIL;
@@ -216,7 +217,8 @@ static esp_err_t cb_header_field(http_parser *parser, const char *at, size_t len
parser_data->last.at = ra->scratch;
parser_data->last.length = 0;
parser_data->status = PARSING_HDR_FIELD;
ra->scratch_size_limit = ra->hdr_buf_size_limit;
ra->scratch_size_limit = ra->max_req_hdr_len;
/* Stop parsing for now and give control to process */
if (pause_parsing(parser, at) != ESP_OK) {
parser_data->error = HTTPD_500_INTERNAL_SERVER_ERROR;
@@ -233,7 +235,7 @@ static esp_err_t cb_header_field(http_parser *parser, const char *at, size_t len
parser_data->last.at = at;
parser_data->last.length = 0;
parser_data->status = PARSING_HDR_FIELD;
ra->scratch_size_limit = ra->hdr_buf_size_limit;
ra->scratch_size_limit = ra->max_req_hdr_len;
/* Increment header count */
ra->req_hdrs_count++;
@@ -415,6 +417,7 @@ static esp_err_t cb_headers_complete(http_parser *parser)
static esp_err_t cb_on_body(http_parser *parser, const char *at, size_t length)
{
parser_data_t *parser_data = (parser_data_t *) parser->data;
/* Check previous status */
if (parser_data->status != PARSING_BODY) {
ESP_LOGE(TAG, LOG_FMT("unexpected state transition"));
@@ -446,6 +449,7 @@ static esp_err_t cb_on_body(http_parser *parser, const char *at, size_t length)
static esp_err_t cb_no_body(http_parser *parser)
{
parser_data_t *parser_data = (parser_data_t *) parser->data;
/* Check previous status */
if (parser_data->status == PARSING_URL) {
ESP_LOGD(TAG, LOG_FMT("no headers"));
@@ -480,26 +484,32 @@ static esp_err_t cb_no_body(http_parser *parser)
return ESP_OK;
}
static int read_block(httpd_req_t *req, size_t offset, size_t length)
static int read_block(httpd_req_t *req, http_parser *parser, size_t offset, size_t length)
{
struct httpd_req_aux *raux = req->aux;
parser_data_t *parser_data = (parser_data_t *) parser->data;
/* Limits the read to scratch buffer size */
ssize_t buf_len = MIN(length, (raux->scratch_size_limit - offset));
if (buf_len <= 0) {
return 0;
}
if (raux->scratch == NULL && buf_len < raux->scratch_size_limit) {
raux->scratch = (char*) malloc(buf_len);
}
else if (raux->scratch != NULL && buf_len < raux->scratch_size_limit) {
raux->scratch = (char*) realloc(raux->scratch, raux->scratch_cur_size + buf_len);
}
/* Calculate the offset of the current position from the start of the buffer,
* as after reallocating the buffer, the base address of the buffer may change.
*/
size_t at_offset = parser_data->last.at - raux->scratch;
/* Allocate the buffer according to offset and buf_len. Offset is
from where the reading will start and buf_len is till what length
the buffer will be read.
*/
raux->scratch = (char*) realloc(raux->scratch, offset + buf_len);
if (raux->scratch == NULL) {
ESP_LOGE(TAG, "Unable to allocate the scratch buffer");
return 0;
}
raux->scratch_cur_size += buf_len;
ESP_LOGD(TAG, "scratch size = %d", raux->scratch_cur_size);
parser_data->last.at = raux->scratch + at_offset;
raux->scratch_cur_size = offset + buf_len;
ESP_LOGD(TAG, "scratch buf qsize = %d", raux->scratch_cur_size);
/* Receive data into buffer. If data is pending (from unrecv) then return
* immediately after receiving pending data, as pending data may just complete
* this request packet. */
@@ -537,7 +547,7 @@ static int parse_block(http_parser *parser, size_t offset, size_t length)
httpd_req_t *req = data->req;
struct httpd_req_aux *raux = req->aux;
size_t nparsed = 0;
data->last.at = raux->scratch;
if (!length) {
/* Parsing is still happening but nothing to
* parse means no more space left on buffer,
@@ -643,7 +653,7 @@ static esp_err_t httpd_parse_req(struct httpd_data *hd)
offset = 0;
do {
/* Read block into scratch buffer */
if ((blk_len = read_block(r, offset, PARSER_BLOCK_SIZE)) < 0) {
if ((blk_len = read_block(r, &parser, offset, PARSER_BLOCK_SIZE)) < 0) {
if (blk_len == HTTPD_SOCK_ERR_TIMEOUT) {
/* Retry read in case of non-fatal timeout error.
* read_block() ensures that the timeout error is
@@ -660,6 +670,7 @@ static esp_err_t httpd_parse_req(struct httpd_data *hd)
/* This is used by the callbacks to track
* data usage of the buffer */
parser_data.raw_datalen = blk_len + offset;
/* Parse data block from buffer */
if ((offset = parse_block(&parser, offset, blk_len)) < 0) {
/* HTTP error occurred.
@@ -697,9 +708,9 @@ static void init_req_aux(struct httpd_req_aux *ra, httpd_config_t *config)
ra->resp_hdrs_count = 0;
ra->scratch = NULL;
ra->scratch_cur_size = 0;
ra->hdr_buf_size_limit = config->hdr_buf_size_limit;
ra->uri_buf_size_limit = config->uri_buf_size_limit;
ra->scratch_size_limit = ra->uri_buf_size_limit;
ra->max_req_hdr_len = (config->max_req_hdr_len > 0) ? config->max_req_hdr_len : CONFIG_HTTPD_MAX_REQ_HDR_LEN;
ra->max_uri_len = (config->max_uri_len > 0) ? config->max_uri_len : CONFIG_HTTPD_MAX_URI_LEN;
ra->scratch_size_limit = ra->max_uri_len;
#if CONFIG_HTTPD_WS_SUPPORT
ra->ws_handshake_detect = false;
#endif
@@ -732,8 +743,8 @@ static void httpd_req_cleanup(httpd_req_t *r)
ra->sd = NULL;
free(ra->scratch);
ra->scratch = NULL;
ra->scratch_cur_size = 0;
ra->scratch_size_limit = 0;
ra->scratch_cur_size = 0;
r->handle = NULL;
r->aux = NULL;
r->user_ctx = NULL;

76
components/esp_http_server/src/httpd_txrx.c
View File

@@ -132,6 +132,7 @@ int httpd_recv_with_opt(httpd_req_t *r, char *buf, size_t buf_len, bool halt_aft
}
return ret;
}
ESP_LOGD(TAG, LOG_FMT("received length = %"NEWLIB_NANO_COMPAT_FORMAT), NEWLIB_NANO_COMPAT_CAST((ret + pending_len)));
return ret + pending_len;
}
@@ -242,29 +243,30 @@ esp_err_t httpd_resp_send(httpd_req_t *r, const char *buf, ssize_t buf_len)
if (buf_len == HTTPD_RESP_USE_STRLEN) {
buf_len = strlen(buf);
}
size_t required_size = snprintf(NULL, 0, httpd_hdr_str, ra->status, ra->content_type, buf_len) + 1;
if (required_size >= HTTPD_MAX_REQ_HDR_LEN) {
return ESP_ERR_HTTPD_RESP_HDR;
}
if (ra->scratch == NULL)
ra->scratch = malloc(required_size);
else {
ra->scratch = realloc(ra->scratch, required_size);
}
if (ra->scratch == NULL) {
ESP_LOGE(TAG, "Unable to allocate scratch buffer");
return ESP_ERR_HTTPD_ALLOC_MEM;
}
ra->scratch_cur_size = required_size;
ESP_LOGD(TAG, "scratch size = %d", ra->scratch_cur_size);
/* Request headers are no longer available */
ra->req_hdrs_count = 0;
/* Size of essential headers is limited by scratch buffer size */
snprintf(ra->scratch, ra->scratch_cur_size, httpd_hdr_str, ra->status, ra->content_type, buf_len);
ESP_LOG_BUFFER_HEXDUMP(TAG, ra->scratch, strlen(ra->scratch), ESP_LOG_INFO);
/* Sending essential headers */
if (httpd_send_all(r, ra->scratch, strlen(ra->scratch)) != ESP_OK) {
/* Calculate the size of the headers. +1 for the null terminator */
size_t required_size = snprintf(NULL, 0, httpd_hdr_str, ra->status, ra->content_type, buf_len) + 1;
if (required_size > ra->max_req_hdr_len) {
return ESP_ERR_HTTPD_RESP_HDR;
}
char *res_buf = malloc(required_size); /* Temporary buffer to store the headers */
if (res_buf == NULL) {
ESP_LOGE(TAG, "Unable to allocate httpd send buffer");
return ESP_ERR_HTTPD_ALLOC_MEM;
}
ESP_LOGD(TAG, "httpd send buffer size = %d", strlen(res_buf));
esp_err_t ret = snprintf(res_buf, required_size, httpd_hdr_str, ra->status, ra->content_type, buf_len);
if (ret < 0 || ret >= required_size) {
free(res_buf);
return ESP_ERR_HTTPD_RESP_HDR;
}
ret = httpd_send_all(r, res_buf, strlen(res_buf));
free(res_buf);
if (ret != ESP_OK) {
return ESP_ERR_HTTPD_RESP_SEND;
}
@@ -329,27 +331,28 @@ esp_err_t httpd_resp_send_chunk(httpd_req_t *r, const char *buf, ssize_t buf_len
/* Request headers are no longer available */
ra->req_hdrs_count = 0;
/* Calculate the size of the headers. +1 for the null terminator */
size_t required_size = snprintf(NULL, 0, httpd_chunked_hdr_str, ra->status, ra->content_type) + 1;
if (required_size >= HTTPD_MAX_REQ_HDR_LEN) {
if (required_size > ra->max_req_hdr_len) {
return ESP_ERR_HTTPD_RESP_HDR;
}
if (ra->scratch == NULL)
ra->scratch = malloc(required_size);
else {
ra->scratch = realloc(ra->scratch, required_size);
}
if (ra->scratch == NULL) {
ESP_LOGE(TAG, "Unable to allocate scratch buffer");
char *res_buf = malloc(required_size); /* Temporary buffer to store the headers */
if (res_buf == NULL) {
ESP_LOGE(TAG, "Unable to allocate httpd send chunk buffer");
return ESP_ERR_HTTPD_ALLOC_MEM;
}
ra->scratch_cur_size = required_size;
ESP_LOGD(TAG, "scratch size = %d", ra->scratch_cur_size);
ESP_LOGD(TAG, "httpd send chunk buffer size = %d", strlen(res_buf));
if (!ra->first_chunk_sent) {
esp_err_t ret = snprintf(res_buf, required_size, httpd_chunked_hdr_str, ra->status, ra->content_type);
if (ret < 0 || ret >= required_size) {
free(res_buf);
return ESP_ERR_HTTPD_RESP_HDR;
}
/* Size of essential headers is limited by scratch buffer size */
snprintf(ra->scratch, ra->scratch_cur_size, httpd_chunked_hdr_str, ra->status, ra->content_type);
/* Sending essential headers */
if (httpd_send_all(r, ra->scratch, strlen(ra->scratch)) != ESP_OK) {
ret = httpd_send_all(r, res_buf, strlen(res_buf));
free(res_buf);
if (ret != ESP_OK) {
return ESP_ERR_HTTPD_RESP_SEND;
}
@@ -671,7 +674,10 @@ esp_err_t httpd_req_async_handler_complete(httpd_req_t *r)
struct httpd_req_aux *ra = r->aux;
ra->sd->for_async_req = false;
free(ra->scratch);
ra->scratch = NULL;
ra->scratch_cur_size = 0;
ra->scratch_size_limit = 0;
free(ra->resp_hdrs);
free(r->aux);
free(r);