fix(storage/fatfs): move test app to correct directory

This commit is contained in:
Tomáš Rohlínek
2024-05-29 14:12:43 +02:00
committed by Tomas Rohlinek
parent 331abf74ff
commit 84ba1a323c
23 changed files with 8 additions and 9 deletions

View File

@@ -1,5 +1,13 @@
# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
tools/test_apps/storage/fatfsgen:
depends_components:
- fatfs
- vfs
disable_test:
- if: IDF_TARGET != "esp32"
reason: only one target needed
tools/test_apps/storage/partition_table_readonly:
disable_test:
- if: IDF_TARGET not in ["esp32", "esp32c3"]

View File

@@ -0,0 +1,6 @@
# The following lines of boilerplate have to be in your project's CMakeLists
# in this exact order for cmake to work correctly
cmake_minimum_required(VERSION 3.16)
include($ENV{IDF_PATH}/tools/cmake/project.cmake)
project(fatfsgen)

View File

@@ -0,0 +1,69 @@
| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-C5 | ESP32-C6 | ESP32-C61 | ESP32-H2 | ESP32-P4 | ESP32-S2 | ESP32-S3 |
| ----------------- | ----- | -------- | -------- | -------- | -------- | --------- | -------- | -------- | -------- | -------- |
# FATFS partition generation example
(See the README.md file in the upper level 'examples' directory for more information about examples.)
This example demonstrates how to use the FATFS partition
generation tool [fatfsgen.py](../../../components/fatfs/fatfsgen.py) to automatically create a FATFS
filesystem image from the contents of a host folder during build, with an option of
automatically flashing the created image on invocation of `idf.py -p PORT flash`.
Beware that the minimal required size of the flash is 4 MB.
You can specify using menuconfig weather example will use read-only or read-write mode. The default option is read-write mode.
To change it just use menuconfig:
```shell
idf.py menuconfig
```
Then select `Example Configuration` a chose `Mode for generated FATFS image` either `Read-Write Mode` or `Read-Only Mode`.
`Read-Only` option indicates generating raw fatfs image without wear levelling support.
On the other hand, for `Read-Write` the generated fatfs image will support wear levelling thus can be mounted in read-write mode.
The following gives an overview of the example:
1. There is a directory `fatfs_image` from which the FATFS filesystem image will be created.
2. The function `fatfs_create_rawflash_image` is used to specify that a FATFS image
should be created during build for the `storage` partition.
For CMake, it is called from [the main component's CMakeLists.txt](./main/CMakeLists.txt).
`FLASH_IN_PROJECT` specifies that the created image
should be flashed on invocation of `idf.py -p PORT flash` together with app, bootloader, partition table, etc.
The image is created on the example's build directory with the output filename `storage.bin`.
3. Upon invocation of `idf.py -p PORT flash monitor`, application loads and
finds there is already a valid FATFS filesystem in the `storage` partition with files same as those in `fatfs_image` directory. The application is then
able to read those files.
## How to use example
### Build and flash
To run the example, type the following command:
```CMake
# CMake
idf.py -p PORT flash monitor
```
(To exit the serial monitor, type ``Ctrl-]``.)
See the Getting Started Guide for full steps to configure and use ESP-IDF to build projects.
## Example output
Here is the example's console output:
```
...
I (322) example: Mounting FAT filesystem
I (332) example: Reading file
I (332) example: Read from file: 'this is test'
I (332) example: Unmounting FAT filesystem
I (342) example: Done
```
The logic of the example is contained in a [single source file](./main/fatfsgen_example_main.c),
and it should be relatively simple to match points in its execution with the log outputs above.

View File

@@ -0,0 +1 @@
This is generated on the host

View File

@@ -0,0 +1 @@
this is test

View File

@@ -0,0 +1 @@
This is generated on the host; long name it has

View File

@@ -0,0 +1 @@
this is test; long name it has

View File

@@ -0,0 +1,30 @@
idf_component_register(SRCS "fatfsgen_example_main.c"
INCLUDE_DIRS ".")
# Create a FATFS image from the contents of the 'fatfs_image' directory
# that fits the partition named 'storage'. FLASH_IN_PROJECT indicates that
# the generated image should be flashed when the entire project is flashed to
# the target with 'idf.py -p PORT flash'.
# If read-only mode is set (CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY)
# the generated image will be raw without wear levelling support.
# Otherwise it will support wear levelling and thus enable read-write mounting of the image in the device.
if(CONFIG_FATFS_LFN_NONE)
set(image ../fatfs_image)
else()
set(image ../fatfs_long_name_image)
endif()
if(CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY)
if(CONFIG_EXAMPLE_FATFS_DEFAULT_DATETIME)
fatfs_create_rawflash_image(storage ${image} FLASH_IN_PROJECT)
else()
fatfs_create_rawflash_image(storage ${image} FLASH_IN_PROJECT PRESERVE_TIME)
endif()
else()
if(CONFIG_EXAMPLE_FATFS_DEFAULT_DATETIME)
fatfs_create_spiflash_image(storage ${image} FLASH_IN_PROJECT)
else()
fatfs_create_spiflash_image(storage ${image} FLASH_IN_PROJECT PRESERVE_TIME)
endif()
endif()

View File

@@ -0,0 +1,24 @@
menu "Example Configuration"
config EXAMPLE_FATFS_MODE_READ_ONLY
bool "Read only mode for generated FATFS image"
default n
help
If read-only mode is set, the generated fatfs image will be raw (without wear levelling support).
Otherwise it will support wear levelling that enables read-write mounting.
config EXAMPLE_FATFS_WRITE_COUNT
int "Number of volumes"
default 1
range 1 600
help
Number of writes to the file (for testing purposes).
config EXAMPLE_FATFS_DEFAULT_DATETIME
bool "Default modification date and time for generated FATFS image"
default n
help
If default datetime is set, all files created in the generated FATFS partition have default time
equal to FATFS origin time (1 January 1980)
endmenu

View File

@@ -0,0 +1,153 @@
/*
* SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Unlicense OR CC0-1.0
*/
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "esp_vfs.h"
#include "esp_vfs_fat.h"
#include "sdkconfig.h"
#if CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY
#define EXAMPLE_FATFS_MODE_READ_ONLY true
#else
#define EXAMPLE_FATFS_MODE_READ_ONLY false
#endif
#if CONFIG_FATFS_LFN_NONE
#define EXAMPLE_FATFS_LONG_NAMES false
#else
#define EXAMPLE_FATFS_LONG_NAMES true
#endif
static const char *TAG = "example";
// Mount path for the partition
const char *base_path = "/spiflash";
// Handle of the wear levelling library instance
static wl_handle_t s_wl_handle = WL_INVALID_HANDLE;
void app_main(void)
{
ESP_LOGI(TAG, "Mounting FAT filesystem");
// To mount device we need name of device partition, define base_path
// and allow format partition in case if it is new one and was not formatted before
const esp_vfs_fat_mount_config_t mount_config = {
.max_files = 4,
.format_if_mount_failed = false,
.allocation_unit_size = CONFIG_WL_SECTOR_SIZE,
.use_one_fat = false,
};
esp_err_t err;
if (EXAMPLE_FATFS_MODE_READ_ONLY){
err = esp_vfs_fat_spiflash_mount_ro(base_path, "storage", &mount_config);
} else {
err = esp_vfs_fat_spiflash_mount_rw_wl(base_path, "storage", &mount_config, &s_wl_handle);
}
if (err != ESP_OK) {
ESP_LOGE(TAG, "Failed to mount FATFS (%s)", esp_err_to_name(err));
return;
}
char line[128];
char *device_filename;
if (EXAMPLE_FATFS_LONG_NAMES){
device_filename = "/spiflash/innerbutverylongname.txt";
} else {
device_filename = "/spiflash/inner.txt";
}
if (!EXAMPLE_FATFS_MODE_READ_ONLY){
// Open file for reading
ESP_LOGI(TAG, "Opening file");
FILE *f;
for(int i = 0; i < CONFIG_EXAMPLE_FATFS_WRITE_COUNT; i++){
f = fopen(device_filename, "wb");
if (f == NULL) {
ESP_LOGE(TAG, "Failed to open file for writing");
return;
}
fprintf(f, "This is written by the device");
fclose(f);
}
ESP_LOGI(TAG, "File written");
// Open file for reading
ESP_LOGI(TAG, "Reading file");
f = fopen(device_filename, "rb");
if (f == NULL) {
ESP_LOGE(TAG, "Failed to open file for reading");
return;
}
fgets(line, sizeof(line), f);
fclose(f);
// strip newline
char *pos = strchr(line, '\n');
if (pos) {
*pos = '\0';
}
ESP_LOGI(TAG, "Read from file: '%s'", line);
}
FILE *f;
char *pos;
ESP_LOGI(TAG, "Reading file");
char *host_filename1;
char *host_filename2;
if (EXAMPLE_FATFS_LONG_NAMES){
host_filename1 = "/spiflash/sublongnames/testlongfilenames.txt";
host_filename2 = "/spiflash/hellolongname.txt";
} else{
host_filename1 = "/spiflash/sub/test.txt";
host_filename2 = "/spiflash/hello.txt";
}
struct stat info;
struct tm timeinfo;
char buffer[32];
if(stat(host_filename1, &info) < 0){
ESP_LOGE(TAG, "Failed to read file stats");
return;
}
localtime_r(&info.st_mtime, &timeinfo);
strftime(buffer, sizeof(buffer), "%Y-%m-%d", &timeinfo);
ESP_LOGI(TAG, "The file '%s' was modified at date: %s", host_filename1, buffer);
if (EXAMPLE_FATFS_MODE_READ_ONLY){
f = fopen(host_filename1, "rb");
} else {
f = fopen(host_filename2, "rb");
}
if (f == NULL) {
ESP_LOGE(TAG, "Failed to open file for reading");
return;
}
fgets(line, sizeof(line), f);
fclose(f);
// strip newline
pos = strchr(line, '\n');
if (pos) {
*pos = '\0';
}
ESP_LOGI(TAG, "Read from file: '%s'", line);
// Unmount FATFS
ESP_LOGI(TAG, "Unmounting FAT filesystem");
if (EXAMPLE_FATFS_MODE_READ_ONLY){
ESP_ERROR_CHECK(esp_vfs_fat_spiflash_unmount_ro(base_path, "storage"));
} else {
ESP_ERROR_CHECK(esp_vfs_fat_spiflash_unmount_rw_wl(base_path, s_wl_handle));
}
ESP_LOGI(TAG, "Done");
}

View File

@@ -0,0 +1,6 @@
# Name, Type, SubType, Offset, Size, Flags
# Note: if you have increased the bootloader size, make sure to update the offsets to avoid overlap
nvs, data, nvs, 0x9000, 0x6000,
phy_init, data, phy, 0xf000, 0x1000,
factory, app, factory, 0x10000, 1M,
storage, data, fat, , 1M,
1 # Name, Type, SubType, Offset, Size, Flags
2 # Note: if you have increased the bootloader size, make sure to update the offsets to avoid overlap
3 nvs, data, nvs, 0x9000, 0x6000,
4 phy_init, data, phy, 0xf000, 0x1000,
5 factory, app, factory, 0x10000, 1M,
6 storage, data, fat, , 1M,

View File

@@ -0,0 +1,226 @@
# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Unlicense OR CC0-1.0
import os
import re
import shutil
import sys
from datetime import datetime
from subprocess import run
from subprocess import STDOUT
from typing import List
import pytest
from pytest_embedded import Dut
idf_path = os.environ['IDF_PATH'] # get value of IDF_PATH from environment
parttool_dir = os.path.join(idf_path, 'components', 'partition_table')
sys.path.append(parttool_dir)
from parttool import PartitionName, ParttoolTarget # noqa E402 # pylint: disable=C0413
def file_(x: str, content_: str = 'hey this is a test') -> dict:
return {
'type': 'file',
'name': x,
'content': content_
}
def generate_local_folder_structure(structure_: dict, path_: str) -> None:
if structure_['type'] == 'folder':
new_path_ = os.path.join(path_, structure_['name'])
os.makedirs(new_path_)
for item_ in structure_['content']:
generate_local_folder_structure(item_, new_path_)
else:
new_path_ = os.path.join(path_, structure_['name'])
with open(new_path_, 'w') as f_:
f_.write(structure_['content'])
def compare_folders(fp1: str, fp2: str) -> bool:
if os.path.isdir(fp1) != os.path.isdir(fp2):
return False
if os.path.isdir(fp1):
if set(os.listdir(fp1)) != set(os.listdir(fp2)):
return False
return all([compare_folders(os.path.join(fp1, path_), os.path.join(fp2, path_)) for path_ in os.listdir(fp1)])
with open(fp1, 'rb') as f1_, open(fp2, 'rb') as f2_:
return f1_.read() == f2_.read()
@pytest.mark.esp32
@pytest.mark.generic
@pytest.mark.parametrize('config', ['test_read_only_partition_gen',
'test_read_only_partition_gen_default_dt',
'test_read_only_partition_gen_ln',
'test_read_only_partition_gen_ln_default_dt',
'test_read_write_partition_gen',
'test_read_write_partition_gen_default_dt',
'test_read_write_partition_gen_ln',
'test_read_write_partition_gen_ln_default_dt',
], indirect=True)
def test_examples_fatfsgen(config: str, dut: Dut) -> None:
# Expects list of strings sequentially
def expect_all(msg_list: List[str], to: int) -> None:
for msg in msg_list:
dut.expect(msg, timeout=to)
# Expects prefix string followed by date in the format 'yyyy-mm-dd'
def expect_date(prefix: str, to: int) -> datetime:
expect_str = prefix + '(\\d+)-(\\d+)-(\\d+)'
match_ = dut.expect(re.compile(str.encode(expect_str)), timeout=to)
year_ = int(match_[1].decode())
month_ = int(match_[2].decode())
day_ = int(match_[3].decode())
return datetime(year_, month_, day_)
# Calculates absolute difference in days between date_reference and date_actual.
# Raises exception if difference exceeds tolerance
def evaluate_dates(date_reference: datetime, date_actual: datetime, days_tolerance: int) -> None:
td = date_actual - date_reference
if abs(td.days) > days_tolerance:
raise Exception(f'Too big date difference. Actual: {date_actual}, reference: {date_reference}, tolerance: {days_tolerance} day(s)')
# Expect timeout
timeout = 20
# We tolerate 30 days difference between actual file creation and date when test was executed.
tolerance = 30
filename_ln = 'sublongnames/testlongfilenames.txt'
filename_sn = 'sub/test.txt'
date_modified = datetime.today()
date_default = datetime(1980, 1, 1)
fatfs_parser_path = os.path.join(idf_path, 'components', 'fatfs', 'fatfsparse.py')
if config in ['test_read_write_partition_gen', 'test_read_write_partition_gen_default_dt']:
filename = filename_sn
filename_expected = f'/spiflash/{filename}'
date_ref = date_default if config == 'test_read_write_partition_gen_default_dt' else date_modified
expect_all(['example: Mounting FAT filesystem',
'example: Opening file',
'example: File written',
'example: Reading file',
'example: Read from file: \'This is written by the device\'',
'example: Reading file'], timeout)
date_act = expect_date(f'The file \'{filename_expected}\' was modified at date: ', timeout)
evaluate_dates(date_ref, date_act, tolerance)
expect_all(['example: Read from file: \'This is generated on the host\'',
'example: Unmounting FAT filesystem',
'example: Done'], timeout)
target = ParttoolTarget(dut.port)
target.read_partition(PartitionName('storage'), 'temp.img')
run(['python', fatfs_parser_path, '--wear-leveling', 'temp.img'], stderr=STDOUT)
folder_ = {
'type': 'folder',
'name': 'SUB',
'content': [
file_('TEST.TXT', content_='this is test\n'),
]
}
struct_: dict = {
'type': 'folder',
'name': 'testf',
'content': [
file_('HELLO.TXT', content_='This is generated on the host\n'),
file_('INNER.TXT', content_='This is written by the device'),
folder_
]
}
generate_local_folder_structure(struct_, path_='.')
try:
assert compare_folders('testf', 'Espressif')
finally:
shutil.rmtree('Espressif', ignore_errors=True)
shutil.rmtree('testf', ignore_errors=True)
elif config in ['test_read_only_partition_gen', 'test_read_only_partition_gen_default_dt']:
filename = filename_sn
filename_expected = f'/spiflash/{filename}'
date_ref = date_default if config == 'test_read_only_partition_gen_default_dt' else date_modified
expect_all(['example: Mounting FAT filesystem',
'example: Reading file'], timeout)
date_act = expect_date(f'The file \'{filename_expected}\' was modified at date: ', timeout)
evaluate_dates(date_ref, date_act, tolerance)
expect_all(['example: Read from file: \'this is test\'',
'example: Unmounting FAT filesystem',
'example: Done'], timeout)
target = ParttoolTarget(dut.port)
target.read_partition(PartitionName('storage'), 'temp.img')
run(['python', fatfs_parser_path, '--long-name-support', 'temp.img'], stderr=STDOUT)
folder_ = {
'type': 'folder',
'name': 'sublongnames',
'content': [
file_('testlongfilenames.txt', content_='this is test; long name it has\n'),
]
}
struct_ = {
'type': 'folder',
'name': 'testf',
'content': [
file_('hellolongname.txt', content_='This is generated on the host; long name it has\n'),
folder_
]
}
generate_local_folder_structure(struct_, path_='.')
try:
assert compare_folders('testf', 'Espressif')
finally:
shutil.rmtree('Espressif', ignore_errors=True)
shutil.rmtree('testf', ignore_errors=True)
elif config in ['test_read_write_partition_gen_ln', 'test_read_write_partition_gen_ln_default_dt']:
filename = filename_ln
filename_expected = f'/spiflash/{filename}'
date_ref = date_default if config == 'test_read_write_partition_gen_ln_default_dt' else date_modified
expect_all(['example: Mounting FAT filesystem',
'example: Opening file',
'example: File written',
'example: Reading file',
'example: Read from file: \'This is written by the device\'',
'example: Reading file'], timeout)
date_act = expect_date(f'The file \'{filename_expected}\' was modified at date: ', timeout)
evaluate_dates(date_ref, date_act, tolerance)
expect_all(['example: Read from file: \'This is generated on the host; long name it has\'',
'example: Unmounting FAT filesystem',
'example: Done'], timeout)
elif config in ['test_read_only_partition_gen_ln', 'test_read_only_partition_gen_ln_default_dt']:
filename = filename_ln
filename_expected = f'/spiflash/{filename}'
date_ref = date_default if config == 'test_read_only_partition_gen_ln_default_dt' else date_modified
expect_all(['example: Mounting FAT filesystem',
'example: Reading file'], timeout)
date_act = expect_date(f'The file \'{filename_expected}\' was modified at date: ', timeout)
evaluate_dates(date_ref, date_act, tolerance)
expect_all(['example: Read from file: \'this is test; long name it has\'',
'example: Unmounting FAT filesystem',
'example: Done'], timeout)
target = ParttoolTarget(dut.port)
target.read_partition(PartitionName('storage'), 'temp.img')
run(['python', fatfs_parser_path, 'temp.img'], stderr=STDOUT)
folder_ = {
'type': 'folder',
'name': 'SUB',
'content': [
file_('TEST.TXT', content_='this is test\n'),
]
}
struct_ = {
'type': 'folder',
'name': 'testf',
'content': [
file_('HELLO.TXT', content_='This is generated on the host\n'),
folder_
]
}
generate_local_folder_structure(struct_, path_='.')
try:
assert compare_folders('testf', 'Espressif')
finally:
shutil.rmtree('Espressif', ignore_errors=True)
shutil.rmtree('testf', ignore_errors=True)

View File

@@ -0,0 +1,5 @@
CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY=y
CONFIG_FATFS_LFN_HEAP=n
CONFIG_FATFS_LFN_STACK=n
CONFIG_FATFS_LFN_NONE=y
CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300

View File

@@ -0,0 +1,6 @@
CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY=y
CONFIG_FATFS_LFN_HEAP=n
CONFIG_FATFS_LFN_STACK=n
CONFIG_FATFS_LFN_NONE=y
CONFIG_EXAMPLE_FATFS_DEFAULT_DATETIME=y
CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300

View File

@@ -0,0 +1,5 @@
CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY=y
CONFIG_FATFS_LFN_HEAP=y
CONFIG_FATFS_LFN_NONE=n
CONFIG_FATFS_LFN_STACK=n
CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300

View File

@@ -0,0 +1,6 @@
CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY=y
CONFIG_FATFS_LFN_HEAP=y
CONFIG_FATFS_LFN_NONE=n
CONFIG_FATFS_LFN_STACK=n
CONFIG_EXAMPLE_FATFS_DEFAULT_DATETIME=y
CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300

View File

@@ -0,0 +1,5 @@
CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY=n
CONFIG_FATFS_LFN_HEAP=n
CONFIG_FATFS_LFN_STACK=n
CONFIG_FATFS_LFN_NONE=y
CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300

View File

@@ -0,0 +1,6 @@
CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY=n
CONFIG_FATFS_LFN_HEAP=n
CONFIG_FATFS_LFN_STACK=n
CONFIG_FATFS_LFN_NONE=y
CONFIG_EXAMPLE_FATFS_DEFAULT_DATETIME=y
CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300

View File

@@ -0,0 +1,5 @@
CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY=n
CONFIG_FATFS_LFN_HEAP=y
CONFIG_FATFS_LFN_NONE=n
CONFIG_FATFS_LFN_STACK=n
CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300

View File

@@ -0,0 +1,6 @@
CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY=n
CONFIG_FATFS_LFN_HEAP=y
CONFIG_FATFS_LFN_NONE=n
CONFIG_FATFS_LFN_STACK=n
CONFIG_EXAMPLE_FATFS_DEFAULT_DATETIME=y
CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300

View File

@@ -0,0 +1,4 @@
CONFIG_PARTITION_TABLE_CUSTOM=y
CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions_example.csv"
CONFIG_PARTITION_TABLE_FILENAME="partitions_example.csv"
CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y