From ec2aad46d0addf82f25abcc63fa6fd09ff58a515 Mon Sep 17 00:00:00 2001 From: Hiroyuki OYAMA Date: Mon, 17 Jun 2024 09:50:54 +0900 Subject: [PATCH] Improved storage handling over 4 GB of FAT. --- LIMITATION.md | 8 +- include/blockdevice/blockdevice.h | 12 ++- include/filesystem/ChaN/ffconf.h | 2 +- src/blockdevice/flash.c | 20 ++-- src/blockdevice/heap.c | 18 ++-- src/blockdevice/loopback.c | 18 ++-- src/blockdevice/sd.c | 16 +-- src/filesystem/fat.c | 14 +-- tests/CMakeLists.txt | 1 + tests/large_file/CMakeLists.txt | 22 ++++ tests/large_file/main.c | 169 ++++++++++++++++++++++++++++++ 11 files changed, 248 insertions(+), 52 deletions(-) create mode 100644 tests/large_file/CMakeLists.txt create mode 100644 tests/large_file/main.c diff --git a/LIMITATION.md b/LIMITATION.md index afbb569..ee834b6 100644 --- a/LIMITATION.md +++ b/LIMITATION.md @@ -4,11 +4,13 @@ When using pico-vfs, there are several limitations and behavior specifics that u ## File System Limitations -1. **Multiple Open Instances**: The current implementations of littlefs and FatFs do not support opening the same file multiple times simultaneously[^1][^2]. This limitation can affect scenarios where multiple accesses to the same file are required concurrently. +1. **Max File Size for FAT**: The maximum single file size of a FAT file system depends on the capacity of the storage medium. Check the size of the SD card used and the type of FAT (FAT16/32/ExFat) automatically assigned. -2. **`core1` Usage with Onboard Flash Block Device**: When operating file systems using the onboard flash block device on `core1`, it is necessary to run the operations from RAM[^3]. +2. **Multiple Open Instances**: The current implementations of littlefs and FatFs do not support opening the same file multiple times simultaneously[^1][^2]. This limitation can affect scenarios where multiple accesses to the same file are required concurrently. -3. **Memory map IO**: `mmap` system call not supported. +3. **`core1` Usage with Onboard Flash Block Device**: When operating file systems using the onboard flash block device on `core1`, it is necessary to run the operations from RAM[^3]. + +4. **Memory maped IO**: `mmap` system call not supported. We recommend reviewing these limitations before designing systems that heavily rely on multicore operations or require high file access availability. diff --git a/include/blockdevice/blockdevice.h b/include/blockdevice/blockdevice.h index a7cdcbc..6091d32 100644 --- a/include/blockdevice/blockdevice.h +++ b/include/blockdevice/blockdevice.h @@ -17,6 +17,8 @@ extern "C" { #include #include +typedef uint64_t bd_size_t; + enum bd_error { BD_ERROR_OK = 0, /*!< no error */ BD_ERROR_DEVICE_ERROR = -4001, /*!< device specific error */ @@ -31,11 +33,11 @@ typedef struct blockdevice { int (*init)(struct blockdevice *device); int (*deinit)(struct blockdevice *device); int (*sync)(struct blockdevice *device); - int (*read)(struct blockdevice *device, const void *buffer, size_t addr, size_t size); - int (*program)(struct blockdevice *device, const void *buffer, size_t addr, size_t size); - int (*erase)(struct blockdevice *device, size_t addr, size_t size); - int (*trim)(struct blockdevice *device, size_t addr, size_t size); - uint64_t (*size)(struct blockdevice *device); + int (*read)(struct blockdevice *device, const void *buffer, bd_size_t addr, bd_size_t size); + int (*program)(struct blockdevice *device, const void *buffer, bd_size_t addr, bd_size_t size); + int (*erase)(struct blockdevice *device, bd_size_t addr, bd_size_t size); + int (*trim)(struct blockdevice *device, bd_size_t addr, bd_size_t size); + bd_size_t (*size)(struct blockdevice *device); size_t read_size; size_t erase_size; size_t program_size; diff --git a/include/filesystem/ChaN/ffconf.h b/include/filesystem/ChaN/ffconf.h index edef181..440b9c6 100644 --- a/include/filesystem/ChaN/ffconf.h +++ b/include/filesystem/ChaN/ffconf.h @@ -207,7 +207,7 @@ / GET_SECTOR_SIZE command. */ -#define FF_LBA64 0 +#define FF_LBA64 1 /* This option switches support for 64-bit LBA. (0:Disable or 1:Enable) / To enable the 64-bit LBA, also exFAT needs to be enabled. (FF_FS_EXFAT == 1) */ diff --git a/src/blockdevice/flash.c b/src/blockdevice/flash.c index f4fc9ee..2b2aa91 100644 --- a/src/blockdevice/flash.c +++ b/src/blockdevice/flash.c @@ -41,51 +41,51 @@ static int sync(blockdevice_t *device) { return 0; } -static int read(blockdevice_t *device, const void *buffer, size_t addr, size_t size) { +static int read(blockdevice_t *device, const void *buffer, bd_size_t addr, bd_size_t size) { blockdevice_flash_config_t *config = device->config; mutex_enter_blocking(&config->_mutex); - const uint8_t *flash_contents = (const uint8_t *)(XIP_BASE + flash_target_offset(device) + addr); - memcpy((uint8_t *)buffer, flash_contents, size); + const uint8_t *flash_contents = (const uint8_t *)(XIP_BASE + flash_target_offset(device) + (size_t)addr); + memcpy((uint8_t *)buffer, flash_contents, (size_t)size); mutex_exit(&config->_mutex); return BD_ERROR_OK; } -static int erase(blockdevice_t *device, size_t addr, size_t size) { +static int erase(blockdevice_t *device, bd_size_t addr, bd_size_t size) { blockdevice_flash_config_t *config = device->config; mutex_enter_blocking(&config->_mutex); uint32_t ints = save_and_disable_interrupts(); - flash_range_erase(flash_target_offset(device) + addr, size); + flash_range_erase(flash_target_offset(device) + addr, (size_t)size); restore_interrupts(ints); mutex_exit(&config->_mutex); return BD_ERROR_OK; } -static int program(blockdevice_t *device, const void *buffer, size_t addr, size_t size) { +static int program(blockdevice_t *device, const void *buffer, bd_size_t addr, bd_size_t size) { blockdevice_flash_config_t *config = device->config; mutex_enter_blocking(&config->_mutex); uint32_t ints = save_and_disable_interrupts(); - flash_range_program(flash_target_offset(device) + addr, buffer, size); + flash_range_program(flash_target_offset(device) + addr, buffer, (size_t)size); restore_interrupts(ints); mutex_exit(&config->_mutex); return BD_ERROR_OK; } -static int trim(blockdevice_t *device, size_t addr, size_t size) { +static int trim(blockdevice_t *device, bd_size_t addr, bd_size_t size) { (void)device; (void)addr; (void)size; return BD_ERROR_OK; } -static uint64_t size(blockdevice_t *device) { +static bd_size_t size(blockdevice_t *device) { blockdevice_flash_config_t *config = device->config; - return (uint64_t)config->length; + return (bd_size_t)config->length; } blockdevice_t *blockdevice_flash_create(uint32_t start, size_t length) { diff --git a/src/blockdevice/heap.c b/src/blockdevice/heap.c index 0c4976e..5f76bbe 100644 --- a/src/blockdevice/heap.c +++ b/src/blockdevice/heap.c @@ -94,47 +94,47 @@ static int sync(blockdevice_t *device) { return BD_ERROR_OK; } -static int read(blockdevice_t *device, const void *buffer, size_t addr, size_t length) { +static int read(blockdevice_t *device, const void *buffer, bd_size_t addr, bd_size_t length) { blockdevice_heap_config_t *config = device->config; mutex_enter_blocking(&config->_mutex); - memcpy((uint8_t *)buffer, config->heap + addr, length); + memcpy((uint8_t *)buffer, config->heap + (size_t)addr, (size_t)length); mutex_exit(&config->_mutex); return BD_ERROR_OK; } -static int erase(blockdevice_t *device, size_t addr, size_t length) { +static int erase(blockdevice_t *device, bd_size_t addr, bd_size_t length) { blockdevice_heap_config_t *config = device->config; mutex_enter_blocking(&config->_mutex); assert(config->heap != NULL); - memset(config->heap + addr, PICO_VFS_BLOCKDEVICE_HEAP_ERASE_VALUE, length); + memset(config->heap + (size_t)addr, PICO_VFS_BLOCKDEVICE_HEAP_ERASE_VALUE, (size_t)length); mutex_exit(&config->_mutex); return BD_ERROR_OK; } -static int program(blockdevice_t *device, const void *buffer, size_t addr, size_t length) { +static int program(blockdevice_t *device, const void *buffer, bd_size_t addr, bd_size_t length) { blockdevice_heap_config_t *config = device->config; mutex_enter_blocking(&config->_mutex); - memcpy(config->heap + addr, buffer, length); + memcpy(config->heap + (size_t)addr, buffer, (size_t)length); mutex_exit(&config->_mutex); return BD_ERROR_OK; } -static int trim(blockdevice_t *device, size_t addr, size_t length) { +static int trim(blockdevice_t *device, bd_size_t addr, bd_size_t length) { (void)device; (void)addr; (void)length; return BD_ERROR_OK; } -static uint64_t size(blockdevice_t *device) { +static bd_size_t size(blockdevice_t *device) { blockdevice_heap_config_t *config = device->config; - return (uint64_t)config->size; + return (bd_size_t)config->size; } blockdevice_t *blockdevice_heap_create(size_t length) { diff --git a/src/blockdevice/loopback.c b/src/blockdevice/loopback.c index 8e5e9c4..96c5f66 100644 --- a/src/blockdevice/loopback.c +++ b/src/blockdevice/loopback.c @@ -75,7 +75,7 @@ static int __sync(blockdevice_t *device) { return BD_ERROR_OK; } -static int __read(blockdevice_t *device, const void *buffer, size_t addr, size_t length) { +static int __read(blockdevice_t *device, const void *buffer, bd_size_t addr, bd_size_t length) { blockdevice_loopback_config_t *config = device->config; mutex_enter_blocking(&config->_mutex); @@ -85,12 +85,12 @@ static int __read(blockdevice_t *device, const void *buffer, size_t addr, size_t return -errno; } - ssize_t read_size = read(config->fildes, (void *)buffer, length); + ssize_t read_size = read(config->fildes, (void *)buffer, (size_t)length); if (read_size == -1) { mutex_exit(&config->_mutex); return -errno; } - if ((size_t)read_size < length) { + if ((size_t)read_size < (size_t)length) { size_t remind = length - read_size; memset((void *)buffer + read_size, 0, remind); } @@ -99,14 +99,14 @@ static int __read(blockdevice_t *device, const void *buffer, size_t addr, size_t return BD_ERROR_OK; } -static int erase(blockdevice_t *device, size_t addr, size_t length) { +static int erase(blockdevice_t *device, bd_size_t addr, bd_size_t length) { (void)device; (void)addr; (void)length; return BD_ERROR_OK; } -static int program(blockdevice_t *device, const void *buffer, size_t addr, size_t length) { +static int program(blockdevice_t *device, const void *buffer, bd_size_t addr, bd_size_t length) { blockdevice_loopback_config_t *config = device->config; mutex_enter_blocking(&config->_mutex); @@ -115,7 +115,7 @@ static int program(blockdevice_t *device, const void *buffer, size_t addr, size_ mutex_exit(&config->_mutex); return -errno; } - ssize_t write_size = write(config->fildes, buffer, length); + ssize_t write_size = write(config->fildes, buffer, (size_t)length); if (write_size == -1) { mutex_exit(&config->_mutex); return -errno; @@ -125,16 +125,16 @@ static int program(blockdevice_t *device, const void *buffer, size_t addr, size_ return BD_ERROR_OK; } -static int trim(blockdevice_t *device, size_t addr, size_t length) { +static int trim(blockdevice_t *device, bd_size_t addr, bd_size_t length) { (void)device; (void)addr; (void)length; return BD_ERROR_OK; } -static uint64_t size(blockdevice_t *device) { +static bd_size_t size(blockdevice_t *device) { blockdevice_loopback_config_t *config = device->config; - return (uint64_t)config->capacity; + return (bd_size_t)config->capacity; } blockdevice_t *blockdevice_loopback_create(const char *path, size_t capacity, size_t block_size) { diff --git a/src/blockdevice/sd.c b/src/blockdevice/sd.c index fcdbdd6..9f04f4c 100644 --- a/src/blockdevice/sd.c +++ b/src/blockdevice/sd.c @@ -755,13 +755,13 @@ static int init(blockdevice_t *device) { return BD_ERROR_OK; } -static bool is_valid_read(blockdevice_t *device, size_t addr, size_t size) { +static bool is_valid_read(blockdevice_t *device, bd_size_t addr, bd_size_t size) { return (addr % device->read_size == 0 && size % device->read_size == 0 && addr + size <= device->size(device)); } -static bool is_valid_program(blockdevice_t *device, size_t addr, size_t size) { +static bool is_valid_program(blockdevice_t *device, bd_size_t addr, bd_size_t size) { return (addr % device->program_size == 0 && size % device->program_size == 0 && addr + size <= device->size(device)); @@ -807,7 +807,7 @@ static int _read(void *_config, uint8_t *buffer, uint32_t length) { return 0; } -static int read(blockdevice_t *device, const void *_buffer, size_t addr, size_t size) { +static int read(blockdevice_t *device, const void *_buffer, bd_size_t addr, bd_size_t size) { blockdevice_sd_config_t *config = device->config; mutex_enter_blocking(&config->_mutex); @@ -895,7 +895,7 @@ static uint8_t _write(void *_config, const uint8_t *buffer, uint8_t token, uint3 } -static int program(blockdevice_t *device, const void *_buffer, size_t addr, size_t size) { +static int program(blockdevice_t *device, const void *_buffer, bd_size_t addr, bd_size_t size) { blockdevice_sd_config_t *config = device->config; mutex_enter_blocking(&config->_mutex); @@ -970,14 +970,14 @@ static int program(blockdevice_t *device, const void *_buffer, size_t addr, size return status; } -static int erase(blockdevice_t *device, size_t addr, size_t size) { +static int erase(blockdevice_t *device, bd_size_t addr, bd_size_t size) { (void)device; (void)addr; (void)size; return 0; } -static bool _is_valid_trim(blockdevice_t *device, size_t addr, size_t size) { +static bool _is_valid_trim(blockdevice_t *device, bd_size_t addr, bd_size_t size) { blockdevice_sd_config_t *config = device->config; return (addr % config->erase_size == 0 && @@ -985,7 +985,7 @@ static bool _is_valid_trim(blockdevice_t *device, size_t addr, size_t size) { addr + size <= device->size(device)); } -static int trim(blockdevice_t *device, size_t addr, size_t size) { +static int trim(blockdevice_t *device, bd_size_t addr, bd_size_t size) { blockdevice_sd_config_t *config = device->config; mutex_enter_blocking(&config->_mutex); @@ -1025,7 +1025,7 @@ static int trim(blockdevice_t *device, size_t addr, size_t size) { return status; } -static uint64_t size(blockdevice_t *device) { +static bd_size_t size(blockdevice_t *device) { blockdevice_sd_config_t *config = device->config; return config->block_size * config->total_sectors; } diff --git a/src/filesystem/fat.c b/src/filesystem/fat.c index 43ebe33..1f865eb 100644 --- a/src/filesystem/fat.c +++ b/src/filesystem/fat.c @@ -113,8 +113,8 @@ DSTATUS disk_status(BYTE pdrv) { DRESULT disk_read(BYTE pdrv, BYTE *buff, LBA_t sector, UINT count) { DWORD ssize = disk_get_sector_size(pdrv); - size_t addr = sector * ssize; - size_t size = count * ssize; + bd_size_t addr = (bd_size_t)sector * ssize; + bd_size_t size = count * ssize; int err = _ffs[(int)pdrv]->read(_ffs[(int)pdrv], (const void *)buff, addr, size); return err ? RES_PARERR : RES_OK; } @@ -122,8 +122,8 @@ DRESULT disk_read(BYTE pdrv, BYTE *buff, LBA_t sector, UINT count) { DRESULT disk_write(BYTE pdrv, const BYTE *buff, LBA_t sector, UINT count) { debug_if(FFS_DBG, "disk_write [%d]\n", pdrv); DWORD ssize = disk_get_sector_size(pdrv); - size_t addr = sector * ssize; - size_t size = count * ssize; + bd_size_t addr = (bd_size_t)sector * ssize; + bd_size_t size = count * ssize; int err = _ffs[pdrv]->erase(_ffs[pdrv], addr, size); if (err) { @@ -170,8 +170,8 @@ DRESULT disk_ioctl(BYTE pdrv, BYTE cmd, void *buff) { } else { DWORD *sectors = (DWORD *)buff; DWORD ssize = disk_get_sector_size(pdrv); - size_t addr = sectors[0] * ssize; - size_t size = (sectors[1] - sectors[0] + 1) * ssize; + bd_size_t addr = (bd_size_t)sectors[0] * ssize; + bd_size_t size = (bd_size_t)(sectors[1] - sectors[0] + 1) * ssize; int err = _ffs[pdrv]->trim(_ffs[pdrv], addr, size); return err ? RES_PARERR : RES_OK; } @@ -240,7 +240,7 @@ static int format(filesystem_t *fs, blockdevice_t *device) { } // erase first handful of blocks - size_t header = 2 * device->erase_size; + bd_size_t header = 2 * device->erase_size; int err = device->erase(device, 0, header); if (err) { mutex_exit(&context->_mutex_format); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4bfb39e..a3982b7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -24,6 +24,7 @@ pico_enable_stdio_usb(unittests 1) add_subdirectory(integration) add_subdirectory(multicore) +add_subdirectory(large_file) find_program(OPENOCD openocd) if(OPENOCD) diff --git a/tests/large_file/CMakeLists.txt b/tests/large_file/CMakeLists.txt new file mode 100644 index 0000000..94f771f --- /dev/null +++ b/tests/large_file/CMakeLists.txt @@ -0,0 +1,22 @@ +set(CMAKE_BUILD_TYPE Debug) + +add_executable(large_file main.c) +target_compile_options(large_file PRIVATE -Werror -Wall -Wextra -Wnull-dereference) +target_link_libraries(large_file PRIVATE + pico_stdlib + blockdevice_sd + filesystem_fat + filesystem_vfs +) +target_link_options(large_file PRIVATE -Wl,--print-memory-usage) +pico_add_extra_outputs(large_file) +pico_enable_stdio_usb(large_file 1) + + +find_program(OPENOCD openocd) +if(OPENOCD) + add_custom_target(run_large_file + COMMAND ${OPENOCD} -f interface/cmsis-dap.cfg -f target/rp2040.cfg -c "adapter speed 5000" -c "program large_file.elf verify reset exit" + DEPENDS large_file + ) +endif() diff --git a/tests/large_file/main.c b/tests/large_file/main.c new file mode 100644 index 0000000..7b552b7 --- /dev/null +++ b/tests/large_file/main.c @@ -0,0 +1,169 @@ +#include "blockdevice/sd.h" +#include "filesystem/fat.h" +#include "filesystem/vfs.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define COLOR_GREEN(format) ("\e[32m" format "\e[0m") +#define BUFFER_SIZE (1024 * 64) +#define HUGE_FILE_SIZE (1U * 1024U * 1024U * 1024U) + +static void print_progress(const char *label, uint64_t current, uint64_t total) { + int num_dots = (int)((double)current / total * (50 - strlen(label))); + int num_spaces = (50 - strlen(label)) - num_dots; + + printf("\r%s ", label); + for (int i = 0; i < num_dots; i++) { + printf("."); + } + for (int i = 0; i < num_spaces; i++) { + printf(" "); + } + printf(" %" PRIu64 "/%" PRIu64 " bytes", current, total); +} + +bool fs_init(void) { + blockdevice_t *sd = blockdevice_sd_create(spi0, + PICO_DEFAULT_SPI_TX_PIN, + PICO_DEFAULT_SPI_RX_PIN, + PICO_DEFAULT_SPI_SCK_PIN, + PICO_DEFAULT_SPI_CSN_PIN, + 24 * MHZ, + false); + filesystem_t *fat = filesystem_fat_create(); + + printf("format / with FAT\n"); + int err = fs_format(fat, sd); + if (err == -1) { + printf("fs_format error: %s", strerror(errno)); + return false; + } + err = fs_mount("/", fat, sd); + if (err == -1) { + printf("fs_mount error: %s", strerror(errno)); + return false; + } + return true; +} + +static uint32_t xor_rand(uint32_t *seed) { + *seed ^= *seed << 13; + *seed ^= *seed >> 17; + *seed ^= *seed << 5; + return *seed; +} + +static uint32_t xor_rand_32bit(uint32_t *seed) { + return xor_rand(seed); +} + + +static uint8_t buffer[BUFFER_SIZE] = {0}; + +static void huge_file_write(uint32_t seed) { + const char *label = "Write"; + absolute_time_t start_at = get_absolute_time(); + char path[256] = {0}; + sprintf(path, "/huge.%lu", seed); + int fd = open(path, O_WRONLY|O_CREAT); + if (fd == -1) { + printf("open error: %s\n", strerror(errno)); + return; + } + + uint32_t counter = seed; + xor_rand(&counter); + uint64_t remind = HUGE_FILE_SIZE; + while (remind > 0) { + size_t chunk = remind % (uint64_t)sizeof(buffer) ? remind % sizeof(buffer) : sizeof(buffer); + uint32_t *b = (uint32_t *)buffer; + for (size_t j = 0; j < (chunk / sizeof(uint32_t)); j++) { + b[j] = xor_rand_32bit(&counter); + } + + ssize_t write_size = write(fd, buffer, chunk); + if (write_size == -1) { + printf("write: error: %s\n", strerror(errno)); + return; + } + remind = remind - write_size; + print_progress(label, HUGE_FILE_SIZE - remind, HUGE_FILE_SIZE); + } + + int err = close(fd); + if (err == -1) { + printf("close error: %s\n", strerror(errno)); + return; + } + + double duration = (double)absolute_time_diff_us(start_at, get_absolute_time()) / 1000 / 1000; + printf(" %.1f KB/s\n", (double)(HUGE_FILE_SIZE) / duration / 1024); +} + +static void huge_file_read(uint32_t seed) { + const char *label = "Read"; + absolute_time_t start_at = get_absolute_time(); + char path[256] = {0}; + sprintf(path, "/huge.%lu", seed); + int fd = open(path, O_RDONLY); + if (fd == -1) { + printf("open error: %s\n", strerror(errno)); + return; + } + + uint32_t counter = seed; + xor_rand(&counter); + uint64_t remind = HUGE_FILE_SIZE; + while (remind > 0) { + size_t chunk = remind % sizeof(buffer) ? remind % sizeof(buffer) : sizeof(buffer); + ssize_t read_size = read(fd, buffer, chunk); + if (read_size == -1) { + printf("read error: %s\n", strerror(errno)); + return; + } + + uint32_t *b = (uint32_t *)buffer; + for (size_t j = 0; j < chunk / sizeof(uint32_t); j++) { + volatile uint32_t v = xor_rand_32bit(&counter); + if (b[j] != v) { + printf("data mismatch\n"); + return; + } + } + remind = remind - read_size; + print_progress(label, HUGE_FILE_SIZE - remind, HUGE_FILE_SIZE); + } + + int err = close(fd); + if (err == -1) { + printf("close error: %s\n", strerror(errno)); + return; + } + + double duration = (double)absolute_time_diff_us(start_at, get_absolute_time()) / 1000 / 1000; + printf(" %.1f KB/s\n", (double)(HUGE_FILE_SIZE) / duration / 1024); +} + + +int main(void) { + stdio_init_all(); + fs_init(); + + printf("10GB write/read test:\n"); + for (size_t i = 1; i <= 10; i++) { + huge_file_write(i); + huge_file_read(i); + } + + printf(COLOR_GREEN("All tests ok\n")); + + while (1) + tight_loop_contents(); +}