Improved storage handling over 4 GB of FAT

LIMITATION.md

@@ -4,11 +4,13 @@ When using pico-vfs, there are several limitations and behavior specifics that u
 
 ## File System Limitations
 
-1. **Multiple Open Instances**: The current implementations of littlefs and FatFs do not support opening the same file multiple times simultaneously[^1][^2]. This limitation can affect scenarios where multiple accesses to the same file are required concurrently.
+1. **Max File Size for FAT**: The maximum single file size of a FAT file system depends on the capacity of the storage medium. Check the size of the SD card used and the type of FAT (FAT16/32/ExFat) automatically assigned.
 
-2. **`core1` Usage with Onboard Flash Block Device**: When operating file systems using the onboard flash block device on `core1`, it is necessary to run the operations from RAM[^3].
+2. **Multiple Open Instances**: The current implementations of littlefs and FatFs do not support opening the same file multiple times simultaneously[^1][^2]. This limitation can affect scenarios where multiple accesses to the same file are required concurrently.
 
-3. **Memory map IO**: `mmap` system call not supported.
+3. **`core1` Usage with Onboard Flash Block Device**: When operating file systems using the onboard flash block device on `core1`, it is necessary to run the operations from RAM[^3].
+
+4. **Memory maped IO**: `mmap` system call not supported.
 
 We recommend reviewing these limitations before designing systems that heavily rely on multicore operations or require high file access availability.
 

include/blockdevice/blockdevice.h

@@ -17,6 +17,8 @@ extern "C" {
 #include <stdint.h>
 #include <stdbool.h>
 
+typedef uint64_t bd_size_t;
+
 enum bd_error {
     BD_ERROR_OK                 = 0,     /*!< no error */
     BD_ERROR_DEVICE_ERROR       = -4001, /*!< device specific error */
@@ -31,11 +33,11 @@ typedef struct blockdevice {
     int (*init)(struct blockdevice *device);
     int (*deinit)(struct blockdevice *device);
     int (*sync)(struct blockdevice *device);
-    int (*read)(struct blockdevice *device, const void *buffer, size_t addr, size_t size);
-    int (*program)(struct blockdevice *device, const void *buffer, size_t addr, size_t size);
-    int (*erase)(struct blockdevice *device, size_t addr, size_t size);
-    int (*trim)(struct blockdevice *device, size_t addr, size_t size);
-    uint64_t (*size)(struct blockdevice *device);
+    int (*read)(struct blockdevice *device, const void *buffer, bd_size_t addr, bd_size_t size);
+    int (*program)(struct blockdevice *device, const void *buffer, bd_size_t addr, bd_size_t size);
+    int (*erase)(struct blockdevice *device, bd_size_t addr, bd_size_t size);
+    int (*trim)(struct blockdevice *device, bd_size_t addr, bd_size_t size);
+    bd_size_t (*size)(struct blockdevice *device);
     size_t read_size;
     size_t erase_size;
     size_t program_size;

include/filesystem/ChaN/ffconf.h

@@ -207,7 +207,7 @@
 /  GET_SECTOR_SIZE command. */
 
 
-#define FF_LBA64		0
+#define FF_LBA64		1
 /* This option switches support for 64-bit LBA. (0:Disable or 1:Enable)
 /  To enable the 64-bit LBA, also exFAT needs to be enabled. (FF_FS_EXFAT == 1) */
 

src/blockdevice/flash.c

@@ -41,51 +41,51 @@ static int sync(blockdevice_t *device) {
     return 0;
 }
 
-static int read(blockdevice_t *device, const void *buffer, size_t addr, size_t size) {
+static int read(blockdevice_t *device, const void *buffer, bd_size_t addr, bd_size_t size) {
     blockdevice_flash_config_t *config = device->config;
 
     mutex_enter_blocking(&config->_mutex);
-    const uint8_t *flash_contents = (const uint8_t *)(XIP_BASE + flash_target_offset(device) + addr);
-    memcpy((uint8_t *)buffer, flash_contents, size);
+    const uint8_t *flash_contents = (const uint8_t *)(XIP_BASE + flash_target_offset(device) + (size_t)addr);
+    memcpy((uint8_t *)buffer, flash_contents, (size_t)size);
     mutex_exit(&config->_mutex);
 
     return BD_ERROR_OK;
 }
 
-static int erase(blockdevice_t *device, size_t addr, size_t size) {
+static int erase(blockdevice_t *device, bd_size_t addr, bd_size_t size) {
     blockdevice_flash_config_t *config = device->config;
 
     mutex_enter_blocking(&config->_mutex);
     uint32_t ints = save_and_disable_interrupts();
-    flash_range_erase(flash_target_offset(device) + addr, size);
+    flash_range_erase(flash_target_offset(device) + addr, (size_t)size);
     restore_interrupts(ints);
     mutex_exit(&config->_mutex);
 
     return BD_ERROR_OK;
 }
 
-static int program(blockdevice_t *device, const void *buffer, size_t addr, size_t size) {
+static int program(blockdevice_t *device, const void *buffer, bd_size_t addr, bd_size_t size) {
     blockdevice_flash_config_t *config = device->config;
 
     mutex_enter_blocking(&config->_mutex);
     uint32_t ints = save_and_disable_interrupts();
-    flash_range_program(flash_target_offset(device) + addr, buffer, size);
+    flash_range_program(flash_target_offset(device) + addr, buffer, (size_t)size);
     restore_interrupts(ints);
     mutex_exit(&config->_mutex);
 
     return BD_ERROR_OK;
 }
 
-static int trim(blockdevice_t *device, size_t addr, size_t size) {
+static int trim(blockdevice_t *device, bd_size_t addr, bd_size_t size) {
     (void)device;
     (void)addr;
     (void)size;
     return BD_ERROR_OK;
 }
 
-static uint64_t size(blockdevice_t *device) {
+static bd_size_t size(blockdevice_t *device) {
     blockdevice_flash_config_t *config = device->config;
-    return (uint64_t)config->length;
+    return (bd_size_t)config->length;
 }
 
 blockdevice_t *blockdevice_flash_create(uint32_t start, size_t length) {

src/blockdevice/heap.c

@@ -94,47 +94,47 @@ static int sync(blockdevice_t *device) {
     return BD_ERROR_OK;
 }
 
-static int read(blockdevice_t *device, const void *buffer, size_t addr, size_t length) {
+static int read(blockdevice_t *device, const void *buffer, bd_size_t addr, bd_size_t length) {
     blockdevice_heap_config_t *config = device->config;
     mutex_enter_blocking(&config->_mutex);
 
-    memcpy((uint8_t *)buffer, config->heap + addr, length);
+    memcpy((uint8_t *)buffer, config->heap + (size_t)addr, (size_t)length);
 
     mutex_exit(&config->_mutex);
     return BD_ERROR_OK;
 }
 
-static int erase(blockdevice_t *device, size_t addr, size_t length) {
+static int erase(blockdevice_t *device, bd_size_t addr, bd_size_t length) {
     blockdevice_heap_config_t *config = device->config;
     mutex_enter_blocking(&config->_mutex);
 
     assert(config->heap != NULL);
-    memset(config->heap + addr, PICO_VFS_BLOCKDEVICE_HEAP_ERASE_VALUE, length);
+    memset(config->heap + (size_t)addr, PICO_VFS_BLOCKDEVICE_HEAP_ERASE_VALUE, (size_t)length);
 
     mutex_exit(&config->_mutex);
     return BD_ERROR_OK;
 }
 
-static int program(blockdevice_t *device, const void *buffer, size_t addr, size_t length) {
+static int program(blockdevice_t *device, const void *buffer, bd_size_t addr, bd_size_t length) {
     blockdevice_heap_config_t *config = device->config;
     mutex_enter_blocking(&config->_mutex);
 
-    memcpy(config->heap + addr, buffer, length);
+    memcpy(config->heap + (size_t)addr, buffer, (size_t)length);
 
     mutex_exit(&config->_mutex);
     return BD_ERROR_OK;
 }
 
-static int trim(blockdevice_t *device, size_t addr, size_t length) {
+static int trim(blockdevice_t *device, bd_size_t addr, bd_size_t length) {
     (void)device;
     (void)addr;
     (void)length;
     return BD_ERROR_OK;
 }
 
-static uint64_t size(blockdevice_t *device) {
+static bd_size_t size(blockdevice_t *device) {
     blockdevice_heap_config_t *config = device->config;
-    return (uint64_t)config->size;
+    return (bd_size_t)config->size;
 }
 
 blockdevice_t *blockdevice_heap_create(size_t length) {

src/blockdevice/loopback.c

@@ -75,7 +75,7 @@ static int __sync(blockdevice_t *device) {
     return BD_ERROR_OK;
 }
 
-static int __read(blockdevice_t *device, const void *buffer, size_t addr, size_t length) {
+static int __read(blockdevice_t *device, const void *buffer, bd_size_t addr, bd_size_t length) {
     blockdevice_loopback_config_t *config = device->config;
     mutex_enter_blocking(&config->_mutex);
 
@@ -85,12 +85,12 @@ static int __read(blockdevice_t *device, const void *buffer, size_t addr, size_t
         return -errno;
     }
 
-    ssize_t read_size = read(config->fildes, (void *)buffer, length);
+    ssize_t read_size = read(config->fildes, (void *)buffer, (size_t)length);
     if (read_size == -1) {
         mutex_exit(&config->_mutex);
         return -errno;
     }
-    if ((size_t)read_size < length) {
+    if ((size_t)read_size < (size_t)length) {
         size_t remind = length - read_size;
         memset((void *)buffer + read_size, 0, remind);
     }
@@ -99,14 +99,14 @@ static int __read(blockdevice_t *device, const void *buffer, size_t addr, size_t
     return BD_ERROR_OK;
 }
 
-static int erase(blockdevice_t *device, size_t addr, size_t length) {
+static int erase(blockdevice_t *device, bd_size_t addr, bd_size_t length) {
     (void)device;
     (void)addr;
     (void)length;
     return BD_ERROR_OK;
 }
 
-static int program(blockdevice_t *device, const void *buffer, size_t addr, size_t length) {
+static int program(blockdevice_t *device, const void *buffer, bd_size_t addr, bd_size_t length) {
     blockdevice_loopback_config_t *config = device->config;
     mutex_enter_blocking(&config->_mutex);
 
@@ -115,7 +115,7 @@ static int program(blockdevice_t *device, const void *buffer, size_t addr, size_
         mutex_exit(&config->_mutex);
         return -errno;
     }
-    ssize_t write_size = write(config->fildes, buffer, length);
+    ssize_t write_size = write(config->fildes, buffer, (size_t)length);
     if (write_size == -1) {
         mutex_exit(&config->_mutex);
         return -errno;
@@ -125,16 +125,16 @@ static int program(blockdevice_t *device, const void *buffer, size_t addr, size_
     return BD_ERROR_OK;
 }
 
-static int trim(blockdevice_t *device, size_t addr, size_t length) {
+static int trim(blockdevice_t *device, bd_size_t addr, bd_size_t length) {
     (void)device;
     (void)addr;
     (void)length;
     return BD_ERROR_OK;
 }
 
-static uint64_t size(blockdevice_t *device) {
+static bd_size_t size(blockdevice_t *device) {
     blockdevice_loopback_config_t *config = device->config;
-    return (uint64_t)config->capacity;
+    return (bd_size_t)config->capacity;
 }
 
 blockdevice_t *blockdevice_loopback_create(const char *path, size_t capacity, size_t block_size) {

src/blockdevice/sd.c

@@ -755,13 +755,13 @@ static int init(blockdevice_t *device) {
     return BD_ERROR_OK;
 }
 
-static bool is_valid_read(blockdevice_t *device, size_t addr, size_t size) {
+static bool is_valid_read(blockdevice_t *device, bd_size_t addr, bd_size_t size) {
     return (addr % device->read_size == 0 &&
             size % device->read_size == 0 &&
             addr + size <= device->size(device));
 }
 
-static bool is_valid_program(blockdevice_t *device, size_t addr, size_t size) {
+static bool is_valid_program(blockdevice_t *device, bd_size_t addr, bd_size_t size) {
     return (addr % device->program_size == 0 &&
             size % device->program_size == 0 &&
             addr + size <= device->size(device));
@@ -807,7 +807,7 @@ static int _read(void *_config, uint8_t *buffer, uint32_t length) {
     return 0;
 }
 
-static int read(blockdevice_t *device, const void *_buffer, size_t addr, size_t size) {
+static int read(blockdevice_t *device, const void *_buffer, bd_size_t addr, bd_size_t size) {
     blockdevice_sd_config_t *config = device->config;
     mutex_enter_blocking(&config->_mutex);
 
@@ -895,7 +895,7 @@ static uint8_t _write(void *_config, const uint8_t *buffer, uint8_t token, uint3
 }
 
 
-static int program(blockdevice_t *device, const void *_buffer, size_t addr, size_t size) {
+static int program(blockdevice_t *device, const void *_buffer, bd_size_t addr, bd_size_t size) {
     blockdevice_sd_config_t *config = device->config;
     mutex_enter_blocking(&config->_mutex);
 
@@ -970,22 +970,22 @@ static int program(blockdevice_t *device, const void *_buffer, size_t addr, size
     return status;
 }
 
-static int erase(blockdevice_t *device, size_t addr, size_t size) {
+static int erase(blockdevice_t *device, bd_size_t addr, bd_size_t size) {
     (void)device;
     (void)addr;
     (void)size;
     return 0;
 }
 
-static bool _is_valid_trim(blockdevice_t *device, size_t addr, size_t size) {
+static bool _is_valid_trim(blockdevice_t *device, bd_size_t addr, bd_size_t size) {
     blockdevice_sd_config_t *config = device->config;
 
     return (addr % config->erase_size == 0 &&
             size % config->erase_size == 0 &&
             addr + size <= device->size(device));
 }
 
-static int trim(blockdevice_t *device, size_t addr, size_t size) {
+static int trim(blockdevice_t *device, bd_size_t addr, bd_size_t size) {
     blockdevice_sd_config_t *config = device->config;
     mutex_enter_blocking(&config->_mutex);
 
@@ -1025,7 +1025,7 @@ static int trim(blockdevice_t *device, size_t addr, size_t size) {
     return status;
 }
 
-static uint64_t size(blockdevice_t *device) {
+static bd_size_t size(blockdevice_t *device) {
     blockdevice_sd_config_t *config = device->config;
     return config->block_size * config->total_sectors;
 }

src/filesystem/fat.c

@@ -113,17 +113,17 @@ DSTATUS disk_status(BYTE pdrv) {
 
 DRESULT disk_read(BYTE pdrv, BYTE *buff, LBA_t sector, UINT count) {
     DWORD ssize = disk_get_sector_size(pdrv);
-    size_t addr = sector * ssize;
-    size_t size = count * ssize;
+    bd_size_t addr = (bd_size_t)sector * ssize;
+    bd_size_t size = count * ssize;
     int err = _ffs[(int)pdrv]->read(_ffs[(int)pdrv], (const void *)buff, addr, size);
     return err ? RES_PARERR : RES_OK;
 }
 
 DRESULT disk_write(BYTE pdrv, const BYTE *buff, LBA_t sector, UINT count) {
     debug_if(FFS_DBG, "disk_write [%d]\n", pdrv);
     DWORD ssize = disk_get_sector_size(pdrv);
-    size_t addr = sector * ssize;
-    size_t size = count * ssize;
+    bd_size_t addr = (bd_size_t)sector * ssize;
+    bd_size_t size = count * ssize;
 
     int err = _ffs[pdrv]->erase(_ffs[pdrv], addr, size);
     if (err) {
@@ -170,8 +170,8 @@ DRESULT disk_ioctl(BYTE pdrv, BYTE cmd, void *buff) {
         } else {
             DWORD *sectors = (DWORD *)buff;
             DWORD ssize = disk_get_sector_size(pdrv);
-            size_t addr = sectors[0] * ssize;
-            size_t size = (sectors[1] - sectors[0] + 1) * ssize;
+            bd_size_t addr = (bd_size_t)sectors[0] * ssize;
+            bd_size_t size = (bd_size_t)(sectors[1] - sectors[0] + 1) * ssize;
             int err = _ffs[pdrv]->trim(_ffs[pdrv], addr, size);
             return err ? RES_PARERR : RES_OK;
         }
@@ -240,7 +240,7 @@ static int format(filesystem_t *fs, blockdevice_t *device) {
     }
 
     // erase first handful of blocks
-    size_t header = 2 * device->erase_size;
+    bd_size_t header = 2 * device->erase_size;
     int err = device->erase(device, 0, header);
     if (err) {
         mutex_exit(&context->_mutex_format);

tests/CMakeLists.txt

@@ -24,6 +24,7 @@ pico_enable_stdio_usb(unittests 1)
 
 add_subdirectory(integration)
 add_subdirectory(multicore)
+add_subdirectory(large_file)
 
 find_program(OPENOCD openocd)
 if(OPENOCD)

tests/large_file/CMakeLists.txt

@@ -0,0 +1,22 @@
+set(CMAKE_BUILD_TYPE Debug)
+
+add_executable(large_file main.c)
+target_compile_options(large_file PRIVATE -Werror -Wall -Wextra -Wnull-dereference)
+target_link_libraries(large_file PRIVATE
+  pico_stdlib
+  blockdevice_sd
+  filesystem_fat
+  filesystem_vfs
+)
+target_link_options(large_file PRIVATE -Wl,--print-memory-usage)
+pico_add_extra_outputs(large_file)
+pico_enable_stdio_usb(large_file 1)
+
+
+find_program(OPENOCD openocd)
+if(OPENOCD)
+  add_custom_target(run_large_file
+    COMMAND ${OPENOCD} -f interface/cmsis-dap.cfg -f target/rp2040.cfg -c "adapter speed 5000" -c "program large_file.elf verify reset exit"
+    DEPENDS large_file
+  )
+endif()