From 0fb1ebc3c0fddfddcd35b771df2248bc329e765c Mon Sep 17 00:00:00 2001 From: sticilface Date: Fri, 6 Nov 2020 20:31:43 +0000 Subject: [PATCH 01/12] Add FSTools with examples of how to convert between SPIFFS and LITTLEFS. --- libraries/FSTools/FSTools.cpp | 249 ++++++++++++++++++ libraries/FSTools/FSTools.h | 95 +++++++ .../examples/Basic_example/Basic_example.ino | 132 ++++++++++ .../custom_FS_example/custom_FS_example.ino | 139 ++++++++++ libraries/FSTools/package.json | 11 + 5 files changed, 626 insertions(+) create mode 100644 libraries/FSTools/FSTools.cpp create mode 100644 libraries/FSTools/FSTools.h create mode 100644 libraries/FSTools/examples/Basic_example/Basic_example.ino create mode 100644 libraries/FSTools/examples/custom_FS_example/custom_FS_example.ino create mode 100644 libraries/FSTools/package.json diff --git a/libraries/FSTools/FSTools.cpp b/libraries/FSTools/FSTools.cpp new file mode 100644 index 0000000000..5bb1940018 --- /dev/null +++ b/libraries/FSTools/FSTools.cpp @@ -0,0 +1,249 @@ +#include "FSTools.h" +#include "LittleFS.h" +#include +#include + + +namespace FST { + + const layout layout_512k32 (0x40273000, 0x4027B000, 0x100, 0x1000 ); + const layout layout_512k64 (0x4026B000, 0x4027B000, 0x100, 0x1000 ); + const layout layout_512k128 (0x4025B000, 0x4027B000, 0x100, 0x1000 ); + + const layout layout_1m64 (0x402EB000, 0x402FB000, 0x100, 0x1000 ); + const layout layout_1m128 (0x402DB000, 0x402FB000, 0x100, 0x1000 ); + const layout layout_1m144 (0x402D7000, 0x402FB000, 0x100, 0x1000 ); + const layout layout_1m160 (0x402D3000, 0x402FB000, 0x100, 0x1000 ); + const layout layout_1m192 (0x402CB000, 0x402FB000, 0x100, 0x1000 ); + const layout layout_1m256 (0x402BB000, 0x402FB000, 0x100, 0x1000 ); + const layout layout_1m512 (0x4027B000, 0x402FB000, 0x100, 0x2000 ); + + const layout layout_2m64 (0x403F0000, 0x403FB000, 0x100, 0x1000 ); + const layout layout_2m128 (0x403E0000, 0x403FB000, 0x100, 0x1000 ); + const layout layout_2m256 (0x403C0000, 0x403FB000, 0x100, 0x1000 ); + const layout layout_2m512 (0x40380000, 0x403FA000, 0x100, 0x2000 ); + const layout layout_2m1m (0x40300000, 0x403FA000, 0x100, 0x2000 ); + + const layout layout_4m1m (0x40500000, 0x405FA000, 0x100, 0x2000 ); + const layout layout_4m2m (0x40400000, 0x405FA000, 0x100, 0x2000 ); + const layout layout_4m3m (0x40300000, 0x405FA000, 0x100, 0x2000 ); + + const layout layout_8m6m (0x40400000, 0x409FA000, 0x100, 0x2000 ); + const layout layout_8m7m (0x40300000, 0x409FA000, 0x100, 0x2000 ); + + const layout layout_16m14m (0x40400000, 0x411FA000, 0x100, 0x2000 ); + const layout layout_16m15m (0x40300000, 0x411FA000, 0x100, 0x2000 ); + +}; + +FSTools::FSTools() +{ + +} + +FSTools::~FSTools() +{ + reset(); +} + + +bool FSTools::attemptToMountFS(fs::FS & fs) +{ + LittleFSConfig littleFSCfg(false); + SPIFFSConfig SPIFFSCfg(false); + // try to apply the "safe" no format config to the FS... doesn't matter which.. just need one to apply.. + if (!fs.setConfig(littleFSCfg) && ! fs.setConfig(SPIFFSCfg) ) + { + return false; + } + return fs.begin(); +} + + +bool FSTools::mountAlternativeFS( FST::FS_t type, const FST::layout layout, bool keepMounted ) +{ + FSConfig * pCfg{nullptr}; + LittleFSConfig littleFSCfg(false); + SPIFFSConfig SPIFFSCfg(false); + reset(); + + switch (type) { + case FST::SPIFFS : { + _pFS.reset( new FS(FSImplPtr(new spiffs_impl::SPIFFSImpl (_getStartAddr(layout) , _getSize(layout) , layout.page, layout.block, 5))) ); + pCfg = &SPIFFSCfg; + break; + } + case FST::LITTLEFS : { + _pFS.reset( new FS(FSImplPtr(new littlefs_impl::LittleFSImpl(_getStartAddr(layout) , _getSize(layout) , layout.page, layout.block, 5))) ); + pCfg = &littleFSCfg; + break; + } + }; + + if (_pFS && pCfg && _pFS->setConfig(*pCfg) && _pFS->begin()) { + if (!keepMounted) { + _pFS->end(); + } + _mounted = true; + _layout = &layout; + return true; + } + + if (_pFS) { + _pFS.reset(); + } + _mounted = false; + return false; + +}; + + +bool FSTools::mounted() +{ + return _mounted; +}; + + + + +bool FSTools::moveFS(fs::FS & destinationFS) +{ + uint32_t sourceFileCount = 0; + uint32_t sourceByteTotal = 0; + bool result = false; + + // do not init new fs... until old one is copied to test... + if (!_mounted || !_pFS) { + //Serial.println("Source FS not mounted"); + return false; + } + + + uint32_t startSector = (ESP.getSketchSize() + FLASH_SECTOR_SIZE - 1) & (~(FLASH_SECTOR_SIZE - 1)); + //uint32_t endSector = (uint32_t)&_FS_start - FST::lowestSPIFFSstartAddr; + uint32_t lowestFSStart = 0x40300000; + + if (_layout) { + lowestFSStart = _layout->startAddr; + } + + uint32_t endSector = lowestFSStart - 0x40200000; + + uint32_t tempFSsize = endSector - startSector; + + //Serial.printf("TempFS: start: %u, end: %u, size: %u, sketchSize = %u, _FS_start = %u\n", startSector, endSector, tempFSsize, ESP.getSketchSize(), (uint32_t)&_FS_start ); + + fileListIterator(*_pFS, "/", [&sourceFileCount, &sourceByteTotal, this](File & f) { + if (f) { + sourceFileCount++; + sourceByteTotal += f.size(); + _dumpFileInfo(f); + } + }); + + //Serial.printf("%u Files Found Total Size = %u\n", sourceFileCount, sourceByteTotal); + //Serial.printf("Size of dummy FS = %u\n", tempFSsize); + + FS tempFS = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(startSector, tempFSsize, FS_PHYS_PAGE, FS_PHYS_BLOCK, 5))); + + if (tempFS.format() && tempFS.begin()) { + if (_copyFS(*_pFS,tempFS)) { + //Serial.println("Files copied to temp File System"); + reset(); + if (destinationFS.format() && destinationFS.begin()) // must format then mount the new FS + { + if (_copyFS(tempFS, destinationFS)) { + //Serial.println("Files copied back to new FS"); + result = true; + } + } else { + //Serial.println("Error Mounting "); + } + } else { + //Serial.println("Copy Failed"); + } + tempFS.end(); + } else { + //Serial.println("Failed to begin() TempFS"); + } + return result; +}; + +void FSTools::reset() +{ + _mounted = false; + _layout = nullptr; + if (_pFS) { + _pFS->end(); + _pFS.reset(); + } +} + +void FSTools::fileListIterator(FS & fs, const char * dirName, FST::FileCb Cb ) +{ + Dir dir = fs.openDir(dirName); + while (dir.next()) { + //const char * fileName = dir.fileName(); + if (dir.isFile()) { + File f = dir.openFile("r"); + if (Cb) { + Cb(f); + } + } else { + fileListIterator(fs, dir.fileName().c_str() , Cb); + } + } +} + + +uint32_t FSTools::_getStartAddr(const FST::layout & layout) +{ + return ( layout.startAddr - 0x40200000 ); +} + +uint32_t FSTools::_getSize(const FST::layout & layout) +{ + return (layout.endAddr - layout.startAddr ); +} + + +void FSTools::_dumpFileInfo(File & f) +{ + if (f) { + //Serial.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size() ); + } +} + +bool FSTools::_copyFS(FS & sourceFS, FS & destFS) +{ + uint32_t sourceFileCount = 0; + uint32_t sourceByteTotal = 0; + + fileListIterator(sourceFS, "/", [&sourceFileCount, &sourceByteTotal](File & f) { + if (f) { + sourceFileCount++; + sourceByteTotal += f.size(); + } + }); + + size_t count = 0; + fileListIterator(sourceFS, "/", [&count, &destFS](File & sourceFile){ + if (sourceFile) { + File destFile = destFS.open(sourceFile.fullName(), "w"); + bool copied = false; + if (destFile) { + destFile.setTimeout(5000); + size_t written = destFile.write(sourceFile); + if (written == sourceFile.size()) { + copied = true; + count++; + } + } + destFile.close(); + sourceFile.close(); + } + }); + + return (count == sourceFileCount); + +} diff --git a/libraries/FSTools/FSTools.h b/libraries/FSTools/FSTools.h new file mode 100644 index 0000000000..ce5eea6534 --- /dev/null +++ b/libraries/FSTools/FSTools.h @@ -0,0 +1,95 @@ +#pragma once + +#include +#include +#include +/* + + A temporary FS is made between the END of the sketch... and the start of the partition you try to mount, to maximise the available space for copying the FS. + The WORST case this is at 0x40300000 which is for a 3m FS on 4m flash.. leaving 460Kb for copying. + + + I have not worked out a way to prevent linking of ALL the layouts except to use #ifdef guards. Ideas welcome as it uses 400B ROM. + */ + +namespace FST { + +struct layout { + layout(uint32_t s, uint32_t e, uint32_t p, uint32_t b) : startAddr(s), endAddr(e), page(p), block(b) {}; + uint32_t startAddr{0}; + uint32_t endAddr{0}; + uint32_t page{0}; + uint32_t block{0}; +}; + +enum FS_t : uint8_t { + SPIFFS, + LITTLEFS +}; + +extern const layout layout_512k32; +extern const layout layout_512k64; +extern const layout layout_512k128; + +extern const layout layout_1m64; +extern const layout layout_1m128; +extern const layout layout_1m144; +extern const layout layout_1m160; +extern const layout layout_1m192; +extern const layout layout_1m256; +extern const layout layout_1m512; + +extern const layout layout_2m64; +extern const layout layout_2m128; +extern const layout layout_2m256; +extern const layout layout_2m512; +extern const layout layout_2m1m; + +extern const layout layout_4m1m; +extern const layout layout_4m2m; +extern const layout layout_4m3m; + +extern const layout layout_8m6m; +extern const layout layout_8m7m; + +extern const layout layout_16m14m; +extern const layout layout_16m15m; + + +typedef std::function FileCb; + + +}; + + + +//376884 +//376452 + + +class FSTools { +public: + + + + FSTools(); + ~FSTools(); + bool attemptToMountFS(fs::FS & fs); + bool mountAlternativeFS( FST::FS_t type, const FST::layout layout, bool keepMounted = false ); + bool mounted(); + bool moveFS(fs::FS & destinationFS); + void reset(); + void fileListIterator(FS & fs, const char * dirName, FST::FileCb Cb ); + +private: + uint32_t _getStartAddr(const FST::layout & layout); + uint32_t _getSize(const FST::layout & layout); + void _dumpFileInfo(File & f); + bool _copyFS(FS & sourceFS, FS & destFS); + + std::unique_ptr _pFS; + bool _mounted{false}; + const FST::layout * _layout{nullptr}; + + +}; \ No newline at end of file diff --git a/libraries/FSTools/examples/Basic_example/Basic_example.ino b/libraries/FSTools/examples/Basic_example/Basic_example.ino new file mode 100644 index 0000000000..b5a7d3279b --- /dev/null +++ b/libraries/FSTools/examples/Basic_example/Basic_example.ino @@ -0,0 +1,132 @@ +/* + + This sketch will convert SPIFFS partitions to LittleFS on ESP8266 + + Change the `TARGET_FS_LAYOUT` to the partition layout that you want target + ie what you are trying to copy from. + + Include in the sketch whatever you want the destination to be, in this case LittleFS, + but it could be SPIFFS to convert back if need be. + + How it works: It creates a LittleFS partition between the end of the sketch and the + start of whatever filesystem you set as target. This has IMPORTANT implications for the + amount of data you can move!!! eg a 4Mb flash module with a 3Mb SPIFFS partition only leaves + about 450k for the temp file system, so if you have more data than that on your 3Mb SPIFFS it + will fail. + + */ + + + +#include +#include +#include +#include +#include +#include + + +#define TARGET_FS_LAYOUT FST::layout_4m3m + +FSTools fstools; + +#ifndef STASSID +#define STASSID "xxxx" +#define STAPSK "xxxx" +#endif + +const char* ssid = STASSID; +const char* password = STAPSK; + + +bool migrateFS() +{ + if (!fstools.attemptToMountFS(LittleFS)) { // Attempts to mount LittleFS without autoformat... + Serial.println(F("Default FS not found")); + if (fstools.mountAlternativeFS( FST::SPIFFS /* FST::LITTLEFS */ , TARGET_FS_LAYOUT , true )) { + Serial.println(F("Alternative found")); + if (fstools.moveFS(LittleFS)) { + Serial.println(F("FileSystem Moved New FS contents:")); + fstools.fileListIterator(LittleFS, "/", [](File & f) { + Serial.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size() ); + }); + return true; + } + } + } + return false; +} + + +void initWiFiOTA() +{ + WiFi.mode(WIFI_STA); + WiFi.begin(ssid, password); + while (WiFi.waitForConnectResult() != WL_CONNECTED) { + Serial.println("Connection Failed! Rebooting..."); + delay(5000); + ESP.restart(); + } + + ArduinoOTA.onStart([]() { + String type; + if (ArduinoOTA.getCommand() == U_FLASH) { + type = "sketch"; + } else { // U_FS + type = "filesystem"; + } + + // NOTE: if updating FS this would be the place to unmount FS using FS.end() + Serial.println("Start updating " + type); + }); + ArduinoOTA.onEnd([]() { + Serial.println("\nEnd"); + }); + ArduinoOTA.onProgress([](unsigned int progress, unsigned int total) { + Serial.printf("Progress: %u%%\r", (progress / (total / 100))); + }); + ArduinoOTA.onError([](ota_error_t error) { + Serial.printf("Error[%u]: ", error); + if (error == OTA_AUTH_ERROR) { + Serial.println("Auth Failed"); + } else if (error == OTA_BEGIN_ERROR) { + Serial.println("Begin Failed"); + } else if (error == OTA_CONNECT_ERROR) { + Serial.println("Connect Failed"); + } else if (error == OTA_RECEIVE_ERROR) { + Serial.println("Receive Failed"); + } else if (error == OTA_END_ERROR) { + Serial.println("End Failed"); + } + }); + ArduinoOTA.begin(); +} + +void setup() { + + WiFi.persistent(false); + WiFi.disconnect(true); + Serial.begin(115200); + + Serial.println(); + Serial.printf("SDK Version: %s\n", ESP.getSdkVersion() ); + Serial.println("Core Version: " + ESP.getCoreVersion() ); + Serial.println("Full Version: " + ESP.getFullVersion() ); + + Serial.printf("Sketch size: %u\n", ESP.getSketchSize()); + Serial.printf("Free size: %u\n", ESP.getFreeSketchSpace()); + + Serial.println("Booting"); + + migrateFS(); // MUST call this before calling your own begin(); + + initWiFiOTA(); + + Serial.println("Ready"); + Serial.print("IP address: "); + Serial.println(WiFi.localIP()); +} + +void loop() { + ArduinoOTA.handle(); +} diff --git a/libraries/FSTools/examples/custom_FS_example/custom_FS_example.ino b/libraries/FSTools/examples/custom_FS_example/custom_FS_example.ino new file mode 100644 index 0000000000..41c5ecb366 --- /dev/null +++ b/libraries/FSTools/examples/custom_FS_example/custom_FS_example.ino @@ -0,0 +1,139 @@ +/* + + This sketch will convert SPIFFS partitions to a custom FS on ESP8266 + + Change the `TARGET_FS_LAYOUT` to the partition layout that you want target + ie what you are trying to copy from. + + This ksetch shows how to create a FS different to the one provided for by the sketch defaults. + This is useful if you need to use an intermediate sketch to move the FS but need to maintain the + sketch size limit of say 512kb. + + How it works: It creates a LittleFS partition between the end of the sketch and the + start of whatever filesystem you set as target. This has IMPORTANT implications for the + amount of data you can move!!! eg a 4Mb flash module with a 3Mb SPIFFS partition only leaves + about 450k for the temp file system, so if you have more data than that on your 3Mb SPIFFS it + will fail. + + */ + + + +#include +#include +#include +#include +#include +#include + + +#define TARGET_FS_LAYOUT FST::layout_4m3m + +const uint32_t startSector = FST::layout_4m1m.startAddr - 0x40200000; +const uint32_t tempFSsize = FST::layout_4m1m.endAddr - FST::layout_4m1m.startAddr; + +fs::FS LittleFS_Different = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(startSector, tempFSsize, FS_PHYS_PAGE, FS_PHYS_BLOCK, 5))); + + +FSTools fstools; + +#ifndef STASSID +#define STASSID "xxxx" +#define STAPSK "xxxx" +#endif + +const char* ssid = STASSID; +const char* password = STAPSK; + + +bool migrateFS() +{ + if (!fstools.attemptToMountFS(LittleFS_Different)) { // Attempts to mount LittleFS without autoformat... + Serial.println(F("Default FS not found")); + if (fstools.mountAlternativeFS( FST::SPIFFS /* FST::LITTLEFS */ , TARGET_FS_LAYOUT , true )) { + Serial.println(F("Alternative found")); + if (fstools.moveFS(LittleFS_Different)) { + Serial.println(F("FileSystem Moved New FS contents:")); + fstools.fileListIterator(LittleFS_Different, "/", [](File & f) { + Serial.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size() ); + }); + return true; + } + } + } + return false; +} + + +void initWiFiOTA() +{ + WiFi.mode(WIFI_STA); + WiFi.begin(ssid, password); + while (WiFi.waitForConnectResult() != WL_CONNECTED) { + Serial.println("Connection Failed! Rebooting..."); + delay(5000); + ESP.restart(); + } + + ArduinoOTA.onStart([]() { + String type; + if (ArduinoOTA.getCommand() == U_FLASH) { + type = "sketch"; + } else { // U_FS + type = "filesystem"; + } + + // NOTE: if updating FS this would be the place to unmount FS using FS.end() + Serial.println("Start updating " + type); + }); + ArduinoOTA.onEnd([]() { + Serial.println("\nEnd"); + }); + ArduinoOTA.onProgress([](unsigned int progress, unsigned int total) { + Serial.printf("Progress: %u%%\r", (progress / (total / 100))); + }); + ArduinoOTA.onError([](ota_error_t error) { + Serial.printf("Error[%u]: ", error); + if (error == OTA_AUTH_ERROR) { + Serial.println("Auth Failed"); + } else if (error == OTA_BEGIN_ERROR) { + Serial.println("Begin Failed"); + } else if (error == OTA_CONNECT_ERROR) { + Serial.println("Connect Failed"); + } else if (error == OTA_RECEIVE_ERROR) { + Serial.println("Receive Failed"); + } else if (error == OTA_END_ERROR) { + Serial.println("End Failed"); + } + }); + ArduinoOTA.begin(); +} + +void setup() { + + WiFi.persistent(false); + WiFi.disconnect(true); + Serial.begin(115200); + + Serial.println(); + Serial.printf("SDK Version: %s\n", ESP.getSdkVersion() ); + Serial.println("Core Version: " + ESP.getCoreVersion() ); + Serial.println("Full Version: " + ESP.getFullVersion() ); + + Serial.printf("Sketch size: %u\n", ESP.getSketchSize()); + Serial.printf("Free size: %u\n", ESP.getFreeSketchSpace()); + + Serial.println("Booting"); + + migrateFS(); // MUST call this before calling your own begin(); + + initWiFiOTA(); + + Serial.println("Ready"); + Serial.print("IP address: "); + Serial.println(WiFi.localIP()); +} + +void loop() { + ArduinoOTA.handle(); +} diff --git a/libraries/FSTools/package.json b/libraries/FSTools/package.json new file mode 100644 index 0000000000..2602e180a7 --- /dev/null +++ b/libraries/FSTools/package.json @@ -0,0 +1,11 @@ +{ + "name": "FSTools", + "keywords": "SPIFFS LittleFS", + "description": "A library that manages convertion between SPIFFS and LITTLEFS as well as mounting partitions outside of sketch default.", + "homepage": "", + "author": "sticilface", + "version": "1.0.0", + "frameworks": "arduino", + "platforms": "esp8266" +} + From 745db9ab55199f280ccf0da1392148a4cfbd63ba Mon Sep 17 00:00:00 2001 From: sticilface Date: Fri, 6 Nov 2020 22:33:01 +0000 Subject: [PATCH 02/12] Oops. Need to pass layout by reference in order to capture the correct address. Took a while to find that. There maybe a better way to store all these configs --- libraries/FSTools/FSTools.cpp | 18 +++++++++--------- libraries/FSTools/FSTools.h | 2 +- libraries/FSTools/package.json | 19 +++++++++---------- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/libraries/FSTools/FSTools.cpp b/libraries/FSTools/FSTools.cpp index 5bb1940018..c281b374fd 100644 --- a/libraries/FSTools/FSTools.cpp +++ b/libraries/FSTools/FSTools.cpp @@ -34,6 +34,7 @@ namespace FST { const layout layout_16m14m (0x40400000, 0x411FA000, 0x100, 0x2000 ); const layout layout_16m15m (0x40300000, 0x411FA000, 0x100, 0x2000 ); + }; FSTools::FSTools() @@ -60,7 +61,7 @@ bool FSTools::attemptToMountFS(fs::FS & fs) } -bool FSTools::mountAlternativeFS( FST::FS_t type, const FST::layout layout, bool keepMounted ) +bool FSTools::mountAlternativeFS( FST::FS_t type, const FST::layout & layout, bool keepMounted ) { FSConfig * pCfg{nullptr}; LittleFSConfig littleFSCfg(false); @@ -119,16 +120,16 @@ bool FSTools::moveFS(fs::FS & destinationFS) } - uint32_t startSector = (ESP.getSketchSize() + FLASH_SECTOR_SIZE - 1) & (~(FLASH_SECTOR_SIZE - 1)); + uint32_t startSector = (ESP.getSketchSize() + FLASH_SECTOR_SIZE - 1) & (~(FLASH_SECTOR_SIZE - 1)); //uint32_t endSector = (uint32_t)&_FS_start - FST::lowestSPIFFSstartAddr; - uint32_t lowestFSStart = 0x40300000; + uint32_t lowestFSStart = 0x40300000; - if (_layout) { - lowestFSStart = _layout->startAddr; - } + if (_layout) { + lowestFSStart = _layout->startAddr; + //Serial.printf("** _layout->startADDR = 0x%08x\n", _layout->startAddr ); + } - uint32_t endSector = lowestFSStart - 0x40200000; - + uint32_t endSector = lowestFSStart - 0x40200000; uint32_t tempFSsize = endSector - startSector; //Serial.printf("TempFS: start: %u, end: %u, size: %u, sketchSize = %u, _FS_start = %u\n", startSector, endSector, tempFSsize, ESP.getSketchSize(), (uint32_t)&_FS_start ); @@ -183,7 +184,6 @@ void FSTools::fileListIterator(FS & fs, const char * dirName, FST::FileCb Cb ) { Dir dir = fs.openDir(dirName); while (dir.next()) { - //const char * fileName = dir.fileName(); if (dir.isFile()) { File f = dir.openFile("r"); if (Cb) { diff --git a/libraries/FSTools/FSTools.h b/libraries/FSTools/FSTools.h index ce5eea6534..2cca0b0e82 100644 --- a/libraries/FSTools/FSTools.h +++ b/libraries/FSTools/FSTools.h @@ -75,7 +75,7 @@ class FSTools { FSTools(); ~FSTools(); bool attemptToMountFS(fs::FS & fs); - bool mountAlternativeFS( FST::FS_t type, const FST::layout layout, bool keepMounted = false ); + bool mountAlternativeFS( FST::FS_t type, const FST::layout & layout, bool keepMounted = false ); bool mounted(); bool moveFS(fs::FS & destinationFS); void reset(); diff --git a/libraries/FSTools/package.json b/libraries/FSTools/package.json index 2602e180a7..d10f43fc87 100644 --- a/libraries/FSTools/package.json +++ b/libraries/FSTools/package.json @@ -1,11 +1,10 @@ { - "name": "FSTools", - "keywords": "SPIFFS LittleFS", - "description": "A library that manages convertion between SPIFFS and LITTLEFS as well as mounting partitions outside of sketch default.", - "homepage": "", - "author": "sticilface", - "version": "1.0.0", - "frameworks": "arduino", - "platforms": "esp8266" -} - + "name": "FSTools", + "keywords": "SPIFFS LittleFS", + "description": "A library that manages convertion between SPIFFS and LITTLEFS as well as mounting partitions outside of sketch default.", + "homepage": "", + "author": "sticilface", + "version": "1.0.0", + "frameworks": "arduino", + "platforms": "esp8266" + } \ No newline at end of file From cbf4073695d9b6b6314461566e24538321042e47 Mon Sep 17 00:00:00 2001 From: sticilface Date: Sat, 7 Nov 2020 08:36:54 +0000 Subject: [PATCH 03/12] Update FSTools.cpp fix ESP.h to Esp.h --- libraries/FSTools/FSTools.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/FSTools/FSTools.cpp b/libraries/FSTools/FSTools.cpp index c281b374fd..fd34514c03 100644 --- a/libraries/FSTools/FSTools.cpp +++ b/libraries/FSTools/FSTools.cpp @@ -1,7 +1,7 @@ #include "FSTools.h" #include "LittleFS.h" #include -#include +#include namespace FST { From d382a33c7d3eaf791f4bbf8e787adbb5d0fb3740 Mon Sep 17 00:00:00 2001 From: sticilface Date: Sat, 7 Nov 2020 19:23:09 +0000 Subject: [PATCH 04/12] Fix unused variable i --- cores/esp8266/FS.h | 1 - 1 file changed, 1 deletion(-) diff --git a/cores/esp8266/FS.h b/cores/esp8266/FS.h index b1f1c528aa..c4715936d4 100644 --- a/cores/esp8266/FS.h +++ b/cores/esp8266/FS.h @@ -88,7 +88,6 @@ class File : public Stream uint8_t obuf[256]; size_t doneLen = 0; size_t sentLen; - int i; while (src.available() > sizeof(obuf)){ src.read(obuf, sizeof(obuf)); From ee1481aa7c827434cfd680703384dce3e7309ea1 Mon Sep 17 00:00:00 2001 From: sticilface Date: Sat, 7 Nov 2020 19:24:31 +0000 Subject: [PATCH 05/12] Parsed with restyle.sh. Compile with all errors. --- libraries/FSTools/FSTools.cpp | 14 ++-- libraries/FSTools/FSTools.h | 9 +-- .../examples/Basic_example/Basic_example.ino | 58 ++++++++--------- .../custom_FS_example/custom_FS_example.ino | 64 +++++++++---------- 4 files changed, 67 insertions(+), 78 deletions(-) diff --git a/libraries/FSTools/FSTools.cpp b/libraries/FSTools/FSTools.cpp index c281b374fd..d71fe1cb26 100644 --- a/libraries/FSTools/FSTools.cpp +++ b/libraries/FSTools/FSTools.cpp @@ -138,7 +138,7 @@ bool FSTools::moveFS(fs::FS & destinationFS) if (f) { sourceFileCount++; sourceByteTotal += f.size(); - _dumpFileInfo(f); + //_dumpFileInfo(f); } }); @@ -207,12 +207,12 @@ uint32_t FSTools::_getSize(const FST::layout & layout) } -void FSTools::_dumpFileInfo(File & f) -{ - if (f) { - //Serial.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size() ); - } -} +// void FSTools::_dumpFileInfo(File & f) +// { +// if (f) { +// Serial.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size() ); +// } +// } bool FSTools::_copyFS(FS & sourceFS, FS & destFS) { diff --git a/libraries/FSTools/FSTools.h b/libraries/FSTools/FSTools.h index 2cca0b0e82..8d48e5cd77 100644 --- a/libraries/FSTools/FSTools.h +++ b/libraries/FSTools/FSTools.h @@ -55,7 +55,6 @@ extern const layout layout_8m7m; extern const layout layout_16m14m; extern const layout layout_16m15m; - typedef std::function FileCb; @@ -63,15 +62,9 @@ typedef std::function FileCb; -//376884 -//376452 - - class FSTools { public: - - FSTools(); ~FSTools(); bool attemptToMountFS(fs::FS & fs); @@ -84,7 +77,7 @@ class FSTools { private: uint32_t _getStartAddr(const FST::layout & layout); uint32_t _getSize(const FST::layout & layout); - void _dumpFileInfo(File & f); +// void _dumpFileInfo(File & f); bool _copyFS(FS & sourceFS, FS & destFS); std::unique_ptr _pFS; diff --git a/libraries/FSTools/examples/Basic_example/Basic_example.ino b/libraries/FSTools/examples/Basic_example/Basic_example.ino index b5a7d3279b..6feca9ea66 100644 --- a/libraries/FSTools/examples/Basic_example/Basic_example.ino +++ b/libraries/FSTools/examples/Basic_example/Basic_example.ino @@ -2,19 +2,19 @@ This sketch will convert SPIFFS partitions to LittleFS on ESP8266 - Change the `TARGET_FS_LAYOUT` to the partition layout that you want target + Change the `TARGET_FS_LAYOUT` to the partition layout that you want target ie what you are trying to copy from. - Include in the sketch whatever you want the destination to be, in this case LittleFS, - but it could be SPIFFS to convert back if need be. + Include in the sketch whatever you want the destination to be, in this case LittleFS, + but it could be SPIFFS to convert back if need be. - How it works: It creates a LittleFS partition between the end of the sketch and the - start of whatever filesystem you set as target. This has IMPORTANT implications for the - amount of data you can move!!! eg a 4Mb flash module with a 3Mb SPIFFS partition only leaves - about 450k for the temp file system, so if you have more data than that on your 3Mb SPIFFS it - will fail. + How it works: It creates a LittleFS partition between the end of the sketch and the + start of whatever filesystem you set as target. This has IMPORTANT implications for the + amount of data you can move!!! eg a 4Mb flash module with a 3Mb SPIFFS partition only leaves + about 450k for the temp file system, so if you have more data than that on your 3Mb SPIFFS it + will fail. - */ +*/ @@ -28,7 +28,7 @@ #define TARGET_FS_LAYOUT FST::layout_4m3m -FSTools fstools; +FSTools fstools; #ifndef STASSID #define STASSID "xxxx" @@ -39,27 +39,25 @@ const char* ssid = STASSID; const char* password = STAPSK; -bool migrateFS() -{ - if (!fstools.attemptToMountFS(LittleFS)) { // Attempts to mount LittleFS without autoformat... - Serial.println(F("Default FS not found")); - if (fstools.mountAlternativeFS( FST::SPIFFS /* FST::LITTLEFS */ , TARGET_FS_LAYOUT , true )) { - Serial.println(F("Alternative found")); - if (fstools.moveFS(LittleFS)) { +bool migrateFS() { + if (!fstools.attemptToMountFS(LittleFS)) { // Attempts to mount LittleFS without autoformat... + Serial.println(F("Default FS not found")); + if (fstools.mountAlternativeFS(FST::SPIFFS /* FST::LITTLEFS */, TARGET_FS_LAYOUT, true)) { + Serial.println(F("Alternative found")); + if (fstools.moveFS(LittleFS)) { Serial.println(F("FileSystem Moved New FS contents:")); fstools.fileListIterator(LittleFS, "/", [](File & f) { - Serial.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size() ); + Serial.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size()); }); - return true; + return true; } } } - return false; + return false; } -void initWiFiOTA() -{ +void initWiFiOTA() { WiFi.mode(WIFI_STA); WiFi.begin(ssid, password); while (WiFi.waitForConnectResult() != WL_CONNECTED) { @@ -107,20 +105,20 @@ void setup() { WiFi.persistent(false); WiFi.disconnect(true); Serial.begin(115200); - - Serial.println(); - Serial.printf("SDK Version: %s\n", ESP.getSdkVersion() ); - Serial.println("Core Version: " + ESP.getCoreVersion() ); - Serial.println("Full Version: " + ESP.getFullVersion() ); + + Serial.println(); + Serial.printf("SDK Version: %s\n", ESP.getSdkVersion()); + Serial.println("Core Version: " + ESP.getCoreVersion()); + Serial.println("Full Version: " + ESP.getFullVersion()); Serial.printf("Sketch size: %u\n", ESP.getSketchSize()); Serial.printf("Free size: %u\n", ESP.getFreeSketchSpace()); - + Serial.println("Booting"); - migrateFS(); // MUST call this before calling your own begin(); + migrateFS(); // MUST call this before calling your own begin(); - initWiFiOTA(); + initWiFiOTA(); Serial.println("Ready"); Serial.print("IP address: "); diff --git a/libraries/FSTools/examples/custom_FS_example/custom_FS_example.ino b/libraries/FSTools/examples/custom_FS_example/custom_FS_example.ino index 41c5ecb366..6ca8d8c9b6 100644 --- a/libraries/FSTools/examples/custom_FS_example/custom_FS_example.ino +++ b/libraries/FSTools/examples/custom_FS_example/custom_FS_example.ino @@ -2,20 +2,20 @@ This sketch will convert SPIFFS partitions to a custom FS on ESP8266 - Change the `TARGET_FS_LAYOUT` to the partition layout that you want target + Change the `TARGET_FS_LAYOUT` to the partition layout that you want target ie what you are trying to copy from. - This ksetch shows how to create a FS different to the one provided for by the sketch defaults. - This is useful if you need to use an intermediate sketch to move the FS but need to maintain the - sketch size limit of say 512kb. + This ksetch shows how to create a FS different to the one provided for by the sketch defaults. + This is useful if you need to use an intermediate sketch to move the FS but need to maintain the + sketch size limit of say 512kb. - How it works: It creates a LittleFS partition between the end of the sketch and the - start of whatever filesystem you set as target. This has IMPORTANT implications for the - amount of data you can move!!! eg a 4Mb flash module with a 3Mb SPIFFS partition only leaves - about 450k for the temp file system, so if you have more data than that on your 3Mb SPIFFS it - will fail. + How it works: It creates a LittleFS partition between the end of the sketch and the + start of whatever filesystem you set as target. This has IMPORTANT implications for the + amount of data you can move!!! eg a 4Mb flash module with a 3Mb SPIFFS partition only leaves + about 450k for the temp file system, so if you have more data than that on your 3Mb SPIFFS it + will fail. - */ +*/ @@ -29,13 +29,13 @@ #define TARGET_FS_LAYOUT FST::layout_4m3m -const uint32_t startSector = FST::layout_4m1m.startAddr - 0x40200000; -const uint32_t tempFSsize = FST::layout_4m1m.endAddr - FST::layout_4m1m.startAddr; +const uint32_t startSector = FST::layout_4m1m.startAddr - 0x40200000; +const uint32_t tempFSsize = FST::layout_4m1m.endAddr - FST::layout_4m1m.startAddr; fs::FS LittleFS_Different = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(startSector, tempFSsize, FS_PHYS_PAGE, FS_PHYS_BLOCK, 5))); -FSTools fstools; +FSTools fstools; #ifndef STASSID #define STASSID "xxxx" @@ -46,27 +46,25 @@ const char* ssid = STASSID; const char* password = STAPSK; -bool migrateFS() -{ - if (!fstools.attemptToMountFS(LittleFS_Different)) { // Attempts to mount LittleFS without autoformat... - Serial.println(F("Default FS not found")); - if (fstools.mountAlternativeFS( FST::SPIFFS /* FST::LITTLEFS */ , TARGET_FS_LAYOUT , true )) { - Serial.println(F("Alternative found")); - if (fstools.moveFS(LittleFS_Different)) { +bool migrateFS() { + if (!fstools.attemptToMountFS(LittleFS_Different)) { // Attempts to mount LittleFS without autoformat... + Serial.println(F("Default FS not found")); + if (fstools.mountAlternativeFS(FST::SPIFFS /* FST::LITTLEFS */, TARGET_FS_LAYOUT, true)) { + Serial.println(F("Alternative found")); + if (fstools.moveFS(LittleFS_Different)) { Serial.println(F("FileSystem Moved New FS contents:")); fstools.fileListIterator(LittleFS_Different, "/", [](File & f) { - Serial.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size() ); + Serial.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size()); }); - return true; + return true; } } } - return false; + return false; } -void initWiFiOTA() -{ +void initWiFiOTA() { WiFi.mode(WIFI_STA); WiFi.begin(ssid, password); while (WiFi.waitForConnectResult() != WL_CONNECTED) { @@ -114,20 +112,20 @@ void setup() { WiFi.persistent(false); WiFi.disconnect(true); Serial.begin(115200); - - Serial.println(); - Serial.printf("SDK Version: %s\n", ESP.getSdkVersion() ); - Serial.println("Core Version: " + ESP.getCoreVersion() ); - Serial.println("Full Version: " + ESP.getFullVersion() ); + + Serial.println(); + Serial.printf("SDK Version: %s\n", ESP.getSdkVersion()); + Serial.println("Core Version: " + ESP.getCoreVersion()); + Serial.println("Full Version: " + ESP.getFullVersion()); Serial.printf("Sketch size: %u\n", ESP.getSketchSize()); Serial.printf("Free size: %u\n", ESP.getFreeSketchSpace()); - + Serial.println("Booting"); - migrateFS(); // MUST call this before calling your own begin(); + migrateFS(); // MUST call this before calling your own begin(); - initWiFiOTA(); + initWiFiOTA(); Serial.println("Ready"); Serial.print("IP address: "); From 4afa2d450250d1f51351ec3922e490d928c21729 Mon Sep 17 00:00:00 2001 From: sticilface Date: Sat, 7 Nov 2020 19:40:22 +0000 Subject: [PATCH 06/12] remove unused variable --- libraries/FSTools/FSTools.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/libraries/FSTools/FSTools.cpp b/libraries/FSTools/FSTools.cpp index 33f6b40b7a..e1fb8a214f 100644 --- a/libraries/FSTools/FSTools.cpp +++ b/libraries/FSTools/FSTools.cpp @@ -230,12 +230,10 @@ bool FSTools::_copyFS(FS & sourceFS, FS & destFS) fileListIterator(sourceFS, "/", [&count, &destFS](File & sourceFile){ if (sourceFile) { File destFile = destFS.open(sourceFile.fullName(), "w"); - bool copied = false; if (destFile) { destFile.setTimeout(5000); size_t written = destFile.write(sourceFile); if (written == sourceFile.size()) { - copied = true; count++; } } From 5022cf0ff752e37a58488a7ea74958201dad8c6e Mon Sep 17 00:00:00 2001 From: sticilface Date: Sat, 7 Nov 2020 19:40:45 +0000 Subject: [PATCH 07/12] fix different sign complication error --- cores/esp8266/FS.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cores/esp8266/FS.h b/cores/esp8266/FS.h index c4715936d4..d3883838ad 100644 --- a/cores/esp8266/FS.h +++ b/cores/esp8266/FS.h @@ -89,7 +89,7 @@ class File : public Stream size_t doneLen = 0; size_t sentLen; - while (src.available() > sizeof(obuf)){ + while (src.available() > (int)sizeof(obuf)){ src.read(obuf, sizeof(obuf)); sentLen = write(obuf, sizeof(obuf)); doneLen = doneLen + sentLen; From 9feb47ab06c2c6a8cf4abe7262d93decf9299433 Mon Sep 17 00:00:00 2001 From: sticilface Date: Sun, 15 Nov 2020 11:01:42 +0000 Subject: [PATCH 08/12] Fix indentation to spaces Run test/restyle.sh Remove commented code Use #ifdef blocks for debugging. `DEBUG_ESP_CORE` and `DEBUG_ESP_PORT` use `static constexpr layout` which saves ROM ~500B for unused vars --- libraries/FSTools/FSTools.cpp | 360 +++++++++++++++++----------------- libraries/FSTools/FSTools.h | 123 ++++++------ 2 files changed, 246 insertions(+), 237 deletions(-) diff --git a/libraries/FSTools/FSTools.cpp b/libraries/FSTools/FSTools.cpp index e1fb8a214f..9ef602c88c 100644 --- a/libraries/FSTools/FSTools.cpp +++ b/libraries/FSTools/FSTools.cpp @@ -4,38 +4,13 @@ #include -namespace FST { - const layout layout_512k32 (0x40273000, 0x4027B000, 0x100, 0x1000 ); - const layout layout_512k64 (0x4026B000, 0x4027B000, 0x100, 0x1000 ); - const layout layout_512k128 (0x4025B000, 0x4027B000, 0x100, 0x1000 ); +#if defined(DEBUG_ESP_CORE) +#define FSTOOLSDEBUG(_1, ...) { DEBUG_ESP_PORT.printf_P( PSTR(_1),##__VA_ARGS__); } +#else +#define FSTOOLSDEBUG(...) {} +#endif - const layout layout_1m64 (0x402EB000, 0x402FB000, 0x100, 0x1000 ); - const layout layout_1m128 (0x402DB000, 0x402FB000, 0x100, 0x1000 ); - const layout layout_1m144 (0x402D7000, 0x402FB000, 0x100, 0x1000 ); - const layout layout_1m160 (0x402D3000, 0x402FB000, 0x100, 0x1000 ); - const layout layout_1m192 (0x402CB000, 0x402FB000, 0x100, 0x1000 ); - const layout layout_1m256 (0x402BB000, 0x402FB000, 0x100, 0x1000 ); - const layout layout_1m512 (0x4027B000, 0x402FB000, 0x100, 0x2000 ); - - const layout layout_2m64 (0x403F0000, 0x403FB000, 0x100, 0x1000 ); - const layout layout_2m128 (0x403E0000, 0x403FB000, 0x100, 0x1000 ); - const layout layout_2m256 (0x403C0000, 0x403FB000, 0x100, 0x1000 ); - const layout layout_2m512 (0x40380000, 0x403FA000, 0x100, 0x2000 ); - const layout layout_2m1m (0x40300000, 0x403FA000, 0x100, 0x2000 ); - - const layout layout_4m1m (0x40500000, 0x405FA000, 0x100, 0x2000 ); - const layout layout_4m2m (0x40400000, 0x405FA000, 0x100, 0x2000 ); - const layout layout_4m3m (0x40300000, 0x405FA000, 0x100, 0x2000 ); - - const layout layout_8m6m (0x40400000, 0x409FA000, 0x100, 0x2000 ); - const layout layout_8m7m (0x40300000, 0x409FA000, 0x100, 0x2000 ); - - const layout layout_16m14m (0x40400000, 0x411FA000, 0x100, 0x2000 ); - const layout layout_16m15m (0x40300000, 0x411FA000, 0x100, 0x2000 ); - - -}; FSTools::FSTools() { @@ -44,204 +19,235 @@ FSTools::FSTools() FSTools::~FSTools() { - reset(); + reset(); } bool FSTools::attemptToMountFS(fs::FS & fs) { - LittleFSConfig littleFSCfg(false); - SPIFFSConfig SPIFFSCfg(false); - // try to apply the "safe" no format config to the FS... doesn't matter which.. just need one to apply.. - if (!fs.setConfig(littleFSCfg) && ! fs.setConfig(SPIFFSCfg) ) - { - return false; - } - return fs.begin(); + LittleFSConfig littleFSCfg(false); + SPIFFSConfig SPIFFSCfg(false); + // try to apply the "safe" no format config to the FS... doesn't matter which.. just need one to apply correctly.. + if (!fs.setConfig(littleFSCfg) && ! fs.setConfig(SPIFFSCfg)) + { + return false; + } + return fs.begin(); } -bool FSTools::mountAlternativeFS( FST::FS_t type, const FST::layout & layout, bool keepMounted ) +bool FSTools::mountAlternativeFS(FST::FS_t type, const FST::layout & layout, bool keepMounted) { - FSConfig * pCfg{nullptr}; + FSConfig * pCfg{nullptr}; LittleFSConfig littleFSCfg(false); - SPIFFSConfig SPIFFSCfg(false); + SPIFFSConfig SPIFFSCfg(false); reset(); - switch (type) { - case FST::SPIFFS : { - _pFS.reset( new FS(FSImplPtr(new spiffs_impl::SPIFFSImpl (_getStartAddr(layout) , _getSize(layout) , layout.page, layout.block, 5))) ); - pCfg = &SPIFFSCfg; - break; - } - case FST::LITTLEFS : { - _pFS.reset( new FS(FSImplPtr(new littlefs_impl::LittleFSImpl(_getStartAddr(layout) , _getSize(layout) , layout.page, layout.block, 5))) ); - pCfg = &littleFSCfg; - break; - } - }; - - if (_pFS && pCfg && _pFS->setConfig(*pCfg) && _pFS->begin()) { - if (!keepMounted) { - _pFS->end(); - } - _mounted = true; - _layout = &layout; - return true; - } - - if (_pFS) { - _pFS.reset(); - } - _mounted = false; - return false; - -}; + switch (type) + { + case FST::SPIFFS : + { + _pFS.reset(new FS(FSImplPtr(new spiffs_impl::SPIFFSImpl(_getStartAddr(layout), _getSize(layout), layout.page, layout.block, 5)))); + pCfg = &SPIFFSCfg; + break; + } + case FST::LITTLEFS : + { + _pFS.reset(new FS(FSImplPtr(new littlefs_impl::LittleFSImpl(_getStartAddr(layout), _getSize(layout), layout.page, layout.block, 5)))); + pCfg = &littleFSCfg; + break; + } + }; + if (_pFS && pCfg && _pFS->setConfig(*pCfg) && _pFS->begin()) + { + if (!keepMounted) + { + _pFS->end(); + } + _mounted = true; + _layout = &layout; + return true; + } -bool FSTools::mounted() -{ - return _mounted; + if (_pFS) + { + _pFS.reset(); + } + _mounted = false; + return false; }; +bool FSTools::mounted() +{ + return _mounted; +}; bool FSTools::moveFS(fs::FS & destinationFS) { - uint32_t sourceFileCount = 0; - uint32_t sourceByteTotal = 0; - bool result = false; - - // do not init new fs... until old one is copied to test... - if (!_mounted || !_pFS) { - //Serial.println("Source FS not mounted"); - return false; - } - - - uint32_t startSector = (ESP.getSketchSize() + FLASH_SECTOR_SIZE - 1) & (~(FLASH_SECTOR_SIZE - 1)); - //uint32_t endSector = (uint32_t)&_FS_start - FST::lowestSPIFFSstartAddr; - uint32_t lowestFSStart = 0x40300000; - - if (_layout) { - lowestFSStart = _layout->startAddr; - //Serial.printf("** _layout->startADDR = 0x%08x\n", _layout->startAddr ); - } - - uint32_t endSector = lowestFSStart - 0x40200000; - uint32_t tempFSsize = endSector - startSector; - - //Serial.printf("TempFS: start: %u, end: %u, size: %u, sketchSize = %u, _FS_start = %u\n", startSector, endSector, tempFSsize, ESP.getSketchSize(), (uint32_t)&_FS_start ); - - fileListIterator(*_pFS, "/", [&sourceFileCount, &sourceByteTotal, this](File & f) { - if (f) { - sourceFileCount++; - sourceByteTotal += f.size(); - //_dumpFileInfo(f); + uint32_t sourceFileCount = 0; + uint32_t sourceByteTotal = 0; + bool result = false; + + if (!_mounted || !_pFS) + { + FSTOOLSDEBUG("Source FS not mounted\n"); + return false; + } + + uint32_t startSector = (ESP.getSketchSize() + FLASH_SECTOR_SIZE - 1) & (~(FLASH_SECTOR_SIZE - 1)); + uint32_t lowestFSStart = 0x40300000; + + if (_layout) + { + lowestFSStart = _layout->startAddr; + FSTOOLSDEBUG("_layout->startADDR = 0x%08x\n", _layout->startAddr); } + + uint32_t endSector = lowestFSStart - 0x40200000; + uint32_t tempFSsize = endSector - startSector; + + FSTOOLSDEBUG("TempFS: start: %u, end: %u, size: %u, sketchSize = %u, _FS_start = %u\n", startSector, endSector, tempFSsize, ESP.getSketchSize(), (uint32_t)&_FS_start); + + fileListIterator(*_pFS, "/", [&sourceFileCount, &sourceByteTotal, this](File & f) + { + if (f) + { + sourceFileCount++; + sourceByteTotal += f.size(); + +#ifdef DEBUG_ESP_CORE + _dumpFileInfo(f); +#endif + + } }); - //Serial.printf("%u Files Found Total Size = %u\n", sourceFileCount, sourceByteTotal); - //Serial.printf("Size of dummy FS = %u\n", tempFSsize); + FSTOOLSDEBUG("%u Files Found Total Size = %u\n", sourceFileCount, sourceByteTotal); + FSTOOLSDEBUG("Size of dummy FS = %u\n", tempFSsize); FS tempFS = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(startSector, tempFSsize, FS_PHYS_PAGE, FS_PHYS_BLOCK, 5))); - if (tempFS.format() && tempFS.begin()) { - if (_copyFS(*_pFS,tempFS)) { - //Serial.println("Files copied to temp File System"); - reset(); - if (destinationFS.format() && destinationFS.begin()) // must format then mount the new FS - { - if (_copyFS(tempFS, destinationFS)) { - //Serial.println("Files copied back to new FS"); - result = true; - } - } else { - //Serial.println("Error Mounting "); - } - } else { - //Serial.println("Copy Failed"); - } - tempFS.end(); - } else { - //Serial.println("Failed to begin() TempFS"); - } - return result; -}; + if (tempFS.format() && tempFS.begin()) + { + if (_copyFS(*_pFS, tempFS)) + { + FSTOOLSDEBUG("Files copied to temp File System\n"); + reset(); + if (destinationFS.format() && destinationFS.begin()) // must format then mount the new FS + { + if (_copyFS(tempFS, destinationFS)) + { + FSTOOLSDEBUG("Files copied back to new FS\n"); + result = true; + } + } + else + { + FSTOOLSDEBUG("Error Mounting\n"); + } + } + else + { + FSTOOLSDEBUG("Copy Failed\n"); + } + tempFS.end(); + } + else + { + FSTOOLSDEBUG("Failed to begin() TempFS\n"); + } + return result; +}; void FSTools::reset() { - _mounted = false; - _layout = nullptr; - if (_pFS) { - _pFS->end(); - _pFS.reset(); - } + _mounted = false; + _layout = nullptr; + if (_pFS) + { + _pFS->end(); + _pFS.reset(); + } } -void FSTools::fileListIterator(FS & fs, const char * dirName, FST::FileCb Cb ) +void FSTools::fileListIterator(FS & fs, const char * dirName, FST::FileCb Cb) { Dir dir = fs.openDir(dirName); - while (dir.next()) { - if (dir.isFile()) { - File f = dir.openFile("r"); - if (Cb) { - Cb(f); + while (dir.next()) + { + if (dir.isFile()) + { + File f = dir.openFile("r"); + if (Cb) + { + Cb(f); + } + } + else + { + fileListIterator(fs, dir.fileName().c_str(), Cb); } - } else { - fileListIterator(fs, dir.fileName().c_str() , Cb); - } } } uint32_t FSTools::_getStartAddr(const FST::layout & layout) { - return ( layout.startAddr - 0x40200000 ); + return (layout.startAddr - 0x40200000); } uint32_t FSTools::_getSize(const FST::layout & layout) { - return (layout.endAddr - layout.startAddr ); -} - + return (layout.endAddr - layout.startAddr); +} -// void FSTools::_dumpFileInfo(File & f) -// { -// if (f) { -// Serial.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size() ); -// } -// } +#ifdef DEBUG_ESP_CORE +void FSTools::_dumpFileInfo(File & f) +{ + if (f) + { + DEBUG_ESP_PORT.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size()); + } +} +#endif bool FSTools::_copyFS(FS & sourceFS, FS & destFS) { - uint32_t sourceFileCount = 0; - uint32_t sourceByteTotal = 0; - - fileListIterator(sourceFS, "/", [&sourceFileCount, &sourceByteTotal](File & f) { - if (f) { - sourceFileCount++; - sourceByteTotal += f.size(); - } + uint32_t sourceFileCount = 0; + uint32_t sourceByteTotal = 0; + + fileListIterator(sourceFS, "/", [&sourceFileCount, &sourceByteTotal](File & f) + { + if (f) + { + sourceFileCount++; + sourceByteTotal += f.size(); + } }); - size_t count = 0; - fileListIterator(sourceFS, "/", [&count, &destFS](File & sourceFile){ - if (sourceFile) { - File destFile = destFS.open(sourceFile.fullName(), "w"); - if (destFile) { - destFile.setTimeout(5000); - size_t written = destFile.write(sourceFile); - if (written == sourceFile.size()) { - count++; - } - } - destFile.close(); - sourceFile.close(); + size_t count = 0; + fileListIterator(sourceFS, "/", [&count, &destFS](File & sourceFile) + { + if (sourceFile) + { + File destFile = destFS.open(sourceFile.fullName(), "w"); + if (destFile) + { + destFile.setTimeout(5000); // this value was chosen empirically as it failed with default timeout. + size_t written = destFile.write(sourceFile); + if (written == sourceFile.size()) + { + count++; + } + } + destFile.close(); + sourceFile.close(); } - }); + }); + + return (count == sourceFileCount); - return (count == sourceFileCount); - } diff --git a/libraries/FSTools/FSTools.h b/libraries/FSTools/FSTools.h index 8d48e5cd77..4722a1742b 100644 --- a/libraries/FSTools/FSTools.h +++ b/libraries/FSTools/FSTools.h @@ -1,88 +1,91 @@ #pragma once -#include +#include #include #include /* - - A temporary FS is made between the END of the sketch... and the start of the partition you try to mount, to maximise the available space for copying the FS. - The WORST case this is at 0x40300000 which is for a 3m FS on 4m flash.. leaving 460Kb for copying. + A temporary FS is made between the END of the sketch... and the start of the partition you try to mount, to maximise the available space for copying the FS. + The WORST case this is at 0x40300000 which is for a 3m FS on 4m flash.. leaving 460Kb for copying. - I have not worked out a way to prevent linking of ALL the layouts except to use #ifdef guards. Ideas welcome as it uses 400B ROM. - */ +*/ -namespace FST { -struct layout { - layout(uint32_t s, uint32_t e, uint32_t p, uint32_t b) : startAddr(s), endAddr(e), page(p), block(b) {}; - uint32_t startAddr{0}; - uint32_t endAddr{0}; - uint32_t page{0}; - uint32_t block{0}; -}; -enum FS_t : uint8_t { - SPIFFS, - LITTLEFS -}; +namespace FST +{ -extern const layout layout_512k32; -extern const layout layout_512k64; -extern const layout layout_512k128; + struct layout + { + constexpr layout(uint32_t s, uint32_t e, uint32_t p, uint32_t b) : startAddr(s), endAddr(e), page(p), block(b) {}; + const uint32_t startAddr{0}; + const uint32_t endAddr{0}; + const uint32_t page{0}; + const uint32_t block{0}; + }; -extern const layout layout_1m64; -extern const layout layout_1m128; -extern const layout layout_1m144; -extern const layout layout_1m160; -extern const layout layout_1m192; -extern const layout layout_1m256; -extern const layout layout_1m512; + enum FS_t : uint8_t + { + SPIFFS, + LITTLEFS + }; -extern const layout layout_2m64; -extern const layout layout_2m128; -extern const layout layout_2m256; -extern const layout layout_2m512; -extern const layout layout_2m1m; + static constexpr layout layout_512k32 = { 0x40273000, 0x4027B000, 0x100, 0x1000 }; + static constexpr layout layout_512k64 = { 0x4026B000, 0x4027B000, 0x100, 0x1000 }; + static constexpr layout layout_512k128 = { 0x4025B000, 0x4027B000, 0x100, 0x1000 }; -extern const layout layout_4m1m; -extern const layout layout_4m2m; -extern const layout layout_4m3m; + static constexpr layout layout_1m64 = { 0x402EB000, 0x402FB000, 0x100, 0x1000 }; + static constexpr layout layout_1m128 = { 0x402DB000, 0x402FB000, 0x100, 0x1000 }; + static constexpr layout layout_1m144 = { 0x402D7000, 0x402FB000, 0x100, 0x1000 }; + static constexpr layout layout_1m160 = { 0x402D3000, 0x402FB000, 0x100, 0x1000 }; + static constexpr layout layout_1m192 = { 0x402CB000, 0x402FB000, 0x100, 0x1000 }; + static constexpr layout layout_1m256 = { 0x402BB000, 0x402FB000, 0x100, 0x1000 }; + static constexpr layout layout_1m512 = { 0x4027B000, 0x402FB000, 0x100, 0x2000 }; -extern const layout layout_8m6m; -extern const layout layout_8m7m; + static constexpr layout layout_2m64 = { 0x403F0000, 0x403FB000, 0x100, 0x1000 }; + static constexpr layout layout_2m128 = { 0x403E0000, 0x403FB000, 0x100, 0x1000 }; + static constexpr layout layout_2m256 = { 0x403C0000, 0x403FB000, 0x100, 0x1000 }; + static constexpr layout layout_2m512 = { 0x40380000, 0x403FA000, 0x100, 0x2000 }; + static constexpr layout layout_2m1m = { 0x40300000, 0x403FA000, 0x100, 0x2000 }; -extern const layout layout_16m14m; -extern const layout layout_16m15m; + static constexpr layout layout_4m1m = { 0x40500000, 0x405FA000, 0x100, 0x2000 }; + static constexpr layout layout_4m2m = { 0x40400000, 0x405FA000, 0x100, 0x2000 }; + static constexpr layout layout_4m3m = { 0x40300000, 0x405FA000, 0x100, 0x2000 }; -typedef std::function FileCb; + static constexpr layout layout_8m6m = { 0x40400000, 0x409FA000, 0x100, 0x2000 }; + static constexpr layout layout_8m7m = { 0x40300000, 0x409FA000, 0x100, 0x2000 }; + static constexpr layout layout_16m14m = { 0x40400000, 0x411FA000, 0x100, 0x2000 }; + static constexpr layout layout_16m15m = { 0x40300000, 0x411FA000, 0x100, 0x2000 }; -}; + typedef std::function FileCb; +}; -class FSTools { +class FSTools +{ public: - FSTools(); - ~FSTools(); - bool attemptToMountFS(fs::FS & fs); - bool mountAlternativeFS( FST::FS_t type, const FST::layout & layout, bool keepMounted = false ); - bool mounted(); - bool moveFS(fs::FS & destinationFS); - void reset(); - void fileListIterator(FS & fs, const char * dirName, FST::FileCb Cb ); + FSTools(); + ~FSTools(); + bool attemptToMountFS(fs::FS & fs); + bool mountAlternativeFS(FST::FS_t type, const FST::layout & layout, bool keepMounted = false); + bool mounted(); + bool moveFS(fs::FS & destinationFS); + void reset(); + void fileListIterator(FS & fs, const char * dirName, FST::FileCb Cb); private: - uint32_t _getStartAddr(const FST::layout & layout); - uint32_t _getSize(const FST::layout & layout); -// void _dumpFileInfo(File & f); - bool _copyFS(FS & sourceFS, FS & destFS); - - std::unique_ptr _pFS; - bool _mounted{false}; - const FST::layout * _layout{nullptr}; - + uint32_t _getStartAddr(const FST::layout & layout); + uint32_t _getSize(const FST::layout & layout); +#ifdef DEBUG_ESP_CORE + void _dumpFileInfo(File & f); +#endif + bool _copyFS(FS & sourceFS, FS & destFS); + + std::unique_ptr _pFS; + bool _mounted{false}; + const FST::layout * _layout{nullptr}; }; \ No newline at end of file From 9f80841319e854ed96091d94488c64e0a941ebe8 Mon Sep 17 00:00:00 2001 From: sticilface Date: Fri, 20 Nov 2020 20:52:13 +0000 Subject: [PATCH 09/12] Update FSTools.cpp Add yield in between file copy --- libraries/FSTools/FSTools.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/FSTools/FSTools.cpp b/libraries/FSTools/FSTools.cpp index 9ef602c88c..c18e968b76 100644 --- a/libraries/FSTools/FSTools.cpp +++ b/libraries/FSTools/FSTools.cpp @@ -245,6 +245,7 @@ bool FSTools::_copyFS(FS & sourceFS, FS & destFS) } destFile.close(); sourceFile.close(); + yield(); } }); From 319f1d066dcbaa70edf64c5a5c084e642e5b9393 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 15 Jun 2022 23:54:10 +0200 Subject: [PATCH 10/12] style --- .../FSTools/examples/Basic_example/Basic_example.ino | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libraries/FSTools/examples/Basic_example/Basic_example.ino b/libraries/FSTools/examples/Basic_example/Basic_example.ino index 6feca9ea66..acacb362a5 100644 --- a/libraries/FSTools/examples/Basic_example/Basic_example.ino +++ b/libraries/FSTools/examples/Basic_example/Basic_example.ino @@ -40,13 +40,13 @@ const char* password = STAPSK; bool migrateFS() { - if (!fstools.attemptToMountFS(LittleFS)) { // Attempts to mount LittleFS without autoformat... + if (!fstools.attemptToMountFS(LittleFS)) { // Attempts to mount LittleFS without autoformat... Serial.println(F("Default FS not found")); - if (fstools.mountAlternativeFS(FST::SPIFFS /* FST::LITTLEFS */, TARGET_FS_LAYOUT, true)) { + if (fstools.mountAlternativeFS(FST::SPIFFS /* FST::LITTLEFS */, TARGET_FS_LAYOUT, true)) { Serial.println(F("Alternative found")); if (fstools.moveFS(LittleFS)) { Serial.println(F("FileSystem Moved New FS contents:")); - fstools.fileListIterator(LittleFS, "/", [](File & f) { + fstools.fileListIterator(LittleFS, "/", [](File& f) { Serial.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size()); }); return true; @@ -70,7 +70,7 @@ void initWiFiOTA() { String type; if (ArduinoOTA.getCommand() == U_FLASH) { type = "sketch"; - } else { // U_FS + } else { // U_FS type = "filesystem"; } @@ -116,7 +116,7 @@ void setup() { Serial.println("Booting"); - migrateFS(); // MUST call this before calling your own begin(); + migrateFS(); // MUST call this before calling your own begin(); initWiFiOTA(); From 7e9968ca88121c348d8804b57985d6cd4d208549 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 15 Jun 2022 23:55:49 +0200 Subject: [PATCH 11/12] style --- .../custom_FS_example/custom_FS_example.ino | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/libraries/FSTools/examples/custom_FS_example/custom_FS_example.ino b/libraries/FSTools/examples/custom_FS_example/custom_FS_example.ino index 6ca8d8c9b6..3dc2e804fc 100644 --- a/libraries/FSTools/examples/custom_FS_example/custom_FS_example.ino +++ b/libraries/FSTools/examples/custom_FS_example/custom_FS_example.ino @@ -30,7 +30,7 @@ #define TARGET_FS_LAYOUT FST::layout_4m3m const uint32_t startSector = FST::layout_4m1m.startAddr - 0x40200000; -const uint32_t tempFSsize = FST::layout_4m1m.endAddr - FST::layout_4m1m.startAddr; +const uint32_t tempFSsize = FST::layout_4m1m.endAddr - FST::layout_4m1m.startAddr; fs::FS LittleFS_Different = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(startSector, tempFSsize, FS_PHYS_PAGE, FS_PHYS_BLOCK, 5))); @@ -39,7 +39,7 @@ FSTools fstools; #ifndef STASSID #define STASSID "xxxx" -#define STAPSK "xxxx" +#define STAPSK "xxxx" #endif const char* ssid = STASSID; @@ -47,13 +47,13 @@ const char* password = STAPSK; bool migrateFS() { - if (!fstools.attemptToMountFS(LittleFS_Different)) { // Attempts to mount LittleFS without autoformat... + if (!fstools.attemptToMountFS(LittleFS_Different)) { // Attempts to mount LittleFS without autoformat... Serial.println(F("Default FS not found")); - if (fstools.mountAlternativeFS(FST::SPIFFS /* FST::LITTLEFS */, TARGET_FS_LAYOUT, true)) { + if (fstools.mountAlternativeFS(FST::SPIFFS /* FST::LITTLEFS */, TARGET_FS_LAYOUT, true)) { Serial.println(F("Alternative found")); if (fstools.moveFS(LittleFS_Different)) { Serial.println(F("FileSystem Moved New FS contents:")); - fstools.fileListIterator(LittleFS_Different, "/", [](File & f) { + fstools.fileListIterator(LittleFS_Different, "/", [](File& f) { Serial.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size()); }); return true; @@ -77,7 +77,7 @@ void initWiFiOTA() { String type; if (ArduinoOTA.getCommand() == U_FLASH) { type = "sketch"; - } else { // U_FS + } else { // U_FS type = "filesystem"; } @@ -123,7 +123,7 @@ void setup() { Serial.println("Booting"); - migrateFS(); // MUST call this before calling your own begin(); + migrateFS(); // MUST call this before calling your own begin(); initWiFiOTA(); From 3c43f18510e39e919d8f3dbcd935d3ee6698729d Mon Sep 17 00:00:00 2001 From: david gauchard Date: Thu, 16 Jun 2022 00:11:18 +0200 Subject: [PATCH 12/12] style --- libraries/FSTools/examples/Basic_example/Basic_example.ino | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/FSTools/examples/Basic_example/Basic_example.ino b/libraries/FSTools/examples/Basic_example/Basic_example.ino index acacb362a5..efa8b5740d 100644 --- a/libraries/FSTools/examples/Basic_example/Basic_example.ino +++ b/libraries/FSTools/examples/Basic_example/Basic_example.ino @@ -32,7 +32,7 @@ FSTools fstools; #ifndef STASSID #define STASSID "xxxx" -#define STAPSK "xxxx" +#define STAPSK "xxxx" #endif const char* ssid = STASSID;