BREAKING CHANGE: repartition: double sketch size

this updates the partition scheme for devices with 4 MB of flash memory
to have only a single app partition, doubling its size, but sacrificing
their OTA update capability. the replacement environment for
"generic_esop32" is called "generic_esp32_4mb_no_ota".

the new default partition scheme is targeted for devices with at least 8
MB of flash memory. the previous two app partitions are merged into one,
and one additional app partition of the same new size is added to the
back of the table. the change preserves the littlefs partition (position
and size), allowing for upgrades without loosing the configuration.

another new environment is added, called "generic_esp32_8mb", which uses
the new default partition layout.

environment "generic" is removed. it was merely a variant of
"generic_esp32" with some pins pre-defined. we want users to install a
pin_mapping.json and use the generic firmwares.

environments for boards that have no version with at least 8 MB of flash
memory are setup using the 4 MB partition layout (no OTA updates).

all users must flash the factory.bin for their respective environment
using esptool (or compatible software) using the USB port of their
board. in other words: updating to the new partition scheme using an OTA
update is NOT possible.

hint: the ESP32-S3 builds have a smaller code footprint. this means
ESP32-S3 boards can be updated using OTA without updating the partition
layout on the devices for some unspecified time longer, i.e., until
their firmware binary actually becomes too large for the old sketch
partition size.

the non-factory binary for generic_esp32_4mb_no_ota is NOT collected as
an artifact. going forward, users must update using the factory binary
and using the USB connection to their boards.

.github/workflows/build.yml

@@ -1,4 +1,4 @@
-name: OpenDTU-onBattery Build
+name: OpenDTU-OnBattery Build
 
 on:
   push:
@@ -8,7 +8,7 @@ on:
     branches:
       - master
       - development
-    tags-ignore:        
+    tags-ignore:
       - 'v**'
   pull_request:
     paths-ignore:
@@ -120,6 +120,7 @@ jobs:
           name: opendtu-onbattery-${{ matrix.environment }}
           path: |
             .pio/build/${{ matrix.environment }}/opendtu-onbattery-${{ matrix.environment }}.bin
+            !.pio/build/generic_esp32_4mb_no_ota/opendtu-onbattery-generic_esp32_4mb_no_ota.bin
             .pio/build/${{ matrix.environment }}/opendtu-onbattery-${{ matrix.environment }}.factory.bin
 
   release:
@@ -130,14 +131,14 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      
+
       - name: Get tags
         run: git fetch --force --tags origin
 
       - name: Get openDTU core release
         run: |
           echo "OPEN_DTU_CORE_RELEASE=$(git for-each-ref --sort=creatordate --format '%(refname) %(creatordate)' refs/tags | grep 'refs/tags/v' | tail -1 | sed 's#.*/##' | sed 's/ .*//')" >> $GITHUB_ENV
-      
+
       - name: Create openDTU-core-release-Badge
         uses: schneegans/dynamic-badges-action@v1.6.0
         with:

partitions_custom_4mb.csv

@@ -1,6 +1,5 @@
 # Name,   Type, SubType, Offset,   Size,    Flags
 nvs,      data, nvs,     0x9000,   0x5000,
 otadata,  data, ota,     0xe000,   0x2000,
-app0,     app,  ota_0,   0x10000,  0x1E0000,
-app1,     app,  ota_1,   0x1F0000, 0x1E0000,
+app0,     app,  ota_0,   0x10000,  0x3C0000,
 spiffs,   data, spiffs,  0x3D0000, 0x30000,

partitions_custom_8mb.csv

@@ -0,0 +1,6 @@
+# Name,   Type, SubType, Offset,   Size,    Flags
+nvs,      data, nvs,     0x9000,   0x5000,
+otadata,  data, ota,     0xe000,   0x2000,
+app0,     app,  ota_0,   0x10000,  0x3C0000,
+spiffs,   data, spiffs,  0x3D0000, 0x30000,
+app1,     app,  ota_1,   0x400000, 0x3C0000,

platformio.ini

@@ -9,17 +9,14 @@
 ; https://docs.platformio.org/page/projectconf.html
 
 [platformio]
-default_envs = generic_esp32
+default_envs = generic_esp32s3_usb
 extra_configs =
     platformio_override.ini
 
 [env]
 ; Make sure to NOT add any spaces in the custom_ci_action property
 ; (also the position in the file is important)
-; disabling generic and generic_esp32 until we decided on how to handle
-; the overflowing sketch partition for all ESP32 with 4MB flash.
-; custom_ci_action = generic,generic_esp32,generic_esp32s3,generic_esp32s3_usb
-custom_ci_action = generic_esp32s3,generic_esp32s3_usb
+custom_ci_action = generic_esp32_4mb_no_ota,generic_esp32_8mb,generic_esp32s3,generic_esp32s3_usb
 
 framework = arduino
 platform = espressif32@6.7.0
@@ -57,7 +54,7 @@ extra_scripts =
     pre:pio-scripts/patch_apply.py
     post:pio-scripts/create_factory_bin.py
 
-board_build.partitions = partitions_custom_4mb.csv
+board_build.partitions = partitions_custom_8mb.csv
 board_build.filesystem = littlefs
 board_build.embed_files =
     webapp_dist/index.html.gz
@@ -78,7 +75,13 @@ upload_protocol = esptool
 ; upload_port = COM4
 
 
-[env:generic_esp32]
+[env:generic_esp32_4mb_no_ota]
+board = esp32dev
+build_flags = ${env.build_flags}
+board_build.partitions = partitions_custom_4mb.csv
+
+
+[env:generic_esp32_8mb]
 board = esp32dev
 build_flags = ${env.build_flags}
 
@@ -95,12 +98,14 @@ build_flags = ${env.build_flags}
 
 [env:generic_esp32c3]
 board = esp32-c3-devkitc-02
+board_build.partitions = partitions_custom_4mb.csv
 custom_patches = ${env.custom_patches}
 build_flags = ${env.build_flags}
 
 
 [env:generic_esp32c3_usb]
 board = esp32-c3-devkitc-02
+board_build.partitions = partitions_custom_4mb.csv
 custom_patches = ${env.custom_patches}
 build_flags = ${env.build_flags}
     -DARDUINO_USB_MODE=1
@@ -120,17 +125,6 @@ build_flags = ${env.build_flags}
     -DARDUINO_USB_CDC_ON_BOOT=1
 
 
-[env:generic]
-board = esp32dev
-build_flags = ${env.build_flags}
-    -DHOYMILES_PIN_MISO=19
-    -DHOYMILES_PIN_MOSI=23
-    -DHOYMILES_PIN_SCLK=18
-    -DHOYMILES_PIN_IRQ=16
-    -DHOYMILES_PIN_CE=4
-    -DHOYMILES_PIN_CS=5
-
-
 [env:olimex_esp32_poe]
 ; https://www.olimex.com/Products/IoT/ESP32/ESP32-POE/open-source-hardware
 board = esp32-poe
@@ -147,6 +141,7 @@ build_flags = ${env.build_flags}
 [env:olimex_esp32_evb]
 ; https://www.olimex.com/Products/IoT/ESP32/ESP32-EVB/open-source-hardware
 board = esp32-evb
+board_build.partitions = partitions_custom_4mb.csv
 build_flags = ${env.build_flags}
     -DHOYMILES_PIN_MISO=15
     -DHOYMILES_PIN_MOSI=2
@@ -159,16 +154,17 @@ build_flags = ${env.build_flags}
 
 [env:d1_mini_esp32]
 board = wemos_d1_mini32
-build_flags = 
-	${env.build_flags}
-	-DHOYMILES_PIN_MISO=19
-	-DHOYMILES_PIN_MOSI=23
-	-DHOYMILES_PIN_SCLK=18
-	-DHOYMILES_PIN_IRQ=16
-	-DHOYMILES_PIN_CE=17
-	-DHOYMILES_PIN_CS=5
-	-DVICTRON_PIN_TX=21
-	-DVICTRON_PIN_RX=22
+board_build.partitions = partitions_custom_4mb.csv
+build_flags =
+    ${env.build_flags}
+    -DHOYMILES_PIN_MISO=19
+    -DHOYMILES_PIN_MOSI=23
+    -DHOYMILES_PIN_SCLK=18
+    -DHOYMILES_PIN_IRQ=16
+    -DHOYMILES_PIN_CE=17
+    -DHOYMILES_PIN_CS=5
+    -DVICTRON_PIN_TX=21
+    -DVICTRON_PIN_RX=22
     -DPYLONTECH_PIN_RX=27
     -DPYLONTECH_PIN_TX=14
     -DHUAWEI_PIN_MISO=12
@@ -181,6 +177,7 @@ build_flags =
 [env:wt32_eth01]
 ; http://www.wireless-tag.com/portfolio/wt32-eth01/
 board = wt32-eth01
+board_build.partitions = partitions_custom_4mb.csv
 build_flags = ${env.build_flags}
     -DHOYMILES_PIN_MISO=4
     -DHOYMILES_PIN_MOSI=2
@@ -207,6 +204,7 @@ build_flags = ${env.build_flags}
 ; https://www.makershop.de/plattformen/esp8266/wemos-lolin32/
 ; https://www.az-delivery.de/products/esp32-lolin-lolin32
 board = lolin32_lite
+board_build.partitions = partitions_custom_4mb.csv
 build_flags = ${env.build_flags}
     -DHOYMILES_PIN_MISO=19
     -DHOYMILES_PIN_MOSI=23
@@ -217,6 +215,7 @@ build_flags = ${env.build_flags}
 
 [env:lolin_s2_mini]
 board = lolin_s2_mini
+board_build.partitions = partitions_custom_4mb.csv
 build_flags = ${env.build_flags}
     -DHOYMILES_PIN_MISO=13
     -DHOYMILES_PIN_MOSI=11