From 7074bede97178990342366970ecdd51cb3efe4a2 Mon Sep 17 00:00:00 2001 From: Peter Dragun Date: Wed, 1 Nov 2023 17:55:29 +0100 Subject: [PATCH] feat: add support for intel hex format --- docs/en/esptool/advanced-commands.rst | 2 +- docs/en/esptool/basic-commands.rst | 25 +++++++-- esptool/__init__.py | 27 ++++++++-- esptool/bin_image.py | 21 ++++++++ esptool/cmds.py | 15 ++++++ setup.py | 1 + test/test_esptool.py | 73 ++++++++++++++++++++++++--- test/test_image_info.py | 45 ++++++++++++++++- test/test_merge_bin.py | 66 ++++++++++++++++++++++++ 9 files changed, 258 insertions(+), 17 deletions(-) diff --git a/docs/en/esptool/advanced-commands.rst b/docs/en/esptool/advanced-commands.rst index 5143963a5..bfc9d2c0c 100644 --- a/docs/en/esptool/advanced-commands.rst +++ b/docs/en/esptool/advanced-commands.rst @@ -48,7 +48,7 @@ The ``dump_mem`` command will dump a region from the chip's memory space to a fi Load a Binary to RAM: load_ram ------------------------------ -The ``load_ram`` command allows the loading of an executable binary image (created with the ``elf2image`` or ``make_image`` commands) directly into RAM, and then immediately executes the program contained within it. +The ``load_ram`` command allows the loading of an executable binary image (created with the ``elf2image`` or ``make_image`` commands) directly into RAM, and then immediately executes the program contained within it. Command also supports ``.hex`` file created by ``merge_bin`` command from supported ``.bin`` files. :: diff --git a/docs/en/esptool/basic-commands.rst b/docs/en/esptool/basic-commands.rst index 880cb5fe3..ead5ac208 100644 --- a/docs/en/esptool/basic-commands.rst +++ b/docs/en/esptool/basic-commands.rst @@ -220,7 +220,7 @@ By default, ``elf2image`` uses the sections in the ELF file to generate each seg Output .bin Image Details: image_info ------------------------------------- -The ``image_info`` command outputs some information (load addresses, sizes, etc) about a ``.bin`` file created by ``elf2image``. +The ``image_info`` command outputs some information (load addresses, sizes, etc) about a ``.bin`` file created by ``elf2image``. Command also supports ``.hex`` file created by ``merge_bin`` command from supported ``.bin`` files. To view more information about the image, such as set flash size, frequency and mode, or extended header information, use the ``--version 2`` option. This extended output will become the default in a future major release. @@ -240,7 +240,7 @@ This information corresponds to the headers described in :ref:`image-format`. Merge Binaries for Flashing: merge_bin -------------------------------------- -The ``merge_bin`` command will merge multiple binary files (of any kind) into a single file that can be flashed to a device later. Any gaps between the input files are padded based on selected output format. +The ``merge_bin`` command will merge multiple binary files (of any kind) into a single file that can be flashed to a device later. Any gaps between the input files are padded based on the selected output format. For example: @@ -248,7 +248,7 @@ For example: esptool.py --chip {IDF_TARGET_NAME} merge_bin -o merged-flash.bin --flash_mode dio --flash_size 4MB 0x1000 bootloader.bin 0x8000 partition-table.bin 0x10000 app.bin -Will create a file ``merged-flash.bin`` with the contents of the other 3 files. This file can be later be written to flash with ``esptool.py write_flash 0x0 merged-flash.bin``. +Will create a file ``merged-flash.bin`` with the contents of the other 3 files. This file can be later written to flash with ``esptool.py write_flash 0x0 merged-flash.bin``. **Common options:** @@ -256,6 +256,7 @@ Will create a file ``merged-flash.bin`` with the contents of the other 3 files. * The ``merge_bin`` command supports the same ``--flash_mode``, ``--flash_size`` and ``--flash_freq`` options as the ``write_flash`` command to override the bootloader flash header (see above for details). These options are applied to the output file contents in the same way as when writing to flash. Make sure to pass the ``--chip`` parameter if using these options, as the supported values and the bootloader offset both depend on the chip. * The ``--format`` option will change the format of the output file. For more information about formats see formats description below. +* The input files can be in either ``bin`` or ``hex`` format and they will be automatically converted to type selected by ``--format`` argument. * It is possible to append options from a text file with ``@filename`` (see the advanced options page :ref:`Specifying Arguments via File ` section for details). As an example, this can be conveniently used with the ESP-IDF build system, which produces a ``flash_args`` file in the build directory of a project: .. code:: sh @@ -264,6 +265,22 @@ Will create a file ``merged-flash.bin`` with the contents of the other 3 files. esptool.py --chip {IDF_TARGET_NAME} merge_bin -o merged-flash.bin @flash_args +HEX Output Format +^^^^^^^^^^^^^^^^^ + +The output of the command will be in `Intel Hex format `__. The gaps between the files won't be padded. + +Intel Hex format offers distinct advantages when compared to the binary format, primarily in the following areas: + +* **Transport**: Intel Hex files are represented in ASCII text format, significantly increasing the likelihood of flawless transfers across various mediums. +* **Size**: Data is carefully allocated to specific memory addresses eliminating the need for unnecessary padding. Binary images often lack detailed addressing information, leading to the inclusion of data for all memory locations from the file's initial address to its end. +* **Validity Checks**: Each line in an Intel Hex file has a checksum to help find errors and make sure data stays unchanged. + +.. code:: sh + + esptool.py --chip {IDF_TARGET_NAME} merge_bin --format hex -o merged-flash.hex --flash_mode dio --flash_size 4MB 0x1000 bootloader.bin 0x8000 partition-table.bin 0x10000 app.bin + + RAW Output Format ^^^^^^^^^^^^^^^^^ @@ -290,7 +307,7 @@ Gaps between the files will be filled with `0x00` bytes. **UF2 options:** -* The ``--chunk-size`` option will set what portion of 512 byte block will be used for data. Common value is 256 bytes. By default the largest possible value will be used. +* The ``--chunk-size`` option will set what portion of 512 byte block will be used for data. A common value is 256 bytes. By default, the largest possible value will be used. * The ``--md5-disable`` option will disable MD5 checksums at the end of each block. This can be useful for integration with e.g. `tinyuf2 `__. .. code:: sh diff --git a/esptool/__init__.py b/esptool/__init__.py index 973d1fd5b..206b6fadd 100644 --- a/esptool/__init__.py +++ b/esptool/__init__.py @@ -38,6 +38,7 @@ import time import traceback +from esptool.bin_image import intel_hex_to_bin from esptool.cmds import ( DETECTED_FLASH_SIZES, chip_id, @@ -185,7 +186,9 @@ def add_spi_connection_arg(parent): parser_load_ram = subparsers.add_parser( "load_ram", help="Download an image to RAM and execute" ) - parser_load_ram.add_argument("filename", help="Firmware image") + parser_load_ram.add_argument( + "filename", help="Firmware image", action=AutoHex2BinAction + ) parser_dump_mem = subparsers.add_parser( "dump_mem", help="Dump arbitrary memory to disk" @@ -357,7 +360,9 @@ def add_spi_flash_subparsers(parent, allow_keep, auto_detect): parser_image_info = subparsers.add_parser( "image_info", help="Dump headers from a binary file (bootloader or application)" ) - parser_image_info.add_argument("filename", help="Image file to parse") + parser_image_info.add_argument( + "filename", help="Image file to parse", action=AutoHex2BinAction + ) parser_image_info.add_argument( "--version", "-v", @@ -601,7 +606,7 @@ def add_spi_flash_subparsers(parent, allow_keep, auto_detect): "--format", "-f", help="Format of the output file", - choices=["raw", "uf2"], + choices=["raw", "uf2", "hex"], default="raw", ) uf2_group = parser_merge_bin.add_argument_group("UF2 format") @@ -1057,6 +1062,20 @@ def __call__(self, parser, namespace, value, option_string=None): setattr(namespace, self.dest, value) +class AutoHex2BinAction(argparse.Action): + """Custom parser class for auto conversion of input files from hex to bin""" + + def __call__(self, parser, namespace, value, option_string=None): + try: + with open(value, "rb") as f: + # if hex file was detected replace hex file with converted temp bin + # otherwise keep the original file + value = intel_hex_to_bin(f).name + except IOError as e: + raise argparse.ArgumentError(self, e) + setattr(namespace, self.dest, value) + + class AddrFilenamePairAction(argparse.Action): """Custom parser class for the address/filename pairs passed as arguments""" @@ -1085,6 +1104,8 @@ def __call__(self, parser, namespace, values, option_string=None): "Must be pairs of an address " "and the binary filename to write there", ) + # check for intel hex files and convert them to bin + argfile = intel_hex_to_bin(argfile, address) pairs.append((address, argfile)) # Sort the addresses and check for overlapping diff --git a/esptool/bin_image.py b/esptool/bin_image.py index 0f3ef2f64..b011b55b5 100644 --- a/esptool/bin_image.py +++ b/esptool/bin_image.py @@ -10,6 +10,10 @@ import os import re import struct +import tempfile +from typing import BinaryIO, Optional + +from intelhex import IntelHex from .loader import ESPLoader from .targets import ( @@ -36,6 +40,23 @@ def align_file_position(f, size): f.seek(align, 1) +def intel_hex_to_bin(file: BinaryIO, start_addr: Optional[int] = None) -> BinaryIO: + """Convert IntelHex file to temp binary file with padding from start_addr + If hex file was detected return temp bin file object; input file otherwise""" + INTEL_HEX_MAGIC = b":" + magic = file.read(1) + file.seek(0) + if magic == INTEL_HEX_MAGIC: + ih = IntelHex() + ih.loadhex(file.name) + file.close() + bin = tempfile.NamedTemporaryFile(suffix=".bin", delete=False) + ih.tobinfile(bin, start=start_addr) + return bin + else: + return file + + def LoadFirmwareImage(chip, image_file): """ Load a firmware image. Can be for any supported SoC. diff --git a/esptool/cmds.py b/esptool/cmds.py index 408c17f67..3a6f914e0 100644 --- a/esptool/cmds.py +++ b/esptool/cmds.py @@ -11,6 +11,8 @@ import time import zlib +from intelhex import IntelHex + from .bin_image import ELFFile, ImageSegment, LoadFirmwareImage from .bin_image import ( ESP8266ROMFirmwareImage, @@ -1337,6 +1339,19 @@ def pad_to(flash_offs): f"Wrote {of.tell():#x} bytes to file {args.output}, " f"ready to flash to offset {args.target_offset:#x}" ) + elif args.format == "hex": + out = IntelHex() + for addr, argfile in input_files: + ihex = IntelHex() + image = argfile.read() + image = _update_image_flash_params(chip_class, addr, args, image) + ihex.frombytes(image, addr) + out.merge(ihex) + out.write_hex_file(args.output) + print( + f"Wrote {os.path.getsize(args.output):#x} bytes to file {args.output}, " + f"ready to flash to offset {args.target_offset:#x}" + ) def version(args): diff --git a/setup.py b/setup.py index f6e084e7e..0f1b2826b 100644 --- a/setup.py +++ b/setup.py @@ -130,6 +130,7 @@ def find_version(*file_paths): "pyserial>=3.0", "reedsolo>=1.5.3,<1.8", "PyYAML>=5.1", + "intelhex", ], packages=find_packages(), include_package_data=True, diff --git a/test/test_esptool.py b/test/test_esptool.py index c399f9c06..c625ea983 100755 --- a/test/test_esptool.py +++ b/test/test_esptool.py @@ -24,6 +24,7 @@ import time from socket import AF_INET, SOCK_STREAM, socket from time import sleep +from typing import List from unittest.mock import MagicMock # Link command line options --port, --chip, --baud, --with-trace, and --preload-port @@ -389,6 +390,42 @@ def test_adjacent_flash(self): self.verify_readback(0, 4096, "images/sector.bin") self.verify_readback(4096, 50 * 1024, "images/fifty_kb.bin") + def test_short_flash_hex(self): + _, f = tempfile.mkstemp(suffix=".hex") + try: + self.run_esptool(f"merge_bin --format hex 0x0 images/one_kb.bin -o {f}") + self.run_esptool(f"write_flash 0x0 {f}") + self.verify_readback(0, 1024, "images/one_kb.bin") + finally: + os.unlink(f) + + def test_adjacent_flash_hex(self): + _, f1 = tempfile.mkstemp(suffix=".hex") + _, f2 = tempfile.mkstemp(suffix=".hex") + try: + self.run_esptool(f"merge_bin --format hex 0x0 images/sector.bin -o {f1}") + self.run_esptool( + f"merge_bin --format hex 0x1000 images/fifty_kb.bin -o {f2}" + ) + self.run_esptool(f"write_flash 0x0 {f1} 0x1000 {f2}") + self.verify_readback(0, 4096, "images/sector.bin") + self.verify_readback(4096, 50 * 1024, "images/fifty_kb.bin") + finally: + os.unlink(f1) + os.unlink(f2) + + def test_adjacent_flash_mixed(self): + _, f = tempfile.mkstemp(suffix=".hex") + try: + self.run_esptool( + f"merge_bin --format hex 0x1000 images/fifty_kb.bin -o {f}" + ) + self.run_esptool(f"write_flash 0x0 images/sector.bin 0x1000 {f}") + self.verify_readback(0, 4096, "images/sector.bin") + self.verify_readback(4096, 50 * 1024, "images/fifty_kb.bin") + finally: + os.unlink(f) + def test_adjacent_independent_flash(self): self.run_esptool("write_flash 0x0 images/sector.bin") self.verify_readback(0, 4096, "images/sector.bin") @@ -949,6 +986,15 @@ def test_explicit_set_size_freq_mode(self): class TestLoadRAM(EsptoolTestCase): # flashing an application not supporting USB-CDC will make # /dev/ttyACM0 disappear and USB-CDC tests will not work anymore + + def verify_output(self, expected_out: List[bytes]): + """Verify that at least one element of expected_out is in serial output""" + with serial.serial_for_url(arg_port, arg_baud) as p: + p.timeout = 5 + output = p.read(100) + print(f"Output: {output}") + assert any(item in output for item in expected_out) + @pytest.mark.quick_test def test_load_ram(self): """Verify load_ram command @@ -957,17 +1003,28 @@ def test_load_ram(self): "Hello world!\n" to the serial port. """ self.run_esptool(f"load_ram images/ram_helloworld/helloworld-{arg_chip}.bin") + self.verify_output( + [b"Hello world!", b'\xce?\x13\x05\x04\xd0\x97A\x11"\xc4\x06\xc67\x04'] + ) + + def test_load_ram_hex(self): + """Verify load_ram command with hex file as input + + The "hello world" binary programs for each chip print + "Hello world!\n" to the serial port. + """ + _, f = tempfile.mkstemp(suffix=".hex") try: - p = serial.serial_for_url(arg_port, arg_baud) - p.timeout = 5 - output = p.read(100) - print(f"Output: {output}") - assert ( - b"Hello world!" in output # xtensa - or b'\xce?\x13\x05\x04\xd0\x97A\x11"\xc4\x06\xc67\x04' in output # C3 + self.run_esptool( + f"merge_bin --format hex -o {f} 0x0 " + f"images/ram_helloworld/helloworld-{arg_chip}.bin" + ) + self.run_esptool(f"load_ram {f}") + self.verify_output( + [b"Hello world!", b'\xce?\x13\x05\x04\xd0\x97A\x11"\xc4\x06\xc67\x04'] ) finally: - p.close() + os.unlink(f) class TestDeepSleepFlash(EsptoolTestCase): diff --git a/test/test_image_info.py b/test/test_image_info.py index d9edef602..3dac7310a 100755 --- a/test/test_image_info.py +++ b/test/test_image_info.py @@ -2,6 +2,7 @@ import os.path import subprocess import sys +import tempfile from conftest import need_to_install_package_err @@ -40,7 +41,9 @@ def run_image_info(self, chip, file, version=None): ] if version is not None: cmd += ["--version", str(version)] - cmd += ["".join([IMAGES_DIR, os.sep, file])] + # if path was passed use the whole path + # if file does not exists try to use file from IMAGES_DIR directory + cmd += [file] if os.path.isfile(file) else ["".join([IMAGES_DIR, os.sep, file])] print("\nExecuting {}".format(" ".join(cmd))) try: @@ -189,3 +192,43 @@ def test_bootloader_info(self): assert "Bootloader version: 1" in out assert "ESP-IDF: v5.2-dev-254-g1950b15" in out assert "Compile time: Apr 25 2023 00:13:32" in out + + def test_intel_hex(self): + # This bootloader binary is built from "hello_world" project + # with default settings, IDF version is v5.2. + # File is converted to Intel Hex using merge_bin + + def convert_bin2hex(file): + subprocess.check_output( + [ + sys.executable, + "-m", + "esptool", + "--chip", + "esp32", + "merge_bin", + "--format", + "hex", + "0x0", + "".join([IMAGES_DIR, os.sep, "bootloader_esp32_v5_2.bin"]), + "-o", + file, + ] + ) + + fd, file = tempfile.mkstemp(suffix=".hex") + try: + convert_bin2hex(file) + out = self.run_image_info("esp32", file, "2") + assert "File size: 26768 (bytes)" in out + assert "Bootloader information" in out + assert "Bootloader version: 1" in out + assert "ESP-IDF: v5.2-dev-254-g1950b15" in out + assert "Compile time: Apr 25 2023 00:13:32" in out + finally: + try: + # make sure that file was closed before removing it + os.close(fd) + except OSError: + pass + os.unlink(file) diff --git a/test/test_merge_bin.py b/test/test_merge_bin.py index 5ca7e0890..83abbff92 100755 --- a/test/test_merge_bin.py +++ b/test/test_merge_bin.py @@ -197,6 +197,72 @@ def test_fill_flash_size_w_target_offset(self): assert helloworld == merged[0xF000 : 0xF000 + len(helloworld)] self.assertAllFF(merged[0xF000 + len(helloworld) :]) + def test_merge_mixed(self): + # convert bootloader to hex + hex = self.run_merge_bin( + "esp32", + [(0x1000, "bootloader_esp32.bin")], + options=["--format", "hex"], + ) + # create a temp file with hex content + with tempfile.NamedTemporaryFile(suffix=".hex", delete=False) as f: + f.write(hex) + # merge hex file with bin file + # output to bin file should be the same as in merge bin + bin + try: + merged = self.run_merge_bin( + "esp32", + [(0x1000, f.name), (0x10000, "ram_helloworld/helloworld-esp32.bin")], + ["--target-offset", "0x1000", "--fill-flash-size", "2MB"], + ) + finally: + os.unlink(f.name) + # full length is without target-offset arg + assert len(merged) == 0x200000 - 0x1000 + + bootloader = read_image("bootloader_esp32.bin") + helloworld = read_image("ram_helloworld/helloworld-esp32.bin") + assert bootloader == merged[: len(bootloader)] + assert helloworld == merged[0xF000 : 0xF000 + len(helloworld)] + self.assertAllFF(merged[0xF000 + len(helloworld) :]) + + def test_merge_bin2hex(self): + merged = self.run_merge_bin( + "esp32", + [ + (0x1000, "bootloader_esp32.bin"), + ], + options=["--format", "hex"], + ) + lines = merged.splitlines() + # hex format - :0300300002337A1E + # :03 0030 00 02337A 1E + # ^data_cnt/2 ^addr ^type ^data ^checksum + + # check for starting address - 0x1000 passed from arg + assert lines[0][3:7] == b"1000" + # pick a random line for testing the format + line = lines[random.randrange(0, len(lines))] + assert line[0] == ord(":") + data_len = int(b"0x" + line[1:3], 16) + # : + len + addr + type + data + checksum + assert len(line) == 1 + 2 + 4 + 2 + data_len * 2 + 2 + # last line is allways :00000001FF + assert lines[-1] == b":00000001FF" + # convert back and verify the result against the source bin file + with tempfile.NamedTemporaryFile(suffix=".hex", delete=False) as hex: + hex.write(merged) + merged_bin = self.run_merge_bin( + "esp32", + [(0x1000, hex.name)], + options=["--format", "raw"], + ) + source = read_image("bootloader_esp32.bin") + # verify that padding was done correctly + assert b"\xFF" * 0x1000 == merged_bin[:0x1000] + # verify the file itself + assert source == merged_bin[0x1000:] + class UF2Block(object): def __init__(self, bs):