From 84dc987be0696a3f5fc5cbbe43c2d216f20a6a2d Mon Sep 17 00:00:00 2001 From: M Hightower <27247790+mhightower83@users.noreply.github.com> Date: Wed, 23 Mar 2022 10:00:41 -0700 Subject: [PATCH] Added command-line parser Support hints for compiler.cache_core. For use when Arduino IDE uses command-line options that override compiler.cache_core. Removed overuse of () Improve FAQ entry --- doc/faq/a06-global-build-options.rst | 146 ++++++++++++++++++++------- platform.txt | 3 +- tools/mkbuildoptglobals.py | 141 +++++++++++++++++++++----- 3 files changed, 228 insertions(+), 62 deletions(-) mode change 100644 => 100755 tools/mkbuildoptglobals.py diff --git a/doc/faq/a06-global-build-options.rst b/doc/faq/a06-global-build-options.rst index 9ffe16f25b..0e65e526f7 100644 --- a/doc/faq/a06-global-build-options.rst +++ b/doc/faq/a06-global-build-options.rst @@ -89,33 +89,79 @@ Global ``.h`` file: ``LowWatermark.ino.globals.h`` #endif -Aggressive Caching of ``core.a`` +Aggressively Cache Compiled core ================================ -The feature “Aggressive Caching of core.a” refers to sharing a single -copy of ``core.a`` across all Arduino IDE Sketch windows. This feature -is on by default. ``core.a`` is an archive file containing the compiled -objects of ``./core/esp8266/*``. Created after your 1ST successful -compilation. All other open sketch builds use this shared file. When you -close all Arduino IDE windows, the core archive file is deleted. - -Without mediation, using global defines or compiler command-line options -could lead to bad builds when the “Aggressively cache compiled core” -feature is enabled. When ``#define`` changes require rebuilding -``core.a`` and multiple Sketches are open, they can no longer reliably -share one cached ``core.a``. In a simple case: The 1st Sketch to be -built has its version of ``core.a`` cached. Other sketches will use this -cached version for their builds. - -When the “Aggressively cache compiled core” feature is enabled and a +This feature appeared with the release of Arduino IDE 1.8.2. The feature +“Aggressively Cache Compiled core” refers to sharing a single copy of +``core.a`` across all Arduino IDE Sketch windows. This feature is on by +default. ``core.a`` is an archive file containing the compiled objects +of ``./core/esp8266/*``. Created after your 1ST successful compilation. +All other open sketch builds use this shared file. When you close all +Arduino IDE windows, the core archive file is deleted. + +This feature is not compatible with using global defines or compiler +command-line options. Without mediation, bad builds could result, when +left enabled. When ``#define`` changes require rebuilding ``core.a`` and +multiple Sketches are open, they can no longer reliably share one cached +``core.a``. In a simple case: The 1st Sketch to be built has its version +of ``core.a`` cached. Other sketches will use this cached version for +their builds. + +There are two solutions to this issue: 1. Turn off the “Aggressively +Cache Compiled core” feature, by setting ``compiler.cache_core=false``. +2. Rely on the not ideal fail-safe, aggressive cache workaround built +into the script. + +Using “compiler.cache_core=false” +--------------------------------- + +There are two ways to turn off the “Aggressively Cache Compiled core” +feature: This can be done with the Arduino IDE command-line or a text +editor. + +Using the Arduino IDE command-line from a system command line, enter the +following: + +:: + + arduino --pref compiler.cache_core=false --save-prefs + +For the text editor, you need to find the location of +``preferences.txt``. From the Arduino IDE, go to *File->Preferences*. +Make note of the path to ``prefereces.txt``. You *cannot* edit the file +while the Arduino IDE is running. Close all Arduino IDE windows and edit +the file ``preferences.txt``. Change ``compiler.cache_core=true`` to +``compiler.cache_core=false`` and save. Then each sketch will maintain +its *own* copy of ``core.a`` built with the customization expressed by +their respective ``build.opt`` file. + +The “workaround” +---------------- + +When the “Aggressively Cache Compiled core” feature is enabled and the global define file is detected, a workaround will turn on and stay on. When you switch between Sketch windows, core will be recompiled and the cache updated. The workaround logic is reset when Arduino IDE is -completely shutdown and restarted. Some operating systems are better at -cleaning up their temp space than others at reboot after a crash. At -least for Windows®, you may need to manually delete the Arduino temp -files and directories after a crash. Otherwise, the workaround logic may -be left on. +completely shutdown and restarted. + +The workaround is not perfect. These issues may be of concern: 1. Dirty +temp space. Arduino build cache files left over from a previous run or +boot. 2. Arduino command-line options: \* override default +preferences.txt file. \* override a preference, specifically +``compiler.cache_core``. 3. Multiple versions of the Arduino IDE running + +**Dirty temp space** + +A minor concern, the workaround is always on. Not an issue for build +accuracy, but ``core.a`` maybe rebuild more often than necessary. + +Some operating systems are better at cleaning up their temp space than +others at reboot after a crash. At least for Windows®, you may need to +manually delete the Arduino temp files and directories after a crash. +Otherwise, the workaround logic may be left on. There is no harm in the +workaround being stuck on, the build will be correct; however, the core +files will occasionally be recompiled when not needed. For some Windows® systems the temp directory can be found near ``C:\Users\\AppData\Local\Temp\arduino*``. Note ``AppData`` is @@ -123,26 +169,52 @@ a hidden directory. For help with this do an Internet search on ``windows disk cleanup``. Or, type ``disk cleanup`` in the Windows® taskbar search box. +With Linux, this problem could occur after an Arduino IDE crash. The +problem would be cleared after a reboot. Or you can manually cleanup the +``/tmp/`` directory before restarting the Arduino IDE. + +**Arduino command-line option overrides** + +The script needs to know the working value of ``compiler.cache_core`` +that the Arduino IDE uses when building. This script can learn the state +through documented locations; however, the Arduino IDE has two +command-line options that can alter the results the Arduino IDE uses +internally. And, the Arduino IDE does not provide a means for a script +to learn the override value. + +These two command-line options are the problem: + +:: + + ./arduino --preferences-file other-preferences.txt + ./arduino --pref compiler.cache_core=false + +Hints for discovering the value of ``compiler.cache_core`` use can be +provided by specifying ``mkbuildoptglobals.extra_flags=...`` in +``platform.local.txt``. + +Examples of hints: + +:: + + mkbuildoptglobals.extra_flags=--preferences_sketch # assume file preferences.txt in the sketch folder + mkbuildoptglobals.extra_flags=--preferences_sketch pref.txt # is relative to the sketch folder + mkbuildoptglobals.extra_flags=--no_cache_core + mkbuildoptglobals.extra_flags=--cache_core + mkbuildoptglobals.extra_flags=--preferences_file other-preferences.txt # relative to IDE or full path + +**Multiple versions of the Arduino IDE running** + You can run multiple Arduino IDE windows as long as you run one version of the Arduino IDE at a time. When testing different versions, completely exit one before starting the next version. For example, Arduino IDE 1.8.19 and Arduino IDE 2.0 work with different temp and build paths. With this combination, the workaround logic sometimes fails -to enable. At the time of this writing, when Arduino IDE 2.0 rc5 exits, -it leaves the temp space dirty. This keeps the workaround active the -next time the IDE is started. If this is an issue, manually delete the -temp files. - -If you think your workflow performance would benefit from keeping a per -Sketch copy of ``core.a``, you can turn off the “Aggressively cache -compiled core” feature. You need to find the location of -``preferences.txt``. From the Arduino IDE, go to *File->Preferences*. -Make note of the path to ``prefereces.txt``. You cannot edit the file -while the Arduino IDE is running. Close all Arduino IDE windows and edit -the file ``preferences.txt``. Change ``compiler.cache_core=true`` to -``compiler.cache_core=false`` and save. Then each sketch will maintain -its *own* copy of ``core.a`` built with the customization expressed by -their respective ``build.opt`` file. +to enable. + +At the time of this writing, when Arduino IDE 2.0 rc5 exits, it leaves +the temp space dirty. This keeps the workaround active the next time the +IDE is started. If this is an issue, manually delete the temp files. Other build confusion ===================== diff --git a/platform.txt b/platform.txt index 797db4654e..5295aa62f3 100644 --- a/platform.txt +++ b/platform.txt @@ -63,6 +63,7 @@ build.spiffs_blocksize= globals.h.source.fqfn={build.source.path}/{build.project_name}.globals.h commonhfile.fqfn={build.core.path}/CommonHFile.h build.opt.fqfn={build.path}/core/build.opt +mkbuildoptglobals.extra_flags= compiler.path={runtime.tools.xtensa-lx106-elf-gcc.path}/bin/ compiler.sdk.path={runtime.platform.path}/tools/sdk @@ -116,7 +117,7 @@ recipe.hooks.sketch.prebuild.pattern="{runtime.tools.python3.path}/python3" -I " recipe.hooks.prebuild.1.pattern="{runtime.tools.python3.path}/python3" -I "{runtime.tools.makecorever}" --build_path "{build.path}" --platform_path "{runtime.platform.path}" --version "{version}" # Handle processing sketch global options -recipe.hooks.prebuild.2.pattern="{runtime.tools.python3.path}/python3" "{runtime.tools.mkbuildoptglobals}" "{runtime.ide.path}" "{build.path}" "{build.opt.fqfn}" "{globals.h.source.fqfn}" "{commonhfile.fqfn}" +recipe.hooks.prebuild.2.pattern="{runtime.tools.python3.path}/python3" "{runtime.tools.mkbuildoptglobals}" "{runtime.ide.path}" {runtime.ide.version} "{build.path}" "{build.opt.fqfn}" "{globals.h.source.fqfn}" "{commonhfile.fqfn}" {mkbuildoptglobals.extra_flags} ## Build the app.ld linker file diff --git a/tools/mkbuildoptglobals.py b/tools/mkbuildoptglobals.py old mode 100644 new mode 100755 index e407514cf2..6e3895b88e --- a/tools/mkbuildoptglobals.py +++ b/tools/mkbuildoptglobals.py @@ -180,11 +180,13 @@ Build does not work as expected. This does not fail often. Maybe PIC NIC. """ +import argparse from shutil import copyfile import glob import os import platform import sys +import textwrap import time # Need to work on signature line used for match to avoid conflicts with @@ -262,8 +264,8 @@ def copy_create_build_file(source_fqfn, build_target_fqfn): return True when file change detected. """ if os.path.exists(source_fqfn): - if (os.path.exists(build_target_fqfn)) and \ - (os.path.getmtime(build_target_fqfn) >= os.path.getmtime(source_fqfn)): + if os.path.exists(build_target_fqfn) and \ + os.path.getmtime(build_target_fqfn) >= os.path.getmtime(source_fqfn): # only copy newer files - do nothing, all is good return False else: @@ -459,19 +461,25 @@ def get_preferences_txt(file_fqfn, key): for line in file: name, value = line.partition("=")[::2] if name.strip().lower() == key: - if value.strip().lower() == 'true': - return True - else: - return False - print_err("Key " + key + " not found in preferences.txt. Default to true.") + val = value.strip().lower() + if val != 'true': + val = False + print_msg(f" preferences.txt: {key}={val}") + return val + print_err(" Key " + key + " not found in preferences.txt. Default to true.") return True # If we don't find it just assume it is set True -def check_preferences_txt(runtime_ide_path): - # return the state of "compiler.cache_core" in preferences.txt - file_fqfn = find_preferences_txt(runtime_ide_path) - if file_fqfn == "": - return True # cannot find file - assume enabled +def check_preferences_txt(runtime_ide_path, preferences_file): + # return the state of "compiler.cache_core" found in preferences.txt + file_fqfn = preferences_file + if file_fqfn != None and os.path.exists(file_fqfn): + pass + else: + file_fqfn = find_preferences_txt(runtime_ide_path) + if file_fqfn == "": + return True # cannot find file - assume enabled + print_msg("Using preferences from " + file_fqfn) return get_preferences_txt(file_fqfn, "compiler.cache_core") @@ -489,34 +497,118 @@ def synchronous_touch(globals_h_fqfn, commonhfile_fqfn): with open(commonhfile_fqfn, 'a'): os.utime(commonhfile_fqfn, ns=(ts.st_atime_ns, ts.st_mtime_ns)) +def determine_cache_state(args, runtime_ide_path, source_globals_h_fqfn): + if args.runtime_ide_version < 10802: + return False + elif args.cache_core != None: + print_msg(f"Preferences override, this prebuild script assumes the 'compiler.cache_core' parameter is set to {args.cache_core}") + print_msg(f"To change, modify 'mkbuildoptglobals.extra_flags=(--cache_core | --no_cache_core)' in 'platform.local.txt'") + return args.cache_core + else: + preferences_fqfn = None + if args.preferences_file != None: + preferences_fqfn = args.preferences_file + elif args.preferences_sketch != None: + preferences_fqfn = os.path.normpath( + os.path.join( + os.path.dirname(source_globals_h_fqfn), + args.preferences_sketch)) + elif args.preferences_env != None: + preferences_fqfn = os.getenv(args.preferences_env) + return check_preferences_txt(runtime_ide_path, preferences_fqfn) + + +""" +TODO sort out which of these are viable solutions + +Possible options for handling problems caused by: + ./arduino --preferences-file other-preferences.txt + ./arduino --pref compiler.cache_core=false + +--cache_core +--no_cache_core +--preferences_file (relative to IDE or full path) +--preferences_sketch (default looks for preferences.txt or specify path relative to sketch folder) +--preferences_env, only works on Linux + + export ARDUINO15_PREFERENCES_FILE=$(realpath other-name-than-default-preferences.txt ) + ./arduino --preferences-file other-name-than-default-preferences.txt + + platform.local.txt: mkbuildoptglobals.extra_flags=--preferences_env + + Tested with: + export ARDUINO15_PREFERENCES_FILE=$(realpath ~/projects/arduino/arduino-1.8.19/portable/preferences.txt) + ~/projects/arduino/arduino-1.8.18/arduino + + + Future Issues + * "--preferences-file" does not work for Arduino IDE 2.0, they plan to address at a future release + * Arduino IDE 2.0 does not support portable, they plan to address at a future release + +""" + + +def parse_args(): + extra_txt = '''\ + Use platform.local.txt 'mkbuildoptglobals.extra_flags=...' to supply override options: + --cache_core | --no_cache_core | --preferences_file PREFERENCES_FILE | ... + + more help at {} + '''.format(docs_url) + parser = argparse.ArgumentParser( + description='Prebuild processing for globals.h and build.opt file', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=textwrap.dedent(extra_txt)) + parser.add_argument('runtime_ide_path', help='Runtime IDE path, {runtime.ide.path}') + parser.add_argument('runtime_ide_version', type=int, help='Runtime IDE Version, {runtime.ide.version}') + parser.add_argument('build_path', help='Build path, {build.path}') + parser.add_argument('build_opt_fqfn', help="Build FQFN to build.opt") + parser.add_argument('source_globals_h_fqfn', help="Source FQFN Sketch.ino.globals.h") + parser.add_argument('commonhfile_fqfn', help="Core Source FQFN CommonHFile.h") + group = parser.add_mutually_exclusive_group(required=False) + group.add_argument('--cache_core', action='store_true', default=None, help='Assume a "compiler.cache_core" value of true') + group.add_argument('--no_cache_core', dest='cache_core', action='store_false', help='Assume a "compiler.cache_core" value of false') + group.add_argument('--preferences_file', help='Full path to preferences file') + group.add_argument('--preferences_sketch', nargs='?', action='store', const="preferences.txt", help='Sketch relative path to preferences file') + if "Linux" == platform.system(): + group.add_argument('--preferences_env', nargs='?', action='store', const="ARDUINO15_PREFERENCES_FILE", help='Use environment variable for path to preferences file') + return parser.parse_args() + # ref epilog, https://stackoverflow.com/a/50021771 + # ref nargs='*'', https://stackoverflow.com/a/4480202 + # ref no '--n' parameter, https://stackoverflow.com/a/21998252 def main(): global build_opt_signature global docs_url num_include_lines = 1 - if len(sys.argv) >= 6: - runtime_ide_path = os.path.normpath(sys.argv[1]) - build_path = os.path.normpath(sys.argv[2]) - build_opt_fqfn = os.path.normpath(sys.argv[3]) - source_globals_h_fqfn = os.path.normpath(sys.argv[4]) - commonhfile_fqfn = os.path.normpath(sys.argv[5]) + args = parse_args() + runtime_ide_path = os.path.normpath(args.runtime_ide_path) + build_path = os.path.normpath(args.build_path) + build_opt_fqfn = os.path.normpath(args.build_opt_fqfn) + source_globals_h_fqfn = os.path.normpath(args.source_globals_h_fqfn) + commonhfile_fqfn = os.path.normpath(args.commonhfile_fqfn) + if commonhfile_fqfn != None and len(commonhfile_fqfn): globals_name = os.path.basename(source_globals_h_fqfn) build_path_core, build_opt_name = os.path.split(build_opt_fqfn) globals_h_fqfn = os.path.join(build_path_core, globals_name) first_time = discover_1st_time_run(build_path) - use_aggressive_caching_workaround = check_preferences_txt(runtime_ide_path) + + use_aggressive_caching_workaround = determine_cache_state(args, runtime_ide_path, source_globals_h_fqfn) if first_time or \ not use_aggressive_caching_workaround or \ not os.path.exists(commonhfile_fqfn): enable_override(False, commonhfile_fqfn) + # A future timestamp on commonhfile_fqfn will cause everything to + # rebuild. This occurred during development and may happen after + # changing the system time. if time.time_ns() < os.stat(commonhfile_fqfn).st_mtime_ns: - print_err(f"Neutralize future timestamp on build file: {commonhfile_fqfn}") touch(commonhfile_fqfn) + print_err(f"Neutralized future timestamp on build file: {commonhfile_fqfn}") if not os.path.exists(build_path_core): os.makedirs(build_path_core) @@ -561,11 +653,11 @@ def main(): # touching commonhfile_fqfn in the source core tree will cause rebuild. # Looks like touching or writing unrelated files in the source core tree will cause rebuild. synchronous_touch(globals_h_fqfn, commonhfile_fqfn) - print_msg("Using 'aggressive caching' workaround.") + print_msg("Using 'aggressive caching' workaround, rebuild shared 'core.a' for current globals.") elif os.path.getsize(globals_h_fqfn): enable_override(True, commonhfile_fqfn) synchronous_touch(globals_h_fqfn, commonhfile_fqfn) - print_msg("Using 'aggressive caching' workaround.") + print_msg("Using 'aggressive caching' workaround, rebuild shared 'core.a' for current globals.") add_include_line(build_opt_fqfn, commonhfile_fqfn) add_include_line(build_opt_fqfn, globals_h_fqfn) @@ -586,8 +678,9 @@ def main(): print_msg(" Read more at " + docs_url) else: - print_err("Too few arguments. Add arguments:") - print_err(" Runtime IDE path, Build path, Build FQFN build.opt, Source FQFN Sketch.ino.globals.h, Core Source FQFN CommonHFile.h") + print_err(parser.parse_args('-h'.split())) + # print_err("Too few arguments. Add arguments:") + # print_err(" Runtime IDE path, Build path, Build FQFN build.opt, Source FQFN Sketch.ino.globals.h, Core Source FQFN CommonHFile.h") handle_error(1) handle_error(0) # commit print buffer