From d9d6a4d5ae0faa2766fa6217dc2fe91667e3e9d9 Mon Sep 17 00:00:00 2001 From: Maxim Prokhorov Date: Tue, 27 May 2025 02:29:26 +0300 Subject: [PATCH 1/5] Tools - makecorever.py explicit output argument Don't hide where the file is actually placed Using pathlib instead of raw strings to simplify path ops Plus, add stdout preview when output is missing Clean-up git-describe <-> version availility checks --- platform.txt | 2 +- tools/makecorever.py | 178 ++++++++++++++++++++++++++------------ tools/platformio-build.py | 8 +- 3 files changed, 128 insertions(+), 60 deletions(-) diff --git a/platform.txt b/platform.txt index c34c719ae7..f3392d652d 100644 --- a/platform.txt +++ b/platform.txt @@ -121,7 +121,7 @@ compiler.elf2hex.extra_flags= ## needs git recipe.hooks.sketch.prebuild.pattern="{runtime.tools.python3.path}/python3" -I "{runtime.tools.signing}" --mode header --publickey "{build.source.path}/public.key" --out "{build.path}/core/Updater_Signing.h" # This is quite a working hack. This form of prebuild hook, while intuitive, is not explicitly documented. -recipe.hooks.prebuild.1.pattern="{runtime.tools.python3.path}/python3" -I "{runtime.tools.makecorever}" --build_path "{build.path}" --platform_path "{runtime.platform.path}" --version "{version}" +recipe.hooks.prebuild.1.pattern="{runtime.tools.python3.path}/python3" -I "{runtime.tools.makecorever}" --git-root "{runtime.platform.path}" --version "{version}" "{build.path}/core/core_version.h" # Handle processing sketch global options recipe.hooks.prebuild.2.pattern="{runtime.tools.python3.path}/python3" -I "{runtime.tools.mkbuildoptglobals}" "{runtime.ide.path}" {runtime.ide.version} "{build.path}" "{build.opt.fqfn}" "{globals.h.source.fqfn}" "{commonhfile.fqfn}" {mkbuildoptglobals.extra_flags} diff --git a/tools/makecorever.py b/tools/makecorever.py index d3f44f1742..fc2c48ba0e 100755 --- a/tools/makecorever.py +++ b/tools/makecorever.py @@ -18,25 +18,60 @@ # along with this program. If not, see . import argparse -import os +import pathlib import subprocess +import sys +from typing import Optional, TextIO -def generate(path, platform_path, version="unspecified", release = False): - def git(*args): - cmd = ["git", "-C", platform_path] - cmd.extend(args) - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True, stderr=subprocess.DEVNULL) - return proc.stdout.readlines()[0].strip() - text = "" +PWD = pathlib.Path(__file__).parent.absolute() + +VERSION_UNSPECIFIED = "unspecified" +VERSION_DEFAULT = ("0", "0", "0") + + +def escape_version(v: str) -> str: + return v.replace("-", "_").replace(".", "_") + + +def check_git(*args: str, cwd: Optional[str]): + cmd = ["git"] + if cwd: + cmd.extend(["-C", cwd]) + cmd.extend(args) + + with subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + universal_newlines=True, + stderr=subprocess.DEVNULL, + ) as proc: + if proc.stdout: + return proc.stdout.readlines()[0].strip() + + return "" + - git_ver = "00000000" +def generate( + out: TextIO, + *, + git_root: pathlib.Path, + hash_length: int = 8, + release: bool, + version: str, +): + git_root = git_root.absolute() + git_cwd = git_root.as_posix() + + def git(*args): + return check_git(*args, cwd=git_cwd) + + git_ver = "0" * hash_length try: - git_ver = git("rev-parse", "--short=8", "HEAD") + git_ver = git("rev-parse", f"--short={hash_length}", "HEAD") except Exception: - pass - text = "#define ARDUINO_ESP8266_GIT_VER 0x{}\n".format(git_ver) + raise # version is # - using Arduino-CLI: @@ -50,70 +85,101 @@ def git(*args): # in any case, get a better version when git is around git_desc = git("describe", "--tags") except Exception: + raise + + if version == VERSION_UNSPECIFIED: + version = git_desc + + version_triple = list(VERSION_DEFAULT) + + try: + version_triple = version.split(".", 2) + except ValueError: pass - text += "#define ARDUINO_ESP8266_GIT_DESC {}\n".format(git_desc) - text += "#define ARDUINO_ESP8266_VERSION {}\n".format(version) - text += "\n" + major, minor, patch = version_triple - version_split = version.split(".") - # major: if present, skip "unix-" in "unix-3" - text += "#define ARDUINO_ESP8266_MAJOR {}\n".format(version_split[0].split("-")[-1]) - text += "#define ARDUINO_ESP8266_MINOR {}\n".format(version_split[1]) - # revision can be ".n" or ".n-dev" or ".n-42-g00d1e5" - revision = version_split[2].split("-") - text += "#define ARDUINO_ESP8266_REVISION {}\n".format(revision[0]) - text += "\n" + major = major.split("-")[-1] + revision = patch.split("-")[0] - # release or dev ? + text = rf"""// ! ! ! DO NOT EDIT, AUTOMATICALLY GENERATED ! ! ! +#define ARDUINO_ESP8266_GIT_VER 0x{git_ver} +#define ARDUINO_ESP8266_GIT_DESC {git_desc} +#define ARDUINO_ESP8266_VERSION {version} + +#define ARDUINO_ESP8266_MAJOR {major} +#define ARDUINO_ESP8266_MINOR {minor} +#define ARDUINO_ESP8266_REVISION {revision} +""" if release: - text += "#define ARDUINO_ESP8266_RELEASE \"{}\"\n".format(git_desc) - text += "#define ARDUINO_ESP8266_RELEASE_{}\n".format(git_desc.replace("-","_").replace(".","_")) + if version != VERSION_UNSPECIFIED: + release_version = version + else: + release_version = git_desc + + text += rf""" +#define ARDUINO_ESP8266_RELEASE \"{release_version}\" +#define ARDUINO_ESP8266_RELEASE_{escape_version(release_version)} +""" else: - text += "#define ARDUINO_ESP8266_DEV 1 // development version\n" - - try: - with open(path, "r") as inp: - old_text = inp.read() - if old_text == text: - return - except Exception: - pass + text += """ +#define ARDUINO_ESP8266_DEV 1 // development version +""" - with open(path, "w") as out: - out.write(text) + out.write(text) if __name__ == "__main__": parser = argparse.ArgumentParser(description="Generate core_version.h") parser.add_argument( - "-b", "--build_path", action="store", required=True, help="build.path variable" + "--git-root", + action="store", + help="ESP8266 Core Git root. In platform.txt, this is {platform.path}", + type=pathlib.Path, + default=PWD / "..", + ) + parser.add_argument( + "--hash-length", + default=8, + type=int, + help="Used in git rev-parse --short=...", ) parser.add_argument( - "-p", - "--platform_path", + "--version", action="store", - required=True, - help="platform.path variable", + default=VERSION_UNSPECIFIED, + help="ESP8266 Core version string. In platform.txt, this is {version}", ) parser.add_argument( - "-v", "--version", action="store", required=True, help="version variable" + "--release", + action="store_true", + default=False, + help="In addition to numeric version, also provide ARDUINO_ESP8266_RELEASE{,_...} definitions", + ) + parser.add_argument( + "output", + nargs="?", + type=str, + default="", ) - parser.add_argument("-i", "--include_dir", default="core") - parser.add_argument("-r", "--release", action="store_true", default=False) args = parser.parse_args() - include_dir = os.path.join(args.build_path, args.include_dir) - try: - os.makedirs(include_dir) - except Exception: - pass + def select_output(s: str) -> TextIO: + if not s: + return sys.stdout - generate( - os.path.join(include_dir, "core_version.h"), - args.platform_path, - version=args.version, - release=args.release - ) + out = pathlib.Path(s) + out.parent.mkdir(parents=True, exist_ok=True) + + return out.open("r", encoding="utf-8") + + with select_output(args.output) as out: + generate( + out, + git_root=args.git_root, + hash_length=args.hash_length, + release=args.release, + version=args.version, + ) diff --git a/tools/platformio-build.py b/tools/platformio-build.py index 20fe8c77fb..fafcac728d 100644 --- a/tools/platformio-build.py +++ b/tools/platformio-build.py @@ -409,18 +409,20 @@ def platform_txt_version(default): if isdir(join(FRAMEWORK_DIR, ".git")): - cmd = '"$PYTHONEXE" "{script}" -b "$BUILD_DIR" -p "{framework_dir}" -v {version}' + out = join("$BUILD_DIR", "core", "core_version.h") + + cmd = '"$PYTHONEXE" "{script}" --git-root "{framework_dir}" --version {version} "$TARGET"' fmt = { "script": join(FRAMEWORK_DIR, "tools", "makecorever.py"), "framework_dir": FRAMEWORK_DIR, - "version": platform_txt_version("unspecified") + "version": platform_txt_version("unspecified"), } env.Prepend(CPPPATH=[ join("$BUILD_DIR", "core") ]) core_version = env.Command( - join("$BUILD_DIR", "core", "core_version.h"), + out, join(FRAMEWORK_DIR, ".git"), env.VerboseAction(cmd.format(**fmt), "Generating $TARGET") ) From 8944d1453e06e30fdca0c53fdd2b7f1c58ec614b Mon Sep 17 00:00:00 2001 From: Maxim Prokhorov Date: Tue, 27 May 2025 05:27:25 +0300 Subject: [PATCH 2/5] typo --- tools/makecorever.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/makecorever.py b/tools/makecorever.py index fc2c48ba0e..2e26a05247 100755 --- a/tools/makecorever.py +++ b/tools/makecorever.py @@ -173,7 +173,7 @@ def select_output(s: str) -> TextIO: out = pathlib.Path(s) out.parent.mkdir(parents=True, exist_ok=True) - return out.open("r", encoding="utf-8") + return out.open("w", encoding="utf-8") with select_output(args.output) as out: generate( From 77da4afa2da795a7e00df626f8bc539bca841c42 Mon Sep 17 00:00:00 2001 From: Maxim Prokhorov Date: Tue, 27 May 2025 06:02:44 +0300 Subject: [PATCH 3/5] fix unhandled exc in git cmd --- tools/makecorever.py | 45 ++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/tools/makecorever.py b/tools/makecorever.py index 2e26a05247..1cabe62d7f 100755 --- a/tools/makecorever.py +++ b/tools/makecorever.py @@ -41,14 +41,20 @@ def check_git(*args: str, cwd: Optional[str]): cmd.extend(["-C", cwd]) cmd.extend(args) - with subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - universal_newlines=True, - stderr=subprocess.DEVNULL, - ) as proc: - if proc.stdout: - return proc.stdout.readlines()[0].strip() + try: + with subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + universal_newlines=True, + stderr=subprocess.DEVNULL, + ) as proc: + if proc.stdout: + lines = proc.stdout.readlines() + return lines[0].strip() + except IndexError: + pass + except FileNotFoundError: + pass return "" @@ -68,10 +74,7 @@ def git(*args): return check_git(*args, cwd=git_cwd) git_ver = "0" * hash_length - try: - git_ver = git("rev-parse", f"--short={hash_length}", "HEAD") - except Exception: - raise + git_ver = git("rev-parse", f"--short={hash_length}", "HEAD") or git_ver # version is # - using Arduino-CLI: @@ -80,22 +83,20 @@ def git(*args): # - using git: # - 5.6.7 (from release script, official release) # - 5.6.7-42-g00d1e5 (from release script, test release) - git_desc = version - try: - # in any case, get a better version when git is around - git_desc = git("describe", "--tags") - except Exception: - raise + + # in any case, get a better version when git is around + git_desc = git("describe", "--tags") or version if version == VERSION_UNSPECIFIED: version = git_desc version_triple = list(VERSION_DEFAULT) - try: - version_triple = version.split(".", 2) - except ValueError: - pass + if version != VERSION_UNSPECIFIED: + try: + version_triple = version.split(".", 2) + except ValueError: + pass major, minor, patch = version_triple From 557ac6cafdcd116de535e26ca2c7f2ed179be80f Mon Sep 17 00:00:00 2001 From: Maxim Prokhorov Date: Tue, 27 May 2025 06:06:45 +0300 Subject: [PATCH 4/5] unused escape --- tools/makecorever.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tools/makecorever.py b/tools/makecorever.py index 1cabe62d7f..d3711560b7 100755 --- a/tools/makecorever.py +++ b/tools/makecorever.py @@ -31,10 +31,6 @@ VERSION_DEFAULT = ("0", "0", "0") -def escape_version(v: str) -> str: - return v.replace("-", "_").replace(".", "_") - - def check_git(*args: str, cwd: Optional[str]): cmd = ["git"] if cwd: @@ -120,7 +116,7 @@ def git(*args): text += rf""" #define ARDUINO_ESP8266_RELEASE \"{release_version}\" -#define ARDUINO_ESP8266_RELEASE_{escape_version(release_version)} +#define ARDUINO_ESP8266_RELEASE_{major}_{minor}_{revision} """ else: text += """ From e641c9f3de22fec915c1faef35a38674d082e4ce Mon Sep 17 00:00:00 2001 From: Maxim Prokhorov Date: Tue, 27 May 2025 06:28:53 +0300 Subject: [PATCH 5/5] RELEASE using same values --- cores/esp8266/Esp-version.cpp | 3 ++- cores/esp8266/Esp.cpp | 2 +- tools/makecorever.py | 7 +------ 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/cores/esp8266/Esp-version.cpp b/cores/esp8266/Esp-version.cpp index c1ff61eb06..85485b2f73 100644 --- a/cores/esp8266/Esp-version.cpp +++ b/cores/esp8266/Esp-version.cpp @@ -20,10 +20,11 @@ #include #include -#include #include // LWIP_HASH_STR (lwip2) #include // BEARSSL_GIT short hash +#include + #define STRHELPER(x) #x #define STR(x) STRHELPER(x) // stringifier diff --git a/cores/esp8266/Esp.cpp b/cores/esp8266/Esp.cpp index 67719dcfe7..8a3ea32515 100644 --- a/cores/esp8266/Esp.cpp +++ b/cores/esp8266/Esp.cpp @@ -254,7 +254,7 @@ String EspClass::getCoreVersion() return String(core_release); } char buf[12]; - snprintf(buf, sizeof(buf), "%08x", core_version); + snprintf(buf, sizeof(buf), PSTR("%08x"), core_version); return String(buf); } diff --git a/tools/makecorever.py b/tools/makecorever.py index d3711560b7..9062e5b4da 100755 --- a/tools/makecorever.py +++ b/tools/makecorever.py @@ -109,13 +109,8 @@ def git(*args): #define ARDUINO_ESP8266_REVISION {revision} """ if release: - if version != VERSION_UNSPECIFIED: - release_version = version - else: - release_version = git_desc - text += rf""" -#define ARDUINO_ESP8266_RELEASE \"{release_version}\" +#define ARDUINO_ESP8266_RELEASE \"{major}.{minor}.{revision}\" #define ARDUINO_ESP8266_RELEASE_{major}_{minor}_{revision} """ else: