From 2ebca1c15855239f24ca10ce2cba33fe74a87469 Mon Sep 17 00:00:00 2001 From: Lucas Saavedra Vaz <32426024+lucasssvaz@users.noreply.github.com> Date: Mon, 7 Jul 2025 12:19:29 -0300 Subject: [PATCH] change(esptool): Upgrade esptool to release v5.0.0 --- .github/scripts/package_esptool.sh | 129 ------------ .github/scripts/update_esptool.py | 236 ++++++++++++++++++++++ package/package_esp32_index.template.json | 60 +++--- 3 files changed, 266 insertions(+), 159 deletions(-) delete mode 100755 .github/scripts/package_esptool.sh create mode 100644 .github/scripts/update_esptool.py diff --git a/.github/scripts/package_esptool.sh b/.github/scripts/package_esptool.sh deleted file mode 100755 index 32b87b277e9..00000000000 --- a/.github/scripts/package_esptool.sh +++ /dev/null @@ -1,129 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# Check version argument -if [[ $# -ne 3 ]]; then - echo "Usage: $0 " - echo "Example: $0 5.0.dev1 /tmp/esptool /tmp/esptool-5.0.dev1.json" - exit 1 -fi - -VERSION=$1 -BASE_FOLDER=$2 -JSON_PATH=$3 - -export COPYFILE_DISABLE=1 - -shopt -s nullglob # So for loop doesn't run if no matches - -# Function to update JSON for a given host -function update_json_for_host { - local host=$1 - local archive=$2 - - # Extract the old url from the JSON for this host, then replace only the filename - old_url=$(jq -r --arg host "$host" ' - .packages[].tools[] | select(.name == "esptool_py") | .systems[] | select(.host == $host) | .url // empty - ' "$tmp_json") - if [[ -n "$old_url" ]]; then - base_url="${old_url%/*}" - url="$base_url/$archive" - else - echo "No old url found for $host" - exit 1 - fi - - archiveFileName="$archive" - checksum="SHA-256:$(shasum -a 256 "$archive" | awk '{print $1}')" - size=$(stat -f%z "$archive") - - # Use jq to update the JSON - jq --arg host "$host" \ - --arg url "$url" \ - --arg archiveFileName "$archiveFileName" \ - --arg checksum "$checksum" \ - --arg size "$size" \ - ' - .packages[].tools[] - |= if .name == "esptool_py" then - .systems = ( - ((.systems // []) | map(select(.host != $host))) + [{ - host: $host, - url: $url, - archiveFileName: $archiveFileName, - checksum: $checksum, - size: $size - }] - ) - else - . - end - ' "$tmp_json" > "$tmp_json.new" && mv "$tmp_json.new" "$tmp_json" -} - -cd "$BASE_FOLDER" - -# Delete all archives before starting -rm -f esptool-*.tar.gz esptool-*.zip - -for dir in esptool-*; do - # Check if directory exists and is a directory - if [[ ! -d "$dir" ]]; then - continue - fi - - base="${dir#esptool-}" - - # Add 'linux-' prefix if base doesn't contain linux/macos/win64 - if [[ "$base" != *linux* && "$base" != *macos* && "$base" != *win64* ]]; then - base="linux-${base}" - fi - - if [[ "$dir" == esptool-win* ]]; then - # Windows zip archive - zipfile="esptool-v${VERSION}-${base}.zip" - echo "Creating $zipfile from $dir ..." - zip -r "$zipfile" "$dir" - else - # Non-Windows: set permissions and tar.gz archive - tarfile="esptool-v${VERSION}-${base}.tar.gz" - echo "Setting permissions and creating $tarfile from $dir ..." - chmod -R u=rwx,g=rx,o=rx "$dir" - tar -cvzf "$tarfile" "$dir" - fi -done - -# After the for loop, update the JSON for each archive -# Create a temporary JSON file to accumulate changes -tmp_json="${JSON_PATH}.tmp" -cp "$JSON_PATH" "$tmp_json" - -for archive in esptool-v"${VERSION}"-*.tar.gz esptool-v"${VERSION}"-*.zip; do - [ -f "$archive" ] || continue - - echo "Updating JSON for $archive" - - # Determine host from archive name - case "$archive" in - *linux-amd64*) host="x86_64-pc-linux-gnu" ;; - *linux-armv7*) host="arm-linux-gnueabihf" ;; - *linux-aarch64*) host="aarch64-linux-gnu" ;; - *macos-amd64*) host="x86_64-apple-darwin" ;; - *macos-arm64*) host="arm64-apple-darwin" ;; - *win64*) hosts=("x86_64-mingw32" "i686-mingw32") ;; - *) echo "Unknown host for $archive"; continue ;; - esac - - # For win64, loop over both hosts; otherwise, use a single host - if [[ "$archive" == *win64* ]]; then - for host in "${hosts[@]}"; do - update_json_for_host "$host" "$archive" - done - else - update_json_for_host "$host" "$archive" - fi -done - -# After all archives are processed, move the temporary JSON to the final file -mv "$tmp_json" "$JSON_PATH" diff --git a/.github/scripts/update_esptool.py b/.github/scripts/update_esptool.py new file mode 100644 index 00000000000..d99462fcb8f --- /dev/null +++ b/.github/scripts/update_esptool.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 + +# This script is used to re-package the esptool if needed and update the JSON file +# for the Arduino ESP32 platform. +# +# The script has only been tested on macOS. +# +# For regular esptool releases, the generated packages already contain the correct permissions, +# extensions and are uploaded to the GitHub release assets. In this case, the script will only +# update the JSON file with the information from the GitHub release. +# +# The script can be used in two modes: +# 1. Local build: The build artifacts must be already downloaded and extracted in the base_folder. +# This is useful for esptool versions that are not yet released and that are grabbed from the +# GitHub build artifacts. +# 2. Release build: The script will get the release information from GitHub and update the JSON file. +# This is useful for esptool versions that are already released and that are uploaded to the +# GitHub release assets. +# +# For local build, the artifacts must be already downloaded and extracted in the base_folder +# set with the -l option. +# For example, a base folder "esptool" should contain the following folders extracted directly +# from the GitHub build artifacts: +# esptool/esptool-linux-aarch64 +# esptool/esptool-linux-amd64 +# esptool/esptool-linux-armv7 +# esptool/esptool-macos-amd64 +# esptool/esptool-macos-arm64 +# esptool/esptool-windows-amd64 + +import argparse +import json +import os +import shutil +import stat +import tarfile +import zipfile +import hashlib +import requests +from pathlib import Path + +def compute_sha256(filepath): + sha256 = hashlib.sha256() + with open(filepath, "rb") as f: + for block in iter(lambda: f.read(4096), b""): + sha256.update(block) + return f"SHA-256:{sha256.hexdigest()}" + +def get_file_size(filepath): + return os.path.getsize(filepath) + +def update_json_for_host(tmp_json_path, version, host, url, archiveFileName, checksum, size): + with open(tmp_json_path) as f: + data = json.load(f) + + for pkg in data.get("packages", []): + for tool in pkg.get("tools", []): + if tool.get("name") == "esptool_py": + tool["version"] = version + + if url is None: + # If the URL is not set, we need to find the old URL and update it + for system in tool.get("systems", []): + if system.get("host") == host: + url = system.get("url").replace(system.get("archiveFileName"), archiveFileName) + break + else: + print(f"No old URL found for host {host}. Using empty URL.") + url = "" + + # Preserve existing systems order and update or append the new system + systems = tool.get("systems", []) + system_updated = False + for i, system in enumerate(systems): + if system.get("host") == host: + systems[i] = { + "host": host, + "url": url, + "archiveFileName": archiveFileName, + "checksum": checksum, + "size": str(size), + } + system_updated = True + break + + if not system_updated: + systems.append({ + "host": host, + "url": url, + "archiveFileName": archiveFileName, + "checksum": checksum, + "size": str(size), + }) + tool["systems"] = systems + + with open(tmp_json_path, "w") as f: + json.dump(data, f, indent=2, sort_keys=False, ensure_ascii=False) + f.write("\n") + +def update_tools_dependencies(tmp_json_path, version): + with open(tmp_json_path) as f: + data = json.load(f) + + for pkg in data.get("packages", []): + for platform in pkg.get("platforms", []): + for dep in platform.get("toolsDependencies", []): + if dep.get("name") == "esptool_py": + dep["version"] = version + + with open(tmp_json_path, "w") as f: + json.dump(data, f, indent=2, sort_keys=False, ensure_ascii=False) + f.write("\n") + +def create_archives(version, base_folder): + archive_files = [] + + for dirpath in Path(base_folder).glob("esptool-*"): + if not dirpath.is_dir(): + continue + + base = dirpath.name[len("esptool-"):] + + if "windows" in dirpath.name: + zipfile_name = f"esptool-v{version}-{base}.zip" + print(f"Creating {zipfile_name} from {dirpath} ...") + with zipfile.ZipFile(zipfile_name, "w", zipfile.ZIP_DEFLATED) as zipf: + for root, _, files in os.walk(dirpath): + for file in files: + full_path = os.path.join(root, file) + zipf.write(full_path, os.path.relpath(full_path, start=dirpath)) + archive_files.append(zipfile_name) + else: + tarfile_name = f"esptool-v{version}-{base}.tar.gz" + print(f"Creating {tarfile_name} from {dirpath} ...") + for root, dirs, files in os.walk(dirpath): + for name in dirs + files: + os.chmod(os.path.join(root, name), stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | + stat.S_IRGRP | stat.S_IXGRP | + stat.S_IROTH | stat.S_IXOTH) + with tarfile.open(tarfile_name, "w:gz") as tar: + tar.add(dirpath, arcname=dirpath.name) + archive_files.append(tarfile_name) + + return archive_files + +def determine_hosts(archive_name): + if "linux-amd64" in archive_name: + return ["x86_64-pc-linux-gnu"] + elif "linux-armv7" in archive_name: + return ["arm-linux-gnueabihf"] + elif "linux-aarch64" in archive_name: + return ["aarch64-linux-gnu"] + elif "macos-amd64" in archive_name: + return ["x86_64-apple-darwin"] + elif "macos-arm64" in archive_name: + return ["arm64-apple-darwin"] + elif "windows-amd64" in archive_name: + return ["x86_64-mingw32", "i686-mingw32"] + else: + return [] + +def update_json_from_local_build(tmp_json_path, version, base_folder, archive_files): + for archive in archive_files: + print(f"Processing archive: {archive}") + hosts = determine_hosts(archive) + if not hosts: + print(f"Skipping unknown archive type: {archive}") + continue + + archive_path = Path(archive) + checksum = compute_sha256(archive_path) + size = get_file_size(archive_path) + + for host in hosts: + update_json_for_host(tmp_json_path, version, host, None, archive_path.name, checksum, size) + +def update_json_from_release(tmp_json_path, version, release_info): + assets = release_info.get("assets", []) + for asset in assets: + if (asset.get("name").endswith(".tar.gz") or asset.get("name").endswith(".zip")) and "esptool" in asset.get("name"): + asset_fname = asset.get("name") + print(f"Processing asset: {asset_fname}") + hosts = determine_hosts(asset_fname) + if not hosts: + print(f"Skipping unknown archive type: {asset_fname}") + continue + + asset_url = asset.get("browser_download_url") + asset_checksum = asset.get("digest").replace("sha256:", "SHA-256:") + asset_size = asset.get("size") + if asset_checksum is None: + asset_checksum = "" + print(f"Asset {asset_fname} has no checksum. Please set the checksum in the JSON file.") + + for host in hosts: + update_json_for_host(tmp_json_path, version, host, asset_url, asset_fname, asset_checksum, asset_size) + +def get_release_info(version): + url = f"https://api.github.com/repos/espressif/esptool/releases/tags/v{version}" + response = requests.get(url) + response.raise_for_status() + return response.json() + +def main(): + parser = argparse.ArgumentParser(description="Repack esptool and update JSON metadata.") + parser.add_argument("version", help="Version of the esptool (e.g. 5.0.dev1)") + parser.add_argument("-l", "--local", dest="base_folder", help="Enable local build mode and set the base folder with unpacked artifacts") + args = parser.parse_args() + + script_dir = Path(__file__).resolve().parent + json_path = (script_dir / "../../package/package_esp32_index.template.json").resolve() + tmp_json_path = Path(str(json_path) + ".tmp") + shutil.copy(json_path, tmp_json_path) + + local_build = args.base_folder is not None + + if local_build: + os.chdir(args.base_folder) + os.environ['COPYFILE_DISABLE'] = 'true' # this disables including resource forks in tar files on macOS + # Clear any existing archive files + for file in Path(args.base_folder).glob("esptool-*.*"): + file.unlink() + archive_files = create_archives(args.version, args.base_folder) + update_json_from_local_build(tmp_json_path, args.version, args.base_folder, archive_files) + else: + release_info = get_release_info(args.version) + update_json_from_release(tmp_json_path, args.version, release_info) + + print(f"Updating esptool version fields to {args.version}") + update_tools_dependencies(tmp_json_path, args.version) + + shutil.move(tmp_json_path, json_path) + print(f"Done. JSON updated at {json_path}") + +if __name__ == "__main__": + main() diff --git a/package/package_esp32_index.template.json b/package/package_esp32_index.template.json index bbb77f8ef5a..5f5d1f0e08b 100644 --- a/package/package_esp32_index.template.json +++ b/package/package_esp32_index.template.json @@ -81,7 +81,7 @@ { "packager": "esp32", "name": "esptool_py", - "version": "5.0.dev1" + "version": "5.0.0" }, { "packager": "esp32", @@ -469,56 +469,56 @@ }, { "name": "esptool_py", - "version": "5.0.dev1", + "version": "5.0.0", "systems": [ { "host": "aarch64-linux-gnu", - "url": "https://github.com/espressif/arduino-esp32/releases/download/3.2.0/esptool-v5.0.dev1-linux-aarch64.tar.gz", - "archiveFileName": "esptool-v5.0.dev1-linux-aarch64.tar.gz", - "checksum": "SHA-256:bfafa7a7723ebbabfd8b6e3ca5ae00bfead0331de923754aeddb43b2c116a078", - "size": "58241736" + "url": "https://github.com/espressif/esptool/releases/download/v5.0.0/esptool-v5.0.0-linux-aarch64.tar.gz", + "archiveFileName": "esptool-v5.0.0-linux-aarch64.tar.gz", + "checksum": "SHA-256:2bf239f3ed76141a957cadb205b94414ec6da9ace4e85f285e247d20a92b83e3", + "size": "58231895" }, { "host": "x86_64-pc-linux-gnu", - "url": "https://github.com/espressif/arduino-esp32/releases/download/3.2.0/esptool-v5.0.dev1-linux-amd64.tar.gz", - "archiveFileName": "esptool-v5.0.dev1-linux-amd64.tar.gz", - "checksum": "SHA-256:acd0486e96586b99d053a1479acbbbfcae8667227c831cdc53a171f9ccfa27ee", - "size": "100740042" + "url": "https://github.com/espressif/esptool/releases/download/v5.0.0/esptool-v5.0.0-linux-amd64.tar.gz", + "archiveFileName": "esptool-v5.0.0-linux-amd64.tar.gz", + "checksum": "SHA-256:3b3835d266ac61f3242758f2fe34e3b33dbe6ee4b5acde005da793356f9f7043", + "size": "100783748" }, { "host": "arm-linux-gnueabihf", - "url": "https://github.com/espressif/arduino-esp32/releases/download/3.2.0/esptool-v5.0.dev1-linux-armv7.tar.gz", - "archiveFileName": "esptool-v5.0.dev1-linux-armv7.tar.gz", - "checksum": "SHA-256:ea77a38681506761bbb7b0b39c130811ed565667b67ebbdb4d6dcc6cb6e07368", - "size": "53451939" + "url": "https://github.com/espressif/esptool/releases/download/v5.0.0/esptool-v5.0.0-linux-armv7.tar.gz", + "archiveFileName": "esptool-v5.0.0-linux-armv7.tar.gz", + "checksum": "SHA-256:e55cd321abecfcf27f72a2bff5d5e19a5365fd400de66d71c5e7218e77556315", + "size": "53461760" }, { "host": "x86_64-apple-darwin", - "url": "https://github.com/espressif/arduino-esp32/releases/download/3.2.0/esptool-v5.0.dev1-macos-amd64.tar.gz", - "archiveFileName": "esptool-v5.0.dev1-macos-amd64.tar.gz", - "checksum": "SHA-256:900a8e90731208bee96647e0e207a43612b9452c2120c4fdc0ff4c6be226257b", - "size": "59631998" + "url": "https://github.com/espressif/esptool/releases/download/v5.0.0/esptool-v5.0.0-macos-amd64.tar.gz", + "archiveFileName": "esptool-v5.0.0-macos-amd64.tar.gz", + "checksum": "SHA-256:424da2bdf0435257ad81bcb7eae6fd8dd7f675ce5b2ee60032f4ecec4d6a5d45", + "size": "59629533" }, { "host": "arm64-apple-darwin", - "url": "https://github.com/espressif/arduino-esp32/releases/download/3.2.0/esptool-v5.0.dev1-macos-arm64.tar.gz", - "archiveFileName": "esptool-v5.0.dev1-macos-arm64.tar.gz", - "checksum": "SHA-256:3653f4de73cb4fc6a25351eaf663708e91c65ae3265d75bd54ca4315a4350bb4", - "size": "56349992" + "url": "https://github.com/espressif/esptool/releases/download/v5.0.0/esptool-v5.0.0-macos-arm64.tar.gz", + "archiveFileName": "esptool-v5.0.0-macos-arm64.tar.gz", + "checksum": "SHA-256:b91dfe1da7b0041376683dec10a91dfb266fbda2fb86ed87c4a034ff7182ee56", + "size": "56343104" }, { "host": "x86_64-mingw32", - "url": "https://github.com/espressif/arduino-esp32/releases/download/3.2.0/esptool-v5.0.dev1-win64.zip", - "archiveFileName": "esptool-v5.0.dev1-win64.zip", - "checksum": "SHA-256:1e8fd89645daf94f2d4406ec73c9004e617ea921079515f9fd749205eece4d6d", - "size": "59102658" + "url": "https://github.com/espressif/esptool/releases/download/v5.0.0/esptool-v5.0.0-windows-amd64.zip", + "archiveFileName": "esptool-v5.0.0-windows-amd64.zip", + "checksum": "SHA-256:2294107f66db6f09b886b337728a981173c9e7eab45a030928a8a5a1370611ca", + "size": "59105322" }, { "host": "i686-mingw32", - "url": "https://github.com/espressif/arduino-esp32/releases/download/3.2.0/esptool-v5.0.dev1-win64.zip", - "archiveFileName": "esptool-v5.0.dev1-win64.zip", - "checksum": "SHA-256:1e8fd89645daf94f2d4406ec73c9004e617ea921079515f9fd749205eece4d6d", - "size": "59102658" + "url": "https://github.com/espressif/esptool/releases/download/v5.0.0/esptool-v5.0.0-windows-amd64.zip", + "archiveFileName": "esptool-v5.0.0-windows-amd64.zip", + "checksum": "SHA-256:2294107f66db6f09b886b337728a981173c9e7eab45a030928a8a5a1370611ca", + "size": "59105322" } ] },