From 7dcc53efebfec92bc883db696a30b97c3c92b146 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 5 Jul 2025 17:38:02 +0200 Subject: [PATCH 1/6] Update main.py --- builder/main.py | 187 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 173 insertions(+), 14 deletions(-) diff --git a/builder/main.py b/builder/main.py index dde87dfde..9ed31aa84 100644 --- a/builder/main.py +++ b/builder/main.py @@ -13,8 +13,10 @@ # limitations under the License. import locale +import json import os import re +import semantic_version import shlex import subprocess import sys @@ -30,8 +32,10 @@ ) from platformio.project.helpers import get_project_dir +from platformio.package.version import pepver_to_semver from platformio.util import get_serial_ports + # Initialize environment and configuration env = DefaultEnvironment() platform = env.PioPlatform() @@ -41,6 +45,165 @@ # Framework directory path FRAMEWORK_DIR = platform.get_package_dir("framework-arduinoespressif32") +python_deps = { + "uv": ">=0.1.0", + "pyyaml": ">=6.0.2", + "rich-click": ">=1.8.6", + "zopfli": ">=0.2.2", + "intelhex": ">=2.3.0", + "rich": ">=14.0.0", + "esp-idf-size": ">=1.6.1" +} + + +def get_packages_to_install(deps, installed_packages): + """Generator for Python packages to install""" + for package, spec in deps.items(): + if package not in installed_packages: + yield package + else: + version_spec = semantic_version.Spec(spec) + if not version_spec.match(installed_packages[package]): + yield package + + +def install_python_deps(): + """Ensure uv package manager is available, install with pip if not""" + try: + result = subprocess.run( + ["uv", "--version"], + capture_output=True, + text=True, + timeout=3 + ) + uv_available = result.returncode == 0 + except (FileNotFoundError, subprocess.TimeoutExpired): + uv_available = False + + if not uv_available: + try: + result = subprocess.run( + [env.subst("$PYTHONEXE"), "-m", "pip", "install", "uv>=0.1.0", "-q", "-q", "-q"], + capture_output=True, + text=True, + timeout=30 # 30 second timeout + ) + if result.returncode != 0: + if result.stderr: + print(f"Error output: {result.stderr.strip()}") + return False + except subprocess.TimeoutExpired: + print("Error: uv installation timed out") + return False + except FileNotFoundError: + print("Error: Python executable not found") + return False + except Exception as e: + print(f"Error installing uv package manager: {e}") + return False + + + def _get_installed_uv_packages(): + result = {} + try: + cmd = ["uv", "pip", "list", "--format=json"] + result_obj = subprocess.run( + cmd, + capture_output=True, + text=True, + encoding='utf-8', + timeout=30 # 30 second timeout + ) + + if result_obj.returncode == 0: + content = result_obj.stdout.strip() + if content: + packages = json.loads(content) + for p in packages: + result[p["name"]] = pepver_to_semver(p["version"]) + else: + print(f"Warning: pip list failed with exit code {result_obj.returncode}") + if result_obj.stderr: + print(f"Error output: {result_obj.stderr.strip()}") + + except subprocess.TimeoutExpired: + print("Warning: uv pip list command timed out") + except (json.JSONDecodeError, KeyError) as e: + print(f"Warning: Could not parse package list: {e}") + except FileNotFoundError: + print("Warning: uv command not found") + except Exception as e: + print(f"Warning! Couldn't extract the list of installed Python packages: {e}") + + return result + + installed_packages = _get_installed_uv_packages() + packages_to_install = list(get_packages_to_install(python_deps, installed_packages)) + + if packages_to_install: + packages_list = [f"{p}{python_deps[p]}" for p in packages_to_install] + + cmd = [ + "uv", "pip", "install", + f"--python={env.subst('$PYTHONEXE')}", + "--quiet", "--upgrade" + ] + packages_list + + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=30 # 30 second timeout for package installation + ) + + if result.returncode != 0: + print(f"Error: Failed to install Python dependencies (exit code: {result.returncode})") + if result.stderr: + print(f"Error output: {result.stderr.strip()}") + return False + + except subprocess.TimeoutExpired: + print("Error: Python dependencies installation timed out") + return False + except FileNotFoundError: + print("Error: uv command not found") + return False + except Exception as e: + print(f"Error installing Python dependencies: {e}") + return False + + return True + + +def install_esptool(env): + """Install esptool from package folder "tool-esptoolpy" using uv package manager""" + try: + subprocess.check_call([env.subst("$PYTHONEXE"), "-c", "import esptool"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + return True + except (subprocess.CalledProcessError, FileNotFoundError): + pass + + esptool_repo_path = env.subst(platform.get_package_dir("tool-esptoolpy") or "") + if esptool_repo_path and os.path.isdir(esptool_repo_path): + try: + subprocess.check_call([ + "uv", "pip", "install", "--quiet", + f"--python={env.subst("$PYTHONEXE")}", + "-e", esptool_repo_path + ]) + return True + except subprocess.CalledProcessError as e: + print(f"Warning: Failed to install esptool: {e}") + return False + + return False + + +install_python_deps() +install_esptool(env) + def BeforeUpload(target, source, env): """ @@ -346,7 +509,7 @@ def check_lib_archive_exists(): "bin", "%s-elf-gdb" % toolchain_arch, ), - OBJCOPY=join(platform.get_package_dir("tool-esptoolpy") or "", "esptool.py"), + OBJCOPY='esptool', RANLIB="%s-elf-gcc-ranlib" % toolchain_arch, SIZETOOL="%s-elf-size" % toolchain_arch, ARFLAGS=["rc"], @@ -356,7 +519,7 @@ def check_lib_archive_exists(): SIZECHECKCMD="$SIZETOOL -A -d $SOURCES", SIZEPRINTCMD="$SIZETOOL -B -d $SOURCES", ERASEFLAGS=["--chip", mcu, "--port", '"$UPLOAD_PORT"'], - ERASECMD='"$PYTHONEXE" "$OBJCOPY" $ERASEFLAGS erase-flash', + ERASECMD='"$OBJCOPY" $ERASEFLAGS erase-flash', # mkspiffs package contains two different binaries for IDF and Arduino MKFSTOOL="mk%s" % filesystem + ( @@ -373,6 +536,7 @@ def check_lib_archive_exists(): ), # Legacy `ESP32_SPIFFS_IMAGE_NAME` is used as the second fallback value # for backward compatibility + ESP32_FS_IMAGE_NAME=env.get( "ESP32_FS_IMAGE_NAME", env.get("ESP32_SPIFFS_IMAGE_NAME", filesystem), @@ -401,7 +565,7 @@ def check_lib_archive_exists(): action=env.VerboseAction( " ".join( [ - '"$PYTHONEXE" "$OBJCOPY"', + "$OBJCOPY", "--chip", mcu, "elf2image", @@ -444,6 +608,7 @@ def check_lib_archive_exists(): if not env.get("PIOFRAMEWORK"): env.SConscript("frameworks/_bare.py", exports="env") + def firmware_metrics(target, source, env): """ Custom target to run esp-idf-size with support for command line parameters @@ -463,11 +628,7 @@ def firmware_metrics(target, source, env): print("Make sure the project is built first with 'pio run'") return - try: - import subprocess - import sys - import shlex - + try: cmd = [env.subst("$PYTHONEXE"), "-m", "esp_idf_size", "--ng"] # Parameters from platformio.ini @@ -510,6 +671,7 @@ def firmware_metrics(target, source, env): print(f"Error: Failed to run firmware metrics: {e}") print("Make sure esp-idf-size is installed: pip install esp-idf-size") + # # Target: Build executable and linkable firmware or FS image # @@ -604,9 +766,7 @@ def firmware_metrics(target, source, env): # Configure upload protocol: esptool elif upload_protocol == "esptool": env.Replace( - UPLOADER=join( - platform.get_package_dir("tool-esptoolpy") or "", "esptool.py" - ), + UPLOADER="esptool", UPLOADERFLAGS=[ "--chip", mcu, @@ -627,8 +787,7 @@ def firmware_metrics(target, source, env): "--flash-size", "detect", ], - UPLOADCMD='"$PYTHONEXE" "$UPLOADER" $UPLOADERFLAGS ' - "$ESP32_APP_OFFSET $SOURCE", + UPLOADCMD='$UPLOADER $UPLOADERFLAGS $ESP32_APP_OFFSET $SOURCE' ) for image in env.get("FLASH_EXTRA_IMAGES", []): env.Append(UPLOADERFLAGS=[image[0], env.subst(image[1])]) @@ -656,7 +815,7 @@ def firmware_metrics(target, source, env): "detect", "$FS_START", ], - UPLOADCMD='"$PYTHONEXE" "$UPLOADER" $UPLOADERFLAGS $SOURCE', + UPLOADCMD='"$UPLOADER" $UPLOADERFLAGS $SOURCE', ) upload_actions = [ From bc49ac55b1eab176470ca8d592479f437af4b96c Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 5 Jul 2025 17:38:41 +0200 Subject: [PATCH 2/6] Update arduino.py --- builder/frameworks/arduino.py | 63 +---------------------------------- 1 file changed, 1 insertion(+), 62 deletions(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index 9232548f6..ee4a1478f 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -22,8 +22,6 @@ http://arduino.cc/en/Reference/HomePage """ -import subprocess -import json import os import sys import shutil @@ -34,23 +32,12 @@ from pathlib import Path from typing import Union, List -import semantic_version from SCons.Script import DefaultEnvironment, SConscript from platformio import fs -from platformio.package.version import pepver_to_semver from platformio.package.manager.tool import ToolPackageManager IS_WINDOWS = sys.platform.startswith("win") -python_deps = { - "wheel": ">=0.35.1", - "rich-click": ">=1.8.6", - "PyYAML": ">=6.0.2", - "intelhex": ">=2.3.0", - "rich": ">=14.0.0", - "esp-idf-size": ">=1.6.1" -} - # Constants for better performance UNICORE_FLAGS = { "CORE32SOLO1", @@ -601,53 +588,6 @@ def has_unicore_flags(): env.Replace(BUILD_UNFLAGS=new_build_unflags) -def get_packages_to_install(deps, installed_packages): - """Generator for packages to install""" - for package, spec in deps.items(): - if package not in installed_packages: - yield package - else: - version_spec = semantic_version.Spec(spec) - if not version_spec.match(installed_packages[package]): - yield package - - -def install_python_deps(): - def _get_installed_pip_packages(): - result = {} - try: - pip_output = subprocess.check_output([ - env.subst("$PYTHONEXE"), - "-m", "pip", "list", "--format=json", - "--disable-pip-version-check" - ]) - packages = json.loads(pip_output) - for p in packages: - result[p["name"]] = pepver_to_semver(p["version"]) - except Exception: - print("Warning! Couldn't extract the list of installed Python " - "packages.") - - return result - - installed_packages = _get_installed_pip_packages() - packages_to_install = list(get_packages_to_install(python_deps, - installed_packages)) - - if packages_to_install: - packages_str = " ".join(f'"{p}{python_deps[p]}"' - for p in packages_to_install) - env.Execute( - env.VerboseAction( - f'"$PYTHONEXE" -m pip install -U -q -q -q {packages_str}', - "Installing Arduino Python dependencies", - ) - ) - - -install_python_deps() - - def get_MD5_hash(phrase): return hashlib.md5(phrase.encode('utf-8')).hexdigest()[:16] @@ -955,6 +895,7 @@ def get_frameworks_in_current_env(): if ("arduino" in pioframework and "espidf" not in pioframework and arduino_lib_compile_flag in ("Inactive", "True")): + # try to remove not needed include path if an lib_ignore entry exists from component_manager import ComponentManager component_manager = ComponentManager(env) @@ -965,8 +906,6 @@ def get_frameworks_in_current_env(): env.AddPostAction("checkprogsize", silent_action) if IS_WINDOWS: - # Smart include path optimization based on bleeding edge configurable - # threshold env.AddBuildMiddleware(smart_include_length_shorten) build_script_path = join(FRAMEWORK_DIR, "tools", "pioarduino-build.py") From c6ae68f56439dcbf37da2cef727fc54713816090 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 5 Jul 2025 17:39:25 +0200 Subject: [PATCH 3/6] Update espidf.py --- builder/frameworks/espidf.py | 106 +++++++---------------------------- 1 file changed, 19 insertions(+), 87 deletions(-) diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index 47583a32d..b665d1666 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -56,68 +56,6 @@ if os.path.exists(map_file): os.remove(map_file) -def install_standard_python_deps(): - def _get_installed_standard_pip_packages(): - result = {} - packages = {} - pip_output = subprocess.check_output( - [ - env.subst("$PYTHONEXE"), - "-m", - "pip", - "list", - "--format=json", - "--disable-pip-version-check", - ] - ) - try: - packages = json.loads(pip_output) - except: - print("Warning! Couldn't extract the list of installed Python packages.") - return {} - for p in packages: - result[p["name"]] = pepver_to_semver(p["version"]) - - return result - - deps = { - "wheel": ">=0.35.1", - "rich-click": ">=1.8.6", - "PyYAML": ">=6.0.2", - "intelhex": ">=2.3.0", - "rich": ">=14.0.0", - "esp-idf-size": ">=1.6.1" - } - - installed_packages = _get_installed_standard_pip_packages() - packages_to_install = [] - for package, spec in deps.items(): - if package not in installed_packages: - packages_to_install.append(package) - else: - version_spec = semantic_version.Spec(spec) - if not version_spec.match(installed_packages[package]): - packages_to_install.append(package) - - if packages_to_install: - env.Execute( - env.VerboseAction( - ( - '"$PYTHONEXE" -m pip install -U -q -q -q ' - + " ".join( - [ - '"%s%s"' % (p, deps[p]) - for p in packages_to_install - ] - ) - ), - "Installing standard Python dependencies", - ) - ) - return - -install_standard_python_deps() - # Allow changes in folders of managed components os.environ["IDF_COMPONENT_OVERWRITE_MANAGED_COMPONENTS"] = "1" @@ -173,7 +111,7 @@ def create_silent_action(action_func): os.rename(ARDUINO_FRAMEWORK_DIR, new_path) ARDUINO_FRAMEWORK_DIR = new_path assert ARDUINO_FRAMEWORK_DIR and os.path.isdir(ARDUINO_FRAMEWORK_DIR) - arduino_libs_mcu = join(platform.get_package_dir("framework-arduinoespressif32-libs"),mcu) + arduino_libs_mcu = join(platform.get_package_dir("framework-arduinoespressif32-libs"), mcu) BUILD_DIR = env.subst("$BUILD_DIR") PROJECT_DIR = env.subst("$PROJECT_DIR") @@ -1548,24 +1486,17 @@ def generate_mbedtls_bundle(sdk_config): def install_python_deps(): - def _get_installed_pip_packages(python_exe_path): + def _get_installed_uv_packages(python_exe_path): result = {} - packages = {} - pip_output = subprocess.check_output( - [ - python_exe_path, - "-m", - "pip", - "list", - "--format=json", - "--disable-pip-version-check", - ] - ) try: - packages = json.loads(pip_output) - except: - print("Warning! Couldn't extract the list of installed Python packages.") + uv_output = subprocess.check_output([ + "uv", "pip", "list", "--python", python_exe_path, "--format=json" + ]) + packages = json.loads(uv_output) + except (subprocess.CalledProcessError, json.JSONDecodeError, OSError) as e: + print(f"Warning! Couldn't extract the list of installed Python packages: {e}") return {} + for p in packages: result[p["name"]] = pepver_to_semver(p["version"]) @@ -1576,7 +1507,7 @@ def _get_installed_pip_packages(python_exe_path): return deps = { - "wheel": ">=0.35.1", + "uv": ">=0.1.0", # https://github.com/platformio/platformio-core/issues/4614 "urllib3": "<2", # https://github.com/platformio/platform-espressif32/issues/635 @@ -1590,7 +1521,7 @@ def _get_installed_pip_packages(python_exe_path): deps["chardet"] = ">=3.0.2,<4" python_exe_path = get_python_exe() - installed_packages = _get_installed_pip_packages(python_exe_path) + installed_packages = _get_installed_uv_packages(python_exe_path) packages_to_install = [] for package, spec in deps.items(): if package not in installed_packages: @@ -1601,21 +1532,22 @@ def _get_installed_pip_packages(python_exe_path): packages_to_install.append(package) if packages_to_install: + packages_str = " ".join(['"%s%s"' % (p, deps[p]) for p in packages_to_install]) + + # Use uv to install packages in the specific Python environment env.Execute( env.VerboseAction( - ( - '"%s" -m pip install -U -q -q -q ' % python_exe_path - + " ".join(['"%s%s"' % (p, deps[p]) for p in packages_to_install]) - ), - "Installing ESP-IDF's Python dependencies", + f'uv pip install --python "{python_exe_path}" {packages_str}', + "Installing ESP-IDF's Python dependencies with uv", ) ) if IS_WINDOWS and "windows-curses" not in installed_packages: + # Install windows-curses in the IDF Python environment env.Execute( env.VerboseAction( - '"%s" -m pip install -q -q -q windows-curses' % python_exe_path, - "Installing windows-curses package", + f'uv pip install --python "{python_exe_path}" windows-curses', + "Installing windows-curses package with uv", ) ) From 6c45be02248f5283dbe833412e8043eee77ed879 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sun, 6 Jul 2025 16:21:33 +0200 Subject: [PATCH 4/6] PR #213 --- platform.py | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/platform.py b/platform.py index fd096ff2b..a059597f6 100644 --- a/platform.py +++ b/platform.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import fnmatch import os import contextlib import json @@ -117,6 +118,20 @@ def safe_remove_directory(path: str) -> bool: return True +@safe_file_operation +def safe_remove_directory_pattern(base_path: str, pattern: str) -> bool: + """Safely remove directories matching a pattern with error handling.""" + if not os.path.exists(base_path): + return True + # Find all directories matching the pattern in the base directory + for item in os.listdir(base_path): + item_path = os.path.join(base_path, item) + if os.path.isdir(item_path) and fnmatch.fnmatch(item, pattern): + shutil.rmtree(item_path) + logger.debug(f"Directory removed: {item_path}") + return True + + @safe_file_operation def safe_copy_file(src: str, dst: str) -> bool: """Safely copy files with error handling.""" @@ -148,6 +163,17 @@ def _get_tool_paths(self, tool_name: str) -> Dict[str, str]: """Get centralized path calculation for tools with caching.""" if tool_name not in self._tools_cache: tool_path = os.path.join(self.packages_dir, tool_name) + # Remove all directories containing '@' in their name + try: + for item in os.listdir(self.packages_dir): + if '@' in item and item.startswith(tool_name): + item_path = os.path.join(self.packages_dir, item) + if os.path.isdir(item_path): + safe_remove_directory(item_path) + logger.debug(f"Removed directory with '@' in name: {item_path}") + except OSError as e: + logger.error(f"Error scanning packages directory for '@' directories: {e}") + self._tools_cache[tool_name] = { 'tool_path': tool_path, 'package_path': os.path.join(tool_path, "package.json"), @@ -297,9 +323,19 @@ def _handle_existing_tool( logger.debug(f"Tool {tool_name} found with correct version") return True - # Wrong version, reinstall + # Wrong version, reinstall - remove similar paths too logger.info(f"Reinstalling {tool_name} due to version mismatch") + + tool_base_name = os.path.basename(paths['tool_path']) + packages_dir = os.path.dirname(paths['tool_path']) + + # Remove similar directories with version suffixes FIRST (e.g., xtensa@src, xtensa.12232) + safe_remove_directory_pattern(packages_dir, f"{tool_base_name}@*") + safe_remove_directory_pattern(packages_dir, f"{tool_base_name}.*") + + # Then remove the main tool directory (if it still exists) safe_remove_directory(paths['tool_path']) + return self.install_tool(tool_name, retry_count + 1) def _configure_arduino_framework(self, frameworks: List[str]) -> None: From 0d28781d1eebd5fed673b06042a9b99ae8050871 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Mon, 7 Jul 2025 20:01:16 +0200 Subject: [PATCH 5/6] fix syntax --- builder/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/main.py b/builder/main.py index 9ed31aa84..fb39bf24b 100644 --- a/builder/main.py +++ b/builder/main.py @@ -190,7 +190,7 @@ def install_esptool(env): try: subprocess.check_call([ "uv", "pip", "install", "--quiet", - f"--python={env.subst("$PYTHONEXE")}", + f"--python={env.subst('$PYTHONEXE')}", "-e", esptool_repo_path ]) return True From 1873818e673248e44140539e346e44bba828227d Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Tue, 8 Jul 2025 11:11:43 +0200 Subject: [PATCH 6/6] fix map path for windows --- builder/frameworks/espidf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index b665d1666..65ade59a5 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -1785,7 +1785,7 @@ def get_python_exe(): # This will add the linker flag for the map file extra_cmake_args.append( - f'-DCMAKE_EXE_LINKER_FLAGS=-Wl,-Map={os.path.join(BUILD_DIR, env.subst("$PROGNAME") + ".map")}' + f'-DCMAKE_EXE_LINKER_FLAGS=-Wl,-Map={fs.to_unix_path(os.path.join(BUILD_DIR, env.subst("$PROGNAME") + ".map"))}' ) # Add any extra args from board config