use uv as package manager for resolving python lib dependencies (#212)

Author: Jason2866

builder/frameworks/arduino.py

@@ -22,8 +22,6 @@
 http://arduino.cc/en/Reference/HomePage
 """
 
-import subprocess
-import json
 import os
 import sys
 import shutil
@@ -34,23 +32,12 @@
 from pathlib import Path
 from typing import Union, List
 
-import semantic_version
 from SCons.Script import DefaultEnvironment, SConscript
 from platformio import fs
-from platformio.package.version import pepver_to_semver
 from platformio.package.manager.tool import ToolPackageManager
 
 IS_WINDOWS = sys.platform.startswith("win")
 
-python_deps = {
-    "wheel": ">=0.35.1",
-    "rich-click": ">=1.8.6",
-    "PyYAML": ">=6.0.2",
-    "intelhex": ">=2.3.0",
-    "rich": ">=14.0.0",
-    "esp-idf-size": ">=1.6.1"
-}
-
 # Constants for better performance
 UNICORE_FLAGS = {
     "CORE32SOLO1",
@@ -601,53 +588,6 @@ def has_unicore_flags():
     env.Replace(BUILD_UNFLAGS=new_build_unflags)
 
 
-def get_packages_to_install(deps, installed_packages):
-    """Generator for packages to install"""
-    for package, spec in deps.items():
-        if package not in installed_packages:
-            yield package
-        else:
-            version_spec = semantic_version.Spec(spec)
-            if not version_spec.match(installed_packages[package]):
-                yield package
-
-
-def install_python_deps():
-    def _get_installed_pip_packages():
-        result = {}
-        try:
-            pip_output = subprocess.check_output([
-                env.subst("$PYTHONEXE"),
-                "-m", "pip", "list", "--format=json",
-                "--disable-pip-version-check"
-            ])
-            packages = json.loads(pip_output)
-            for p in packages:
-                result[p["name"]] = pepver_to_semver(p["version"])
-        except Exception:
-            print("Warning! Couldn't extract the list of installed Python "
-                  "packages.")
-
-        return result
-
-    installed_packages = _get_installed_pip_packages()
-    packages_to_install = list(get_packages_to_install(python_deps,
-                                                       installed_packages))
-
-    if packages_to_install:
-        packages_str = " ".join(f'"{p}{python_deps[p]}"'
-                                for p in packages_to_install)
-        env.Execute(
-            env.VerboseAction(
-                f'"$PYTHONEXE" -m pip install -U -q -q -q {packages_str}',
-                "Installing Arduino Python dependencies",
-            )
-        )
-
-
-install_python_deps()
-
-
 def get_MD5_hash(phrase):
     return hashlib.md5(phrase.encode('utf-8')).hexdigest()[:16]
 
@@ -955,6 +895,7 @@ def get_frameworks_in_current_env():
 
 if ("arduino" in pioframework and "espidf" not in pioframework and
         arduino_lib_compile_flag in ("Inactive", "True")):
+
     # try to remove not needed include path if an lib_ignore entry exists
     from component_manager import ComponentManager
     component_manager = ComponentManager(env)
@@ -965,8 +906,6 @@ def get_frameworks_in_current_env():
     env.AddPostAction("checkprogsize", silent_action)
 
     if IS_WINDOWS:
-        # Smart include path optimization based on bleeding edge configurable 
-        # threshold
         env.AddBuildMiddleware(smart_include_length_shorten)
 
     build_script_path = join(FRAMEWORK_DIR, "tools", "pioarduino-build.py")

builder/frameworks/espidf.py

@@ -56,68 +56,6 @@
 if os.path.exists(map_file):
     os.remove(map_file)
 
-def install_standard_python_deps():
-    def _get_installed_standard_pip_packages():
-        result = {}
-        packages = {}
-        pip_output = subprocess.check_output(
-            [
-                env.subst("$PYTHONEXE"),
-                "-m",
-                "pip",
-                "list",
-                "--format=json",
-                "--disable-pip-version-check",
-            ]
-        )
-        try:
-            packages = json.loads(pip_output)
-        except:
-            print("Warning! Couldn't extract the list of installed Python packages.")
-            return {}
-        for p in packages:
-            result[p["name"]] = pepver_to_semver(p["version"])
-
-        return result
-
-    deps = {
-        "wheel": ">=0.35.1",
-        "rich-click": ">=1.8.6",
-        "PyYAML": ">=6.0.2",
-        "intelhex": ">=2.3.0",
-        "rich": ">=14.0.0",
-        "esp-idf-size": ">=1.6.1"
-    }
-
-    installed_packages = _get_installed_standard_pip_packages()
-    packages_to_install = []
-    for package, spec in deps.items():
-        if package not in installed_packages:
-            packages_to_install.append(package)
-        else:
-            version_spec = semantic_version.Spec(spec)
-            if not version_spec.match(installed_packages[package]):
-                packages_to_install.append(package)
-
-    if packages_to_install:
-        env.Execute(
-            env.VerboseAction(
-                (
-                    '"$PYTHONEXE" -m pip install -U -q -q -q '
-                    + " ".join(
-                        [
-                            '"%s%s"' % (p, deps[p])
-                            for p in packages_to_install
-                        ]
-                    )
-                ),
-                "Installing standard Python dependencies",
-            )
-        )
-    return
-
-install_standard_python_deps()
-
 # Allow changes in folders of managed components
 os.environ["IDF_COMPONENT_OVERWRITE_MANAGED_COMPONENTS"] = "1"
 
@@ -173,7 +111,7 @@ def create_silent_action(action_func):
         os.rename(ARDUINO_FRAMEWORK_DIR, new_path)
         ARDUINO_FRAMEWORK_DIR = new_path
     assert ARDUINO_FRAMEWORK_DIR and os.path.isdir(ARDUINO_FRAMEWORK_DIR)
-    arduino_libs_mcu = join(platform.get_package_dir("framework-arduinoespressif32-libs"),mcu)
+    arduino_libs_mcu = join(platform.get_package_dir("framework-arduinoespressif32-libs"), mcu)
 
 BUILD_DIR = env.subst("$BUILD_DIR")
 PROJECT_DIR = env.subst("$PROJECT_DIR")
@@ -1548,24 +1486,17 @@ def generate_mbedtls_bundle(sdk_config):
 
 
 def install_python_deps():
-    def _get_installed_pip_packages(python_exe_path):
+    def _get_installed_uv_packages(python_exe_path):
         result = {}
-        packages = {}
-        pip_output = subprocess.check_output(
-            [
-                python_exe_path,
-                "-m",
-                "pip",
-                "list",
-                "--format=json",
-                "--disable-pip-version-check",
-            ]
-        )
         try:
-            packages = json.loads(pip_output)
-        except:
-            print("Warning! Couldn't extract the list of installed Python packages.")
+            uv_output = subprocess.check_output([
+                "uv", "pip", "list", "--python", python_exe_path, "--format=json"
+            ])
+            packages = json.loads(uv_output)
+        except (subprocess.CalledProcessError, json.JSONDecodeError, OSError) as e:
+            print(f"Warning! Couldn't extract the list of installed Python packages: {e}")
             return {}
+        
         for p in packages:
             result[p["name"]] = pepver_to_semver(p["version"])
 
@@ -1576,7 +1507,7 @@ def _get_installed_pip_packages(python_exe_path):
         return
 
     deps = {
-        "wheel": ">=0.35.1",
+        "uv": ">=0.1.0",
         # https://github.com/platformio/platformio-core/issues/4614
         "urllib3": "<2",
         # https://github.com/platformio/platform-espressif32/issues/635
@@ -1590,7 +1521,7 @@ def _get_installed_pip_packages(python_exe_path):
         deps["chardet"] = ">=3.0.2,<4"
 
     python_exe_path = get_python_exe()
-    installed_packages = _get_installed_pip_packages(python_exe_path)
+    installed_packages = _get_installed_uv_packages(python_exe_path)
     packages_to_install = []
     for package, spec in deps.items():
         if package not in installed_packages:
@@ -1601,21 +1532,22 @@ def _get_installed_pip_packages(python_exe_path):
                 packages_to_install.append(package)
 
     if packages_to_install:
+        packages_str = " ".join(['"%s%s"' % (p, deps[p]) for p in packages_to_install])
+        
+        # Use uv to install packages in the specific Python environment
         env.Execute(
             env.VerboseAction(
-                (
-                    '"%s" -m pip install -U -q -q -q ' % python_exe_path
-                    + " ".join(['"%s%s"' % (p, deps[p]) for p in packages_to_install])
-                ),
-                "Installing ESP-IDF's Python dependencies",
+                f'uv pip install --python "{python_exe_path}" {packages_str}',
+                "Installing ESP-IDF's Python dependencies with uv",
             )
         )
 
     if IS_WINDOWS and "windows-curses" not in installed_packages:
+        # Install windows-curses in the IDF Python environment
         env.Execute(
             env.VerboseAction(
-                '"%s" -m pip install -q -q -q windows-curses' % python_exe_path,
-                "Installing windows-curses package",
+                f'uv pip install --python "{python_exe_path}" windows-curses',
+                "Installing windows-curses package with uv",
             )
         )