Better handling of ESP IDF Python dependencies

#1006, #1007, resolves #1013

Now IDF Python deps are installed in a pre-сreated virtual environment.
The name of the IDF venv contains the IDF version to avoid possible conflicts and
unnecessary reinstallation of Python dependencies in cases when Arduino
as an IDF component requires a different version of the IDF package and
hence a different set of Python deps or their versions

Author: valeros

builder/frameworks/espidf.py

@@ -37,11 +37,17 @@
 )
 
 from platformio import fs
+from platformio.compat import IS_WINDOWS
 from platformio.proc import exec_command
-from platformio.util import get_systype
 from platformio.builder.tools.piolib import ProjectAsLibBuilder
 from platformio.package.version import get_original_version, pepver_to_semver
 
+# Added to avoid conflicts between installed Python packages from
+# the IDF virtual environment and PlatformIO Core
+# Note: This workaround can be safely deleted when PlatformIO 6.1.7 is released
+if os.environ.get("PYTHONPATH"):
+    del os.environ["PYTHONPATH"]
+
 env = DefaultEnvironment()
 env.SConscript("_embed_files.py", exports="env")
 
@@ -51,8 +57,7 @@
 idf_variant = mcu.lower()
 
 # Required until Arduino switches to v5
-IDF5 = platform.get_package_version(
-    "framework-espidf").split(".")[1].startswith("5")
+IDF5 = platform.get_package_version("framework-espidf").split(".")[1].startswith("5")
 FRAMEWORK_DIR = platform.get_package_dir("framework-espidf")
 TOOLCHAIN_DIR = platform.get_package_dir(
     "toolchain-%s" % ("riscv32-esp" if mcu == "esp32c3" else ("xtensa-%s" % mcu))
@@ -224,15 +229,15 @@ def populate_idf_env_vars(idf_env):
         os.path.join(TOOLCHAIN_DIR, "bin"),
         platform.get_package_dir("tool-ninja"),
         os.path.join(platform.get_package_dir("tool-cmake"), "bin"),
-        os.path.dirname(env.subst("$PYTHONEXE")),
+        os.path.dirname(get_python_exe()),
     ]
 
     if mcu != "esp32c3":
         additional_packages.append(
             os.path.join(platform.get_package_dir("toolchain-esp32ulp"), "bin"),
         )
 
-    if "windows" in get_systype():
+    if IS_WINDOWS:
         additional_packages.append(platform.get_package_dir("tool-mconf"))
 
     idf_env["PATH"] = os.pathsep.join(additional_packages + [idf_env["PATH"]])
@@ -559,7 +564,7 @@ def generate_project_ld_script(sdk_config, ignore_targets=None):
     }
 
     cmd = (
-        '"$PYTHONEXE" "{script}" --input $SOURCE '
+        '"$ESPIDF_PYTHONEXE" "{script}" --input $SOURCE '
         '--config "{config}" --fragments {fragments} --output $TARGET '
         '--kconfig "{kconfig}" --env-file "{env_file}" '
         '--libraries-file "{libraries_list}" '
@@ -763,7 +768,7 @@ def build_bootloader(sdk_config):
         [
             "-DIDF_TARGET=" + idf_variant,
             "-DPYTHON_DEPS_CHECKED=1",
-            "-DPYTHON=" + env.subst("$PYTHONEXE"),
+            "-DPYTHON=" + get_python_exe(),
             "-DIDF_PATH=" + FRAMEWORK_DIR,
             "-DSDKCONFIG=" + SDKCONFIG_PATH,
             "-DLEGACY_INCLUDE_COMMON_HEADERS=",
@@ -918,7 +923,7 @@ def generate_empty_partition_image(binary_path, image_size):
         binary_path,
         None,
         env.VerboseAction(
-            '"$PYTHONEXE" "%s" %s $TARGET'
+            '"$ESPIDF_PYTHONEXE" "%s" %s $TARGET'
             % (
                 os.path.join(
                     FRAMEWORK_DIR,
@@ -943,7 +948,7 @@ def get_partition_info(pt_path, pt_offset, pt_params):
         env.Exit(1)
 
     cmd = [
-        env.subst("$PYTHONEXE"),
+        get_python_exe(),
         os.path.join(FRAMEWORK_DIR, "components", "partition_table", "parttool.py"),
         "-q",
         "--partition-table-offset",
@@ -1000,7 +1005,7 @@ def generate_mbedtls_bundle(sdk_config):
         FRAMEWORK_DIR, "components", "mbedtls", "esp_crt_bundle"
     )
 
-    cmd = [env.subst("$PYTHONEXE"), os.path.join(default_crt_dir, "gen_crt_bundle.py")]
+    cmd = [get_python_exe(), os.path.join(default_crt_dir, "gen_crt_bundle.py")]
 
     crt_args = ["--input"]
     if sdk_config.get("MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_FULL", False):
@@ -1051,12 +1056,12 @@ def generate_mbedtls_bundle(sdk_config):
 
 
 def install_python_deps():
-    def _get_installed_pip_packages():
+    def _get_installed_pip_packages(python_exe_path):
         result = {}
         packages = {}
         pip_output = subprocess.check_output(
             [
-                env.subst("$PYTHONEXE"),
+                python_exe_path,
                 "-m",
                 "pip",
                 "list",
@@ -1087,7 +1092,8 @@ def _get_installed_pip_packages():
         # Remove specific versions for IDF5 as not required
         deps = {dep: "" for dep in deps}
 
-    installed_packages = _get_installed_pip_packages()
+    python_exe_path = get_python_exe()
+    installed_packages = _get_installed_pip_packages(python_exe_path)
     packages_to_install = []
     for package, spec in deps.items():
         if package not in installed_packages:
@@ -1101,22 +1107,17 @@ def _get_installed_pip_packages():
         env.Execute(
             env.VerboseAction(
                 (
-                    '"$PYTHONEXE" -m pip install -U '
-                    + " ".join(
-                        [
-                            '"%s%s"' % (p, deps[p])
-                            for p in packages_to_install
-                        ]
-                    )
+                    '"%s" -m pip install -U ' % python_exe_path
+                    + " ".join(['"%s%s"' % (p, deps[p]) for p in packages_to_install])
                 ),
                 "Installing ESP-IDF's Python dependencies",
             )
         )
 
-    if "windows" in get_systype() and "windows-curses" not in installed_packages:
+    if IS_WINDOWS and "windows-curses" not in installed_packages:
         env.Execute(
             env.VerboseAction(
-                "$PYTHONEXE -m pip install windows-curses",
+                '"%s" -m pip install windows-curses' % python_exe_path,
                 "Installing windows-curses package",
             )
         )
@@ -1128,15 +1129,59 @@ def _get_installed_pip_packages():
         }:
             env.Execute(
                 env.VerboseAction(
-                    '$PYTHONEXE -m pip install "file://%s/tools/kconfig_new/esp-windows-curses"'
-                    % FRAMEWORK_DIR,
+                    '"%s" -m pip install "file://%s/tools/kconfig_new/esp-windows-curses"'
+                    % (python_exe_path, FRAMEWORK_DIR),
                     "Installing windows-curses package",
                 )
             )
 
 
+def get_python_exe():
+    def _create_venv(venv_dir):
+        pip_path = os.path.join(
+            venv_dir,
+            "Scripts" if IS_WINDOWS else "bin",
+            "pip" + (".exe" if IS_WINDOWS else ""),
+        )
+        if not os.path.isfile(pip_path):
+            # Use the built-in PlatformIO Python to create a standalone IDF virtual env
+            env.Execute(
+                env.VerboseAction(
+                    '"$PYTHONEXE" -m venv --clear "%s"' % venv_dir,
+                    "Creating a virtual environment for IDF Python dependencies",
+                )
+            )
+
+        assert os.path.isfile(
+            pip_path
+        ), "Error: Failed to create a proper virtual environment. Missing the pip binary!"
+
+    # The name of the IDF venv contains the IDF version to avoid possible conflicts and
+    # unnecessary reinstallation of Python dependencies in cases when Arduino
+    # as an IDF component requires a different version of the IDF package and
+    # hence a different set of Python deps or their versions
+    idf_version = get_original_version(platform.get_package_version("framework-espidf"))
+    venv_dir = os.path.join(
+        env.subst("$PROJECT_CORE_DIR"), "penv", ".espidf-" + idf_version)
+
+    if not os.path.isdir(venv_dir):
+        _create_venv(venv_dir)
+
+    python_exe_path = os.path.join(
+        venv_dir,
+        "Scripts" if IS_WINDOWS else "bin",
+        "python" + (".exe" if IS_WINDOWS else ""),
+    )
+
+    assert os.path.isfile(python_exe_path), (
+        "Error: Missing Python executable file `%s`" % python_exe_path
+    )
+
+    return python_exe_path
+
+
 #
-# ESP-IDF requires Python packages with specific versions
+# ESP-IDF requires Python packages with specific versions in a virtual environment
 #
 
 install_python_deps()
@@ -1235,7 +1280,7 @@ def _get_installed_pip_packages():
         "-DIDF_TARGET=" + idf_variant,
         "-DPYTHON_DEPS_CHECKED=1",
         "-DEXTRA_COMPONENT_DIRS:PATH=" + ";".join(extra_components),
-        "-DPYTHON=" + env.subst("$PYTHONEXE"),
+        "-DPYTHON=" + get_python_exe(),
         "-DSDKCONFIG=" + SDKCONFIG_PATH,
     ]
     + click.parser.split_arg_string(board.get("build.cmake_extra_args", "")),
@@ -1373,7 +1418,7 @@ def _skip_prj_source_files(node):
     os.path.join("$BUILD_DIR", "partitions.bin"),
     "$PARTITIONS_TABLE_CSV",
     env.VerboseAction(
-        '"$PYTHONEXE" "%s" -q --offset "%s" --flash-size "%s" $SOURCE $TARGET'
+        '"$ESPIDF_PYTHONEXE" "%s" -q --offset "%s" --flash-size "%s" $SOURCE $TARGET'
         % (
             os.path.join(
                 FRAMEWORK_DIR, "components", "partition_table", "gen_esp32part.py"
@@ -1396,6 +1441,7 @@ def _skip_prj_source_files(node):
 env.Prepend(
     CPPPATH=app_includes["plain_includes"],
     CPPDEFINES=project_defines,
+    ESPIDF_PYTHONEXE=get_python_exe(),
     LINKFLAGS=extra_flags,
     LIBS=libs,
     FLASH_EXTRA_IMAGES=[