nots/*: Стабилизация trunk

Суть изменений:
1. Наполнение папки ~/.nots/nm_store/$MODDIR/node_modules прямо в builder (перед установкой модулей в $BINDIR)
2. Это все равно нужно было бы делать в nots/cli, а теперь оно наполняется заранее, что обеспечивает структуру node_modules выше по уровню, чем virtual-store (важно для кастомных резолверов пакетов, ищущих пакеты выше по иерархии)
3. Убран предварительный запуск `pnpm install`, пакеты качаются один раз (тарболы в ya make)
4. После `ya make` все равно запускается `pnpm install` для гарантии (там еще всякие действия делаются)

# Всем, кого робот призовет в PR
## Пожалуйста, игнорируйте, это массовая проверка, в транк не попадут изменения в ваших проектах
## Нам нужно лишь стриггерить CI, чтобы проверить, что никто не сломался. Правки в pnpm-lock.yaml файлах будет откачены.
commit_hash:d124ecb77bda10bd8e975078382c685ac35c8928

Author: khoden

build/plugins/lib/nots/package_manager/base/package_json.py

@@ -1,11 +1,12 @@
 import json
-import logging
 import os
 
 from six import iteritems
 
+import logging
 from .utils import build_pj_path
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -247,3 +248,6 @@ def covered(k: str) -> bool:
             messages.extend([f"  - {key}" for key in missing_overrides])
 
         return (not messages, messages)
+
+    def get_pnpm_patched_dependencies(self) -> dict[str, str]:
+        return self.data.get("pnpm", {}).get("patchedDependencies", {})

build/plugins/lib/nots/package_manager/base/package_manager.py

@@ -1,13 +1,13 @@
 import os
-import sys
 import subprocess
-
+import sys
 from abc import ABCMeta, abstractmethod
 from six import add_metaclass
 
 from .constants import NPM_REGISTRY_URL
 from .package_json import PackageJson
 from .utils import build_nm_path, build_pj_path
+from .timeit import timeit
 
 
 class PackageManagerError(RuntimeError):
@@ -126,7 +126,8 @@ def get_peers_from_package_json(self):
 
         return [p[prefix_len:] for p in pj.get_workspace_map(ignore_self=True).keys()]
 
-    def _exec_command(self, args, include_defaults=True, script_path=None, env=None):
+    @timeit
+    def _exec_command(self, args, cwd: str, include_defaults=True, script_path=None, env=None):
         if not self.nodejs_bin_path:
             raise PackageManagerError("Unable to execute command: nodejs_bin_path is not configured")
 
@@ -135,9 +136,7 @@ def _exec_command(self, args, include_defaults=True, script_path=None, env=None)
             + args
             + (self._get_default_options() if include_defaults else [])
         )
-        p = subprocess.Popen(
-            cmd, cwd=self.build_path, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env
-        )
+        p = subprocess.Popen(cmd, cwd=cwd, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
         stdout, stderr = p.communicate()
 
         if p.returncode != 0:

build/plugins/lib/nots/package_manager/npm/npm_package_manager.py

@@ -1,6 +1,5 @@
 import os
 
-
 from ..base import BasePackageManager, PackageManagerError
 from ..base.constants import NODE_MODULES_WORKSPACE_BUNDLE_FILENAME
 from ..base.node_modules_bundler import bundle_node_modules
@@ -157,7 +156,7 @@ def _install_node_modules(self):
         env = os.environ.copy()
         env.update({"NPM_CONFIG_CACHE": os.path.join(self.build_path, ".npm-cache")})
 
-        self._exec_command(install_cmd, env=env)
+        self._exec_command(install_cmd, cwd=self.build_path, env=env)
 
     def _prepare_workspace(self):
         lf = self.load_lockfile(build_pre_lockfile_path(self.build_path))

build/plugins/lib/nots/package_manager/pnpm/package_manager.py

@@ -5,8 +5,9 @@
 from .utils import build_lockfile_path, build_pre_lockfile_path, build_ws_config_path
 from .workspace import PnpmWorkspace
 from ..base import BasePackageManager, PackageManagerError
-from ..base.constants import NODE_MODULES_WORKSPACE_BUNDLE_FILENAME
+from ..base.constants import NODE_MODULES_WORKSPACE_BUNDLE_FILENAME, PACKAGE_JSON_FILENAME, PNPM_LOCKFILE_FILENAME
 from ..base.node_modules_bundler import bundle_node_modules
+from ..base.package_json import PackageJson
 from ..base.timeit import timeit
 from ..base.utils import (
     b_rooted,
@@ -46,6 +47,40 @@ def load_lockfile_from_dir(cls, dir_path):
     def get_local_pnpm_store():
         return os.path.join(home_dir(), ".cache", "pnpm-store")
 
+    @timeit
+    def _create_local_node_modules(self, nm_store_path: str, store_dir: str, virtual_store_dir: str):
+        """
+        Creates ~/.nots/nm_store/$MODDIR/node_modules folder (with installed packages and .pnpm/virtual-store)
+
+        Should be used after build for local development ($SOURCE_DIR/node_modules should be a symlink to this folder).
+
+        But now it is also a workaround to provide valid node_modules structure in the parent folder of virtual-store
+        It is needed for fixing custom module resolvers (like in tsc, webpack, etc...), which are trying to find modules in the parents directories
+        """
+
+        # provide files required for `pnpm install`
+        pj = PackageJson.load(os.path.join(self.build_path, PACKAGE_JSON_FILENAME))
+        required_files = [
+            PACKAGE_JSON_FILENAME,
+            PNPM_LOCKFILE_FILENAME,
+            *list(pj.bins_iter()),
+            *pj.get_pnpm_patched_dependencies().values(),
+        ]
+
+        for f in required_files:
+            src = os.path.join(self.build_path, f)
+            if os.path.exists(src):
+                dst = os.path.join(nm_store_path, f)
+                try:
+                    os.remove(dst)
+                except FileNotFoundError:
+                    pass
+
+                os.makedirs(os.path.dirname(dst), exist_ok=True)
+                shutil.copy(src, dst)
+
+        self._run_pnpm_install(store_dir, virtual_store_dir, nm_store_path)
+
     @timeit
     def create_node_modules(self, yatool_prebuilder_path=None, local_cli=False, bundle=True):
         """
@@ -68,7 +103,10 @@ def create_node_modules(self, yatool_prebuilder_path=None, local_cli=False, bund
             # Use single virtual-store location in ~/.nots/nm_store/$MODDIR/node_modules/.pnpm/virtual-store
             virtual_store_dir = os.path.join(build_nm_path(nm_store_path), self._VSTORE_NM_PATH)
 
-        self._run_pnpm_install(store_dir, virtual_store_dir)
+            self._create_local_node_modules(nm_store_path, store_dir, virtual_store_dir)
+
+        self._run_pnpm_install(store_dir, virtual_store_dir, self.build_path)
+
         self._run_apply_addons_if_need(yatool_prebuilder_path, virtual_store_dir)
         self._replace_internal_lockfile_with_original(virtual_store_dir)
 
@@ -81,7 +119,7 @@ def create_node_modules(self, yatool_prebuilder_path=None, local_cli=False, bund
             )
 
     @timeit
-    def _run_pnpm_install(self, store_dir: str, virtual_store_dir: str):
+    def _run_pnpm_install(self, store_dir: str, virtual_store_dir: str, cwd: str):
         install_cmd = [
             "install",
             "--frozen-lockfile",
@@ -100,8 +138,9 @@ def _run_pnpm_install(self, store_dir: str, virtual_store_dir: str):
             virtual_store_dir,
         ]
 
-        self._exec_command(install_cmd)
+        self._exec_command(install_cmd, cwd=cwd)
 
+    @timeit
     def calc_prepare_deps_inouts_and_resources(
         self, store_path: str, has_deps: bool
     ) -> tuple[list[str], list[str], list[str]]:
@@ -122,6 +161,7 @@ def calc_prepare_deps_inouts_and_resources(
 
         return ins, outs, resources
 
+    @timeit
     def calc_node_modules_inouts(self, local_cli: bool, has_deps: bool) -> tuple[list[str], list[str]]:
         """
         Returns input and output paths for command that creates `node_modules` bundle.
@@ -139,6 +179,7 @@ def calc_node_modules_inouts(self, local_cli: bool, has_deps: bool) -> tuple[lis
 
         return ins, outs
 
+    @timeit
     def extract_packages_meta_from_lockfiles(self, lf_paths):
         """
         :type lf_paths: iterable of BaseLockfile
@@ -167,6 +208,7 @@ def _prepare_workspace(self):
 
         return PnpmWorkspace.load(build_ws_config_path(self.build_path))
 
+    @timeit
     def build_workspace(self, tarballs_store: str):
         """
         :rtype: PnpmWorkspace
@@ -182,6 +224,7 @@ def build_workspace(self, tarballs_store: str):
 
         return ws
 
+    @timeit
     def _build_merged_pre_lockfile(self, tarballs_store, dep_paths):
         """
         :type dep_paths: list of str
@@ -199,6 +242,7 @@ def _build_merged_pre_lockfile(self, tarballs_store, dep_paths):
 
         lf.write()
 
+    @timeit
     def _build_merged_workspace_config(self, ws, dep_paths):
         """
         NOTE: This method mutates `ws`.
@@ -223,10 +267,12 @@ def _run_apply_addons_if_need(self, yatool_prebuilder_path, virtual_store_dir):
                 "--virtual-store",
                 virtual_store_dir,
             ],
+            cwd=self.build_path,
             include_defaults=False,
             script_path=os.path.join(yatool_prebuilder_path, "build", "bin", "prebuilder.js"),
         )
 
+    @timeit
     def _replace_internal_lockfile_with_original(self, virtual_store_dir):
         original_lf_path = build_lockfile_path(self.sources_path)
         vs_lf_path = os.path.join(virtual_store_dir, "lock.yaml")
@@ -236,14 +282,15 @@ def _replace_internal_lockfile_with_original(self, virtual_store_dir):
     @timeit
     def _copy_pnpm_patches(self):
         pj = self.load_package_json_from_dir(self.sources_path)
-        patchedDependencies: dict[str, str] = pj.data.get("pnpm", {}).get("patchedDependencies", {})
+        patched_dependencies: dict[str, str] = pj.data.get("pnpm", {}).get("patchedDependencies", {})
 
-        for p in patchedDependencies.values():
+        for p in patched_dependencies.values():
             patch_source_path = os.path.join(self.sources_path, p)
             patch_build_path = os.path.join(self.build_path, p)
             os.makedirs(os.path.dirname(patch_build_path), exist_ok=True)
             shutil.copyfile(patch_source_path, patch_build_path)
 
+    @timeit
     def _get_default_options(self):
         return super(PnpmPackageManager, self)._get_default_options() + [
             "--stream",
@@ -252,5 +299,6 @@ def _get_default_options(self):
             "--no-color",
         ]
 
+    @timeit
     def _get_debug_log_path(self):
         return self._nm_path(".pnpm-debug.log")

build/plugins/lib/nots/package_manager/ya.make

@@ -4,7 +4,6 @@ PY3_LIBRARY()
 
 STYLE_PYTHON()
 
-
 PY_SRCS(
     __init__.py
 )