From d3d15c0aab7b11b2cfe7088948c810eb5a8b18f9 Mon Sep 17 00:00:00 2001 From: Andrian Sevastyanov Date: Fri, 16 May 2025 12:16:30 -0600 Subject: [PATCH 1/8] Optimize yarn workspace discovery --- .../detectables/yarn/YarnLockExtractor.java | 7 +- .../yarn/packagejson/PackageJsonFiles.java | 65 ++++++++++++------- 2 files changed, 43 insertions(+), 29 deletions(-) diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/yarn/YarnLockExtractor.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/yarn/YarnLockExtractor.java index 4d5eb0f2c1..5683e1e962 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/yarn/YarnLockExtractor.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/yarn/YarnLockExtractor.java @@ -90,12 +90,7 @@ private ExcludedIncludedWildcardFilter deriveExcludedIncludedWildcardFilter() { @NotNull private YarnWorkspaces collectWorkspaceData(File dir) throws IOException { - Collection curLevelWorkspaces = packageJsonFiles.readWorkspacePackageJsonFiles(dir); - Collection allWorkspaces = new LinkedList<>(curLevelWorkspaces); - for (YarnWorkspace workspace : curLevelWorkspaces) { - Collection treeBranchWorkspacePackageJsons = packageJsonFiles.readWorkspacePackageJsonFiles(workspace.getWorkspacePackageJson().getDir()); - allWorkspaces.addAll(treeBranchWorkspacePackageJsons); - } + Collection allWorkspaces = packageJsonFiles.readWorkspacePackageJsonFiles(dir); return new YarnWorkspaces(allWorkspaces); } } diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/yarn/packagejson/PackageJsonFiles.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/yarn/packagejson/PackageJsonFiles.java index ccb6e653e8..3e5fabf347 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/yarn/packagejson/PackageJsonFiles.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/yarn/packagejson/PackageJsonFiles.java @@ -38,42 +38,61 @@ public NullSafePackageJson read(File packageJsonFile) throws IOException { @NotNull public Collection readWorkspacePackageJsonFiles(File workspaceDir) throws IOException { - String forwardSlashedWorkspaceDirPath = deriveForwardSlashedPath(workspaceDir); File packageJsonFile = new File(workspaceDir, YarnLockDetectable.YARN_PACKAGE_JSON); List workspaceDirPatterns = extractWorkspaceDirPatterns(packageJsonFile); + if (workspaceDirPatterns.isEmpty()) { + logger.debug("No workspace patterns found in {}", packageJsonFile.getAbsolutePath()); + return new LinkedList<>(); + } + + List matchers = convertWorkspaceDirPatternsToPathMatchers(workspaceDirPatterns, workspaceDir); + Collection workspaces = new LinkedList<>(); - for (String workspaceSubdirPattern : workspaceDirPatterns) { - logger.trace("workspaceSubdirPattern: {}", workspaceSubdirPattern); - String globString = String.format("glob:%s/%s/package.json", forwardSlashedWorkspaceDirPath, workspaceSubdirPattern); - logger.trace("workspace subdir globString: {}", globString); - PathMatcher matcher = FileSystems.getDefault().getPathMatcher(globString); - Files.walkFileTree(workspaceDir.toPath(), new SimpleFileVisitor() { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - if (matcher.matches(file)) { - logger.trace("\tFound a match: {}", file); - NullSafePackageJson packageJson = read(file.toFile()); - Path rel = workspaceDir.toPath().relativize(file.getParent()); - WorkspacePackageJson workspacePackageJson = new WorkspacePackageJson(file.toFile(), packageJson, rel.toString()); - YarnWorkspace workspace = new YarnWorkspace(workspacePackageJson); - workspaces.add(workspace); + + Files.walkFileTree(workspaceDir.toPath(), new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + if (file.getFileName().toString().equals(YarnLockDetectable.YARN_PACKAGE_JSON)) { // no need to try matching if not package.json + for (PathMatcher matcher : matchers) { + if (matcher.matches(file)) { + logger.trace("\tFound a match: {}", file); + NullSafePackageJson packageJson = read(file.toFile()); + Path rel = workspaceDir.toPath().relativize(file.getParent()); + WorkspacePackageJson workspacePackageJson = new WorkspacePackageJson(file.toFile(), packageJson, rel.toString()); + YarnWorkspace workspace = new YarnWorkspace(workspacePackageJson); + workspaces.add(workspace); + break; // no need to match the same file multiple times + } } - return FileVisitResult.CONTINUE; } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) { + return FileVisitResult.CONTINUE; + } + }); - @Override - public FileVisitResult visitFileFailed(Path file, IOException exc) { - return FileVisitResult.CONTINUE; - } - }); - } if (!workspaceDirPatterns.isEmpty()) { logger.debug("Found {} matching workspace package.json files for workspaces listed in {}", workspaces.size(), packageJsonFile.getAbsolutePath()); } return workspaces; } + private List convertWorkspaceDirPatternsToPathMatchers(List workspaceDirPatterns, File workspaceDir) { + List matchers = new LinkedList<>(); + String forwardSlashedWorkspaceDirPath = deriveForwardSlashedPath(workspaceDir); + for (String workspaceDirPattern : workspaceDirPatterns) { + String globString = String.format("glob:%s/%s/package.json", forwardSlashedWorkspaceDirPath, workspaceDirPattern); + logger.trace("workspace subdir globString: {}", globString); + PathMatcher matcher = FileSystems.getDefault().getPathMatcher(globString); + matchers.add(matcher); + } + return matchers; + } + @NotNull private String deriveForwardSlashedPath(File file) { String forwardSlashWorkspaceDirPath; From 74ed1c2e799d5e96eb3d32af31eeb57c7692f4f5 Mon Sep 17 00:00:00 2001 From: Andrian Sevastyanov Date: Fri, 16 May 2025 15:42:58 -0600 Subject: [PATCH 2/8] Change workspace collection back to two levels --- .../detectable/detectables/yarn/YarnLockExtractor.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/yarn/YarnLockExtractor.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/yarn/YarnLockExtractor.java index 5683e1e962..4d5eb0f2c1 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/yarn/YarnLockExtractor.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/yarn/YarnLockExtractor.java @@ -90,7 +90,12 @@ private ExcludedIncludedWildcardFilter deriveExcludedIncludedWildcardFilter() { @NotNull private YarnWorkspaces collectWorkspaceData(File dir) throws IOException { - Collection allWorkspaces = packageJsonFiles.readWorkspacePackageJsonFiles(dir); + Collection curLevelWorkspaces = packageJsonFiles.readWorkspacePackageJsonFiles(dir); + Collection allWorkspaces = new LinkedList<>(curLevelWorkspaces); + for (YarnWorkspace workspace : curLevelWorkspaces) { + Collection treeBranchWorkspacePackageJsons = packageJsonFiles.readWorkspacePackageJsonFiles(workspace.getWorkspacePackageJson().getDir()); + allWorkspaces.addAll(treeBranchWorkspacePackageJsons); + } return new YarnWorkspaces(allWorkspaces); } } From 89f54731b19ddf5e0b081c3aca464b1a15f8b8c5 Mon Sep 17 00:00:00 2001 From: blackduck-serv-builder Date: Fri, 16 May 2025 17:52:55 -0400 Subject: [PATCH 3/8] Release 10.5.0-SIGQA4-andrians.IDETECT-4713-yarn-workspace-glob-optimization --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 9e24ad29ed..8e354254f0 100644 --- a/build.gradle +++ b/build.gradle @@ -21,7 +21,7 @@ buildscript { group = 'com.blackduck.integration' -version = '10.5.0-SIGQA4-SNAPSHOT' +version = '10.5.0-SIGQA4-andrians.IDETECT-4713-yarn-workspace-glob-optimization' apply plugin: 'com.blackduck.integration.solution' apply plugin: 'org.springframework.boot' From c3c18b90387a7c006eb23e961cc9cbee1c7980a2 Mon Sep 17 00:00:00 2001 From: blackduck-serv-builder Date: Fri, 16 May 2025 18:05:30 -0400 Subject: [PATCH 4/8] Using the next snapshot post release 10.5.0-SIGQA5-andrians.IDETECT-4713-yarn-workspace-glob-optimization-SNAPSHOT --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 8e354254f0..0246571d7e 100644 --- a/build.gradle +++ b/build.gradle @@ -21,7 +21,7 @@ buildscript { group = 'com.blackduck.integration' -version = '10.5.0-SIGQA4-andrians.IDETECT-4713-yarn-workspace-glob-optimization' +version = '10.5.0-SIGQA5-andrians.IDETECT-4713-yarn-workspace-glob-optimization-SNAPSHOT' apply plugin: 'com.blackduck.integration.solution' apply plugin: 'org.springframework.boot' From 205084903299da2f8e4b3ac360e8de222c27535d Mon Sep 17 00:00:00 2001 From: Andrian Sevastyanov Date: Mon, 26 May 2025 10:50:02 -0600 Subject: [PATCH 5/8] Added test cases --- .../packagejson/PackageJsonFilesTest.java | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 detectable/src/test/java/com/blackduck/integration/detectable/detectables/yarn/packagejson/PackageJsonFilesTest.java diff --git a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/yarn/packagejson/PackageJsonFilesTest.java b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/yarn/packagejson/PackageJsonFilesTest.java new file mode 100644 index 0000000000..1dc0ad8205 --- /dev/null +++ b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/yarn/packagejson/PackageJsonFilesTest.java @@ -0,0 +1,101 @@ +package com.blackduck.integration.detectable.detectables.yarn.packagejson; + +import com.blackduck.integration.detectable.detectables.yarn.workspace.YarnWorkspace; +import com.google.gson.Gson; +import org.apache.commons.io.FileUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Collection; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class PackageJsonFilesTest { + + private PackageJsonReader packageJsonReader; + private PackageJsonFiles packageJsonFiles; + private File tempDir; + + @BeforeEach + void setUp() throws IOException { + packageJsonReader = new PackageJsonReader(new Gson()); + packageJsonFiles = new PackageJsonFiles(packageJsonReader); + tempDir = Files.createTempDirectory("yarnws").toFile(); + } + + @AfterEach + void tearDown() throws IOException { + FileUtils.deleteDirectory(tempDir); + } + + @Test + void returnsEmptyWhenNoWorkspacePatterns() throws IOException { + // Setup a root package.json with no workspaces + File rootPackageJson = new File(tempDir, "package.json"); + Files.write(rootPackageJson.toPath(), "{ \"workspaces\": [] }".getBytes(StandardCharsets.UTF_8)); + + Collection result = packageJsonFiles.readWorkspacePackageJsonFiles(tempDir); + assertTrue(result.isEmpty()); + } + + @Test + void findsWorkspacePackageJsonFiles() throws IOException { + // Setup a root package.json with a workspace pattern + File rootPackageJson = new File(tempDir, "package.json"); + Files.write(rootPackageJson.toPath(), "{ \"workspaces\": [\"packages/*\"] }".getBytes(StandardCharsets.UTF_8)); + + // Create a workspace directory and its package.json + File wsDir = new File(tempDir, "packages/a"); + wsDir.mkdirs(); + File wsPackageJson = new File(wsDir, "package.json"); + Files.write(wsPackageJson.toPath(), "{ \"name\": \"a\" }".getBytes(StandardCharsets.UTF_8)); + + Collection result = packageJsonFiles.readWorkspacePackageJsonFiles(tempDir); + assertEquals(1, result.size()); + + YarnWorkspace foundWorkspace = result.iterator().next(); + assertEquals("a", foundWorkspace.getName().orElse(null)); + assertEquals(wsPackageJson.getAbsolutePath(), foundWorkspace.getWorkspacePackageJson().getFile().getAbsolutePath()); + } + + @Test + void findsMultipleWorkspacesWithDoubleWildcard() throws IOException { + // Setup a root package.json with a double wildcard workspace pattern + File rootPackageJson = new File(tempDir, "package.json"); + Files.write(rootPackageJson.toPath(), "{ \"workspaces\": [\"packages/**\"] }".getBytes(StandardCharsets.UTF_8)); + + // Create two workspace directories and their package.json files + File wsDir1 = new File(tempDir, "packages/a"); + wsDir1.mkdirs(); + File wsPackageJson1 = new File(wsDir1, "package.json"); + Files.write(wsPackageJson1.toPath(), "{ \"name\": \"a\" }".getBytes(StandardCharsets.UTF_8)); + + File wsDir2 = new File(tempDir, "packages/b/subdir"); + wsDir2.mkdirs(); + File wsPackageJson2 = new File(wsDir2, "package.json"); + Files.write(wsPackageJson2.toPath(), "{ \"name\": \"b\" }".getBytes(StandardCharsets.UTF_8)); + + Collection result = packageJsonFiles.readWorkspacePackageJsonFiles(tempDir); + assertEquals(2, result.size()); + + // Verify the workspaces + YarnWorkspace workspaceA = result.stream() + .filter(ws -> "a".equals(ws.getName().orElse(null))) + .findFirst() + .orElseThrow(() -> new AssertionError("Workspace 'a' not found")); + assertEquals(wsPackageJson1.getAbsolutePath(), workspaceA.getWorkspacePackageJson().getFile().getAbsolutePath()); + + YarnWorkspace workspaceB = result.stream() + .filter(ws -> "b".equals(ws.getName().orElse(null))) + .findFirst() + .orElseThrow(() -> new AssertionError("Workspace 'b' not found")); + assertEquals(wsPackageJson2.getAbsolutePath(), workspaceB.getWorkspacePackageJson().getFile().getAbsolutePath()); + } +} From 0f18de33b2ff61a95d5d04af13bd8b69d319fe99 Mon Sep 17 00:00:00 2001 From: Andrian Sevastyanov Date: Wed, 25 Jun 2025 15:08:36 -0600 Subject: [PATCH 6/8] Bump up blackduck-common version to latest --- shared-version.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared-version.properties b/shared-version.properties index 1666f23893..2c39ca70b9 100644 --- a/shared-version.properties +++ b/shared-version.properties @@ -1,2 +1,2 @@ -gradle.ext.blackDuckCommonVersion='67.0.11' +gradle.ext.blackDuckCommonVersion='67.0.12' gradle.ext.springBootVersion='2.7.12' From f28f0bb321b04c3c75f79f14caffba41ef1ceed1 Mon Sep 17 00:00:00 2001 From: blackduck-serv-builder Date: Fri, 27 Jun 2025 10:36:55 -0400 Subject: [PATCH 7/8] Release 10.6.0-SIGQA7-andrians.yarn-optimizations --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 9e6ab83748..07ed19edf8 100644 --- a/build.gradle +++ b/build.gradle @@ -21,7 +21,7 @@ buildscript { group = 'com.blackduck.integration' -version = '10.6.0-SIGQA7-SNAPSHOT' +version = '10.6.0-SIGQA7-andrians.yarn-optimizations' apply plugin: 'com.blackduck.integration.solution' apply plugin: 'org.springframework.boot' From 40474efff94c181f2307886e66fa7acf1134dc35 Mon Sep 17 00:00:00 2001 From: blackduck-serv-builder Date: Fri, 27 Jun 2025 10:47:03 -0400 Subject: [PATCH 8/8] Using the next snapshot post release 10.6.0-SIGQA8-andrians.yarn-optimizations-SNAPSHOT --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 07ed19edf8..d184553297 100644 --- a/build.gradle +++ b/build.gradle @@ -21,7 +21,7 @@ buildscript { group = 'com.blackduck.integration' -version = '10.6.0-SIGQA7-andrians.yarn-optimizations' +version = '10.6.0-SIGQA8-andrians.yarn-optimizations-SNAPSHOT' apply plugin: 'com.blackduck.integration.solution' apply plugin: 'org.springframework.boot'