feat: parallelize the apt package installation chunks

aminya · aminya · commit dac72316dfb2 · 2022-11-22T21:33:06.000-08:00
diff --git a/dist/node12/setup_cpp.js b/dist/node12/setup_cpp.js
diff --git a/dist/node12/setup_cpp.js.map b/dist/node12/setup_cpp.js.map
diff --git a/dist/node16/setup_cpp.js b/dist/node16/setup_cpp.js
diff --git a/dist/node16/setup_cpp.js.map b/dist/node16/setup_cpp.js.map
diff --git a/src/bazel/bazel.ts b/src/bazel/bazel.ts
@@ -36,7 +36,7 @@ export async function setupBazel(version: string, _setupDir: string, _arch: stri
           "-c",
           `echo "deb [arch=amd64 signed-by=${keyFileName}] https://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list`,
         ])
-        return setupAptPack("bazel", version, [], true)
+        return setupAptPack([{ name: "bazel", version }], true)
       }
       throw new Error(`Unsupported linux distribution`)
     }
diff --git a/src/ccache/ccache.ts b/src/ccache/ccache.ts
@@ -22,7 +22,7 @@ export function setupCcache(version: string, _setupDir: string, _arch: string) {
       } else if (hasDnf()) {
         return setupDnfPack("ccache", version)
       } else if (isUbuntu()) {
-        return setupAptPack("ccache", version)
+        return setupAptPack([{ name: "ccache", version }])
       }
       throw new Error(`Unsupported linux distribution`)
     }
diff --git a/src/cppcheck/cppcheck.ts b/src/cppcheck/cppcheck.ts
@@ -25,7 +25,7 @@ export async function setupCppcheck(version: string | undefined, _setupDir: stri
       } else if (hasDnf()) {
         return setupDnfPack("ccache", version)
       } else if (isUbuntu()) {
-        return setupAptPack("cppcheck", version)
+        return setupAptPack([{ name: "cppcheck", version }])
       }
       throw new Error(`Unsupported linux distribution`)
     }
diff --git a/src/doxygen/doxygen.ts b/src/doxygen/doxygen.ts
@@ -67,18 +67,18 @@ export async function setupDoxygen(version: string, setupDir: string, arch: stri
         } else if (hasDnf()) {
           return setupDnfPack("doxygen", version)
         } else if (isUbuntu()) {
-          installationInfo = await setupAptPack("doxygen", version)
+          installationInfo = await setupAptPack([{ name: "doxygen", version }])
         } else {
           throw new Error(`Unsupported linux distributions`)
         }
       } else if (isUbuntu()) {
         try {
           // doxygen on stable Ubuntu repositories is very old. So, we use get the binary from the website itself
           installationInfo = await setupBin("doxygen", version, getDoxygenPackageInfo, setupDir, arch)
-          await setupAptPack("libclang-cpp9")
+          await setupAptPack([{ name: "libclang-cpp9" }])
         } catch (err) {
           notice(`Failed to download doxygen binary. ${err}. Falling back to apt-get.`)
-          installationInfo = await setupAptPack("doxygen", undefined)
+          installationInfo = await setupAptPack([{ name: "doxygen" }])
         }
       } else {
         throw new Error(`Unsupported linux distributions`)
diff --git a/src/gcc/gcc.ts b/src/gcc/gcc.ts
@@ -96,15 +96,17 @@ export async function setupGcc(version: string, setupDir: string, arch: string)
           setupDnfPack("gcc-c++", version)
           setupDnfPack("libstdc++-devel", undefined)
         } else if (isUbuntu()) {
-          await setupAptPack("gcc", version, ["ppa:ubuntu-toolchain-r/test"])
-          installationInfo = await setupAptPack("g++", version, [])
+          installationInfo = await setupAptPack([
+            { name: "gcc", version, repositories: ["ppa:ubuntu-toolchain-r/test"] },
+            { name: "g++", version, repositories: ["ppa:ubuntu-toolchain-r/test"] },
+          ])
         }
       } else {
         info(`Install g++-multilib because gcc for ${arch} was requested`)
         if (isArch()) {
           setupPacmanPack("gcc-multilib", version)
         } else if (isUbuntu()) {
-          await setupAptPack("gcc-multilib", version, ["ppa:ubuntu-toolchain-r/test"])
+          await setupAptPack([{ name: "gcc-multilib", version, repositories: ["ppa:ubuntu-toolchain-r/test"] }])
         }
       }
       break
diff --git a/src/graphviz/graphviz.ts b/src/graphviz/graphviz.ts
@@ -25,7 +25,7 @@ export async function setupGraphviz(version: string, _setupDir: string, _arch: s
       } else if (hasDnf()) {
         return setupDnfPack("graphviz", version)
       } else if (isUbuntu()) {
-        return setupAptPack("graphviz", version)
+        return setupAptPack([{ name: "graphviz", version }])
       }
       throw new Error(`Unsupported linux distribution`)
     }
diff --git a/src/kcov/kcov.ts b/src/kcov/kcov.ts
@@ -50,8 +50,7 @@ async function buildKcov(file: string, dest: string) {
       setupDnfPack("libdwarf-devel")
       setupDnfPack("libcurl-devel")
     } else if (isUbuntu()) {
-      await setupAptPack("libdw-dev")
-      await setupAptPack("libcurl4-openssl-dev")
+      await setupAptPack([{ name: "libdw-dev" }, { name: "libcurl4-openssl-dev" }])
     }
   }
   const buildDir = join(out, "build")
@@ -102,7 +101,7 @@ export async function setupKcov(versionGiven: string, setupDir: string, arch: st
     } else if (hasDnf()) {
       setupDnfPack("binutils")
     } else if (isUbuntu()) {
-      await setupAptPack("libbinutils")
+      await setupAptPack([{ name: "libbinutils" }])
     }
     return installationInfo
   } else {
diff --git a/src/llvm/llvm.ts b/src/llvm/llvm.ts
@@ -52,9 +52,9 @@ async function setupLLVMDeps(arch: string, version: string) {
     if (isUbuntu()) {
       const majorVersion = parseInt(version.split(".")[0], 10)
       if (majorVersion <= 10) {
-        await setupAptPack("libtinfo5")
+        await setupAptPack([{ name: "libtinfo5" }])
       } else {
-        await setupAptPack("libtinfo-dev")
+        await setupAptPack([{ name: "libtinfo-dev" }])
       }
     }
     // TODO: install libtinfo on other distros
diff --git a/src/make/make.ts b/src/make/make.ts
@@ -25,7 +25,7 @@ export async function setupMake(version: string, _setupDir: string, _arch: strin
       } else if (hasDnf()) {
         return setupDnfPack("make", version)
       } else if (isUbuntu()) {
-        return setupAptPack("make", version)
+        return setupAptPack([{ name: "make", version }])
       }
       throw new Error(`Unsupported linux distribution`)
     }
diff --git a/src/nala/nala.ts b/src/nala/nala.ts
@@ -33,12 +33,12 @@ export async function setupNala(version: string, _setupDir: string, _arch: strin
 
   try {
     if (version !== "legacy") {
-      await setupAptPack("nala", undefined, [], true)
+      await setupAptPack([{ name: "nala" }], true)
     } else {
-      await setupAptPack("nala-legacy", undefined, [], true)
+      await setupAptPack([{ name: "nala-legacy" }], true)
     }
   } catch (err) {
-    await setupAptPack("nala-legacy", undefined, [], true)
+    await setupAptPack([{ name: "nala-legacy" }], true)
   }
 
   binDir = "/usr/bin" // eslint-disable-line require-atomic-updates
diff --git a/src/powershell/powershell.ts b/src/powershell/powershell.ts
@@ -33,7 +33,7 @@ export async function setupPowershell(version: string | undefined, _setupDir: st
         ])
         return setupDnfPack("powershell", version)
       } else if (isUbuntu()) {
-        await setupAptPack("curl")
+        await setupAptPack([{ name: "curl" }])
         const ubuntuVerSplitted = (await ubuntuVersion())!
         const ubuntuVersionString = `${ubuntuVerSplitted[0]}.0${ubuntuVerSplitted[1]}`
 
@@ -53,7 +53,7 @@ export async function setupPowershell(version: string | undefined, _setupDir: st
         //   `echo "deb [arch=amd64 signed-by=${keyFileName}] https://packages.microsoft.com/repos/microsoft-debian-bullseye-prod bullseye main" > /etc/apt/sources.list.d/microsoft.list`,
         // ])
 
-        return setupAptPack("powershell", version, [], true)
+        return setupAptPack([{ name: "powershell", version }], true)
       }
       throw new Error(`Unsupported linux distribution`)
     }
diff --git a/src/python/python.ts b/src/python/python.ts
@@ -69,8 +69,7 @@ export async function setupPythonViaSystem(
         installInfo = setupDnfPack("python3", version)
         setupDnfPack("python3-pip")
       } else if (isUbuntu()) {
-        installInfo = await setupAptPack("python3", version)
-        await setupAptPack("python3-pip")
+        installInfo = await setupAptPack([{ name: "python3", version }, { name: "python3-pip" }])
       } else {
         throw new Error(`Unsupported linux distributions`)
       }
@@ -118,7 +117,7 @@ export async function setupPythonAndPip(): Promise<string> {
     } else if (hasDnf()) {
       setupDnfPack("python3-pip")
     } else if (isUbuntu()) {
-      await setupAptPack("python3-pip")
+      await setupAptPack([{ name: "python3-pip" }])
     }
   }
 
diff --git a/src/sevenzip/sevenzip.ts b/src/sevenzip/sevenzip.ts
@@ -23,7 +23,7 @@ export function setupSevenZip(version: string, _setupDir: string, _arch: string)
         setupDnfPack("p7zip", version)
         return setupDnfPack("p7zip-plugins", version)
       } else if (isUbuntu()) {
-        return setupAptPack("p7zip-full", version)
+        return setupAptPack([{ name: "p7zip-full", version }])
       }
       throw new Error(`Unsupported linux distribution`)
     }
diff --git a/src/utils/env/ubuntu_version.ts b/src/utils/env/ubuntu_version.ts
@@ -8,7 +8,7 @@ export async function ubuntuVersion(): Promise<number[] | null> {
   try {
     if (isUbuntu()) {
       if (which.sync("lsb_release", { nothrow: true }) === null) {
-        await setupAptPack("lsb-release")
+        await setupAptPack([{ name: "lsb-release" }])
       }
       const versionSplitted = await getUbuntuVersion()
 
diff --git a/src/utils/setup/setupAptPack.ts b/src/utils/setup/setupAptPack.ts
@@ -8,20 +8,24 @@ import which from "which"
 import pathExists from "path-exists"
 import { promises as fsPromises } from "fs"
 const { appendFile } = fsPromises
+import execa from "execa"
 
 let didUpdate: boolean = false
 let didInit: boolean = false
 
+export type AptPackage = {
+  name: string
+  version?: string
+  repositories?: string[]
+}
+
 /** A function that installs a package using apt */
-export async function setupAptPack(
-  name: string,
-  version?: string,
-  repositories: string[] = [],
-  update = false
-): Promise<InstallationInfo> {
+export async function setupAptPack(packages: AptPackage[], update = false): Promise<InstallationInfo> {
   const apt: string = getApt()
 
-  info(`Installing ${name} ${version ?? ""} via ${apt}`)
+  for (const { name, version } of packages) {
+    info(`Installing ${name} ${version ?? ""} via ${apt}`)
+  }
 
   process.env.DEBIAN_FRONTEND = "noninteractive"
 
@@ -35,25 +39,34 @@ export async function setupAptPack(
     didInit = true
   }
 
-  if (Array.isArray(repositories) && repositories.length !== 0) {
-    for (const repo of repositories) {
+  const allRepositories = [...new Set(packages.flatMap((pack) => pack.repositories ?? []))]
+
+  if (allRepositories.length !== 0) {
+    for (const repo of allRepositories) {
       // eslint-disable-next-line no-await-in-loop
-      execRootSync("add-apt-repository", ["--update", "-y", repo])
+      execRootSync("add-apt-repository", ["-y", repo])
     }
+
     updateRepos(apt)
   }
 
+  const aptArgs = await Promise.all(packages.map((pack) => getAptArg(pack.name, pack.version)))
+  execRootSync(apt, ["install", "--fix-broken", "-y", ...aptArgs])
+
+  return { binDir: "/usr/bin/" }
+}
+
+async function getAptArg(name: string, version: string | undefined) {
   if (version !== undefined && version !== "") {
-    try {
-      execRootSync(apt, ["install", "--fix-broken", "-y", `${name}=${version}`])
-    } catch {
-      execRootSync(apt, ["install", "--fix-broken", "-y", `${name}-${version}`])
+    const { stdout } = await execa("apt-cache", ["search", "--names-only", `^${name}-${version}$`])
+    if (stdout.trim() !== "") {
+      return `${name}-${version}`
+    } else {
+      return `${name}=${version}`
     }
   } else {
-    execRootSync(apt, ["install", "--fix-broken", "-y", name])
+    return name
   }
-
-  return { binDir: "/usr/bin/" }
 }
 
 function getApt() {
@@ -118,7 +131,7 @@ export async function addAptKeyViaDownload(name: string, url: string) {
   const fileName = `/etc/apt/trusted.gpg.d/${name}`
   if (!(await pathExists(fileName))) {
     initGpg()
-    await setupAptPack("curl", undefined)
+    await setupAptPack([{ name: "curl" }], undefined)
     execRootSync("bash", ["-c", `curl -s ${url} | gpg --no-default-keyring --keyring gnupg-ring:${fileName} --import`])
     execRootSync("chmod", ["644", fileName])
   }
diff --git a/src/utils/setup/setupBin.ts b/src/utils/setup/setupBin.ts
@@ -103,9 +103,7 @@ export async function setupBin(
           setupDnfPack("tar")
           setupDnfPack("xz")
         } else if (isUbuntu()) {
-          await setupAptPack("unzip")
-          await setupAptPack("tar")
-          await setupAptPack("xz-utils")
+          await setupAptPack([{ name: "unzip" }, { name: "tar" }, { name: "xz-utils" }])
         }
       }
       // eslint-disable-next-line require-atomic-updates
diff --git a/src/vcpkg/vcpkg.ts b/src/vcpkg/vcpkg.ts
@@ -36,12 +36,14 @@ export async function setupVcpkg(_version: string, setupDir: string, _arch: stri
         setupDnfPack("git")
         setupDnfPack("pkg-config")
       } else if (isUbuntu()) {
-        await setupAptPack("curl")
-        await setupAptPack("zip")
-        await setupAptPack("unzip")
-        await setupAptPack("tar")
-        await setupAptPack("git")
-        await setupAptPack("pkg-config")
+        await setupAptPack([
+          { name: "curl" },
+          { name: "zip" },
+          { name: "unzip" },
+          { name: "tar" },
+          { name: "git" },
+          { name: "pkg-config" },
+        ])
       }
     }
 

Original file line number	Diff line number	Diff line change
`@@ -36,7 +36,7 @@ export async function setupBazel(version: string, _setupDir: string, _arch: stri`
`36`	`36`	`"-c",`
`37`	`37`	`echo "deb [arch=amd64 signed-by=${keyFileName}] https://storage.googleapis.com/bazel-apt stable jdk1.8" \| tee /etc/apt/sources.list.d/bazel.list`,
`38`	`38`	`])`
`39`		`- return setupAptPack("bazel", version, [], true)`
	`39`	`+ return setupAptPack([{ name: "bazel", version }], true)`
`40`	`40`	`}`
`41`	`41`	throw new Error(`Unsupported linux distribution`)
`42`	`42`	`}`
Original file line number	Diff line number	Diff line change
`@@ -22,7 +22,7 @@ export function setupCcache(version: string, _setupDir: string, _arch: string) {`
`22`	`22`	`} else if (hasDnf()) {`
`23`	`23`	`return setupDnfPack("ccache", version)`
`24`	`24`	`} else if (isUbuntu()) {`
`25`		`- return setupAptPack("ccache", version)`
	`25`	`+ return setupAptPack([{ name: "ccache", version }])`
`26`	`26`	`}`
`27`	`27`	throw new Error(`Unsupported linux distribution`)
`28`	`28`	`}`
Original file line number	Diff line number	Diff line change
`@@ -25,7 +25,7 @@ export async function setupCppcheck(version: string \| undefined, _setupDir: stri`
`25`	`25`	`} else if (hasDnf()) {`
`26`	`26`	`return setupDnfPack("ccache", version)`
`27`	`27`	`} else if (isUbuntu()) {`
`28`		`- return setupAptPack("cppcheck", version)`
	`28`	`+ return setupAptPack([{ name: "cppcheck", version }])`
`29`	`29`	`}`
`30`	`30`	throw new Error(`Unsupported linux distribution`)
`31`	`31`	`}`