Merge pull request #413 from nix-community/worker-count

builder_worker: make number of workers configureable

Author: Mic92

buildbot_nix/buildbot_nix/__init__.py

@@ -988,6 +988,7 @@ async def run_vc(
         pr_head = build_props.getProperty("github.head.sha") or build_props.getProperty(
             "head_sha"
         )
+        auth_workdir = self._get_auth_data_workdir()
 
         # Not a PR, fallback to default behavior
         if merge_base is None or pr_head is None:
@@ -1007,33 +1008,42 @@ async def run_vc(
             self.build.path_module.join(self.workdir, ".git")
         )
 
-        if not has_git:
-            await self._dovccmd(["clone", "--recurse-submodules", self.repourl, "."])
+        try:
+            await self._git_auth.download_auth_files_if_needed(auth_workdir)
 
-        patched = await self.sourcedirIsPatched()
+            if not has_git:
+                await self._dovccmd(
+                    ["clone", "--recurse-submodules", self.repourl, "."]
+                )
 
-        if patched:
-            await self._dovccmd(["clean", "-f", "-f", "-d", "-x"])
+            patched = await self.sourcedirIsPatched()
 
-        await self._dovccmd(["fetch", "-f", "-t", self.repourl, merge_base, pr_head])
+            if patched:
+                await self._dovccmd(["clean", "-f", "-f", "-d", "-x"])
 
-        await self._dovccmd(["checkout", "--detach", "-f", pr_head])
+            await self._dovccmd(
+                ["fetch", "-f", "-t", self.repourl, merge_base, pr_head]
+            )
 
-        await self._dovccmd(
-            [
-                "-c",
-                "user.email=buildbot@example.com",
-                "-c",
-                "user.name=buildbot",
-                "merge",
-                "--no-ff",
-                "-m",
-                f"Merge {merge_base} into {pr_head}",
-                merge_base,
-            ]
-        )
-        self.updateSourceProperty("got_revision", pr_head)
-        return await self.parseCommitDescription()
+            await self._dovccmd(["checkout", "--detach", "-f", pr_head])
+
+            await self._dovccmd(
+                [
+                    "-c",
+                    "user.email=buildbot@example.com",
+                    "-c",
+                    "user.name=buildbot",
+                    "merge",
+                    "--no-ff",
+                    "-m",
+                    f"Merge {merge_base} into {pr_head}",
+                    merge_base,
+                ]
+            )
+            self.updateSourceProperty("got_revision", pr_head)
+            return await self.parseCommitDescription()
+        finally:
+            await self._git_auth.remove_auth_files_if_needed(auth_workdir)
 
 
 def nix_eval_config(
@@ -1060,6 +1070,12 @@ def nix_eval_config(
             submodules=True,
             haltOnFailure=True,
             logEnviron=False,
+            sshPrivateKey=project.private_key_path.read_text()
+            if project.private_key_path
+            else None,
+            sshKnownHosts=project.known_hosts_path.read_text()
+            if project.known_hosts_path
+            else None,
         ),
     )
     drv_gcroots_dir = util.Interpolate(
@@ -1368,6 +1384,8 @@ def nix_skipped_build_config(
         collapseRequests=False,
         env={},
         factory=factory,
+        do_build_if=lambda build: do_register_gcroot_if(build, branch_config_dict)
+        and outputs_path is not None,
     )
 
 
@@ -1417,6 +1435,12 @@ def buildbot_effects_config(
             method="clean",
             submodules=True,
             haltOnFailure=True,
+            sshPrivateKey=project.private_key_path.read_text()
+            if project.private_key_path
+            else None,
+            sshKnownHosts=project.known_hosts_path.read_text()
+            if project.known_hosts_path
+            else None,
         ),
     )
     secrets_list = []

buildbot_nix/buildbot_nix/gitea_projects.py

@@ -61,6 +61,8 @@ def __init__(
 
     def get_project_url(self) -> str:
         url = urlparse(self.config.instance_url)
+        if self.config.ssh_private_key_file:
+            return self.data.ssh_url
         return f"{url.scheme}://git:%(secret:{self.config.token_file})s@{url.hostname}/{self.name}"
 
     def create_change_source(self) -> ChangeSource | None:
@@ -113,6 +115,14 @@ def belongs_to_org(self) -> bool:
         # TODO Gitea doesn't include this information
         return False  # self.data["owner"]["type"] == "Organization"
 
+    @property
+    def private_key_path(self) -> Path | None:
+        return self.config.ssh_private_key_file
+
+    @property
+    def known_hosts_path(self) -> Path | None:
+        return self.config.ssh_known_hosts_file
+
 
 class GiteaBackend(GitBackend):
     config: GiteaConfig

buildbot_nix/buildbot_nix/github_projects.py

@@ -770,6 +770,14 @@ def topics(self) -> list[str]:
     def belongs_to_org(self) -> bool:
         return self.data.owner.ttype == "Organization"
 
+    @property
+    def private_key_path(self) -> Path | None:
+        return None
+
+    @property
+    def known_hosts_path(self) -> Path | None:
+        return None
+
 
 def refresh_projects(
     github_token: str,

buildbot_nix/buildbot_nix/models.py

@@ -67,6 +67,9 @@ class GiteaConfig(BaseModel):
     oauth_id: str | None
     oauth_secret_file: Path | None
 
+    ssh_private_key_file: Path | None
+    ssh_known_hosts_file: Path | None
+
     @property
     def token(self) -> str:
         return read_secret_file(self.token_file)

buildbot_nix/buildbot_nix/projects.py

@@ -1,4 +1,5 @@
 from abc import ABC, abstractmethod
+from pathlib import Path
 from typing import Any
 
 from buildbot.changes.base import ChangeSource
@@ -125,3 +126,13 @@ def topics(self) -> list[str]:
     @abstractmethod
     def belongs_to_org(self) -> bool:
         pass
+
+    @property
+    @abstractmethod
+    def private_key_path(self) -> Path | None:
+        pass
+
+    @property
+    @abstractmethod
+    def known_hosts_path(self) -> Path | None:
+        pass

buildbot_nix/buildbot_nix/pull_based/project.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 from typing import Any
 from urllib.parse import ParseResult, urlparse
 
@@ -99,3 +100,11 @@ def topics(self) -> list[str]:
     @property
     def belongs_to_org(self) -> bool:
         return False
+
+    @property
+    def private_key_path(self) -> Path | None:
+        return None
+
+    @property
+    def known_hosts_path(self) -> Path | None:
+        return None

buildbot_nix/buildbot_nix/worker.py

@@ -25,9 +25,11 @@ class WorkerConfig:
     worker_name: str = field(
         default_factory=lambda: os.environ.get("WORKER_NAME", socket.gethostname())
     )
-    worker_count: int = int(
-        os.environ.get("WORKER_COUNT", str(multiprocessing.cpu_count())),
-    )
+    worker_count_str: str | None = os.environ.get("WORKER_COUNT")
+    if worker_count_str is not None:
+        worker_count = int(worker_count_str)
+    else:
+        worker_count = multiprocessing.cpu_count()
     buildbot_dir: Path = field(
         default_factory=lambda: Path(require_env("BUILDBOT_DIR"))
     )

examples/worker.nix

@@ -3,5 +3,8 @@
   services.buildbot-nix.worker = {
     enable = true;
     workerPasswordFile = pkgs.writeText "worker-password-file" "XXXXXXXXXXXXXXXXXXXX";
+    # The number of workers to start (default: 0 == the number of CPU cores).
+    # If you experience flaky builds under high load, try to reduce this value.
+    # workers = 0;
   };
 }

nix/master.nix

@@ -313,6 +313,23 @@ in
             If null, all projects that the buildbot Gitea user has access to, are built.
           '';
         };
+
+        sshPrivateKeyFile = lib.mkOption {
+          type = lib.types.nullOr lib.types.path;
+          default = null;
+          description = ''
+            If non-null the specified SSH key will be used to fetch all configured repositories.
+          '';
+        };
+
+        sshKnownHostsFile = lib.mkOption {
+          type = lib.types.nullOr lib.types.path;
+          default = null;
+          description = ''
+            If non-null the specified known hosts file will be matched against when connecting to
+            repositories over SSH.
+          '';
+        };
       };
       github = {
         enable = lib.mkEnableOption "Enable GitHub integration" // {
@@ -702,6 +719,8 @@ in
                       instance_url = cfg.gitea.instanceUrl;
                       oauth_id = cfg.gitea.oauthId;
                       topic = cfg.gitea.topic;
+                      ssh_private_key_file = cfg.gitea.sshPrivateKeyFile;
+                      ssh_known_hosts_file = cfg.gitea.sshKnownHostsFile;
                     };
                 github =
                   if !cfg.github.enable then

nix/worker.nix

@@ -56,6 +56,14 @@ in
         default = pkgs.nix-eval-jobs;
         description = "nix-eval-jobs to use for evaluation";
       };
+      workers = lib.mkOption {
+        type = lib.types.int;
+        default = 0;
+        description = ''
+          The number of workers to start (default: 0 == to the number of CPU cores).
+          If you experience flaky builds under high load, try to reduce this value.
+        '';
+      };
       masterUrl = lib.mkOption {
         type = lib.types.str;
         default = "tcp:host=localhost:port=9989";
@@ -114,6 +122,7 @@ in
       }/${packages.python.sitePackages}";
       environment.MASTER_URL = cfg.masterUrl;
       environment.BUILDBOT_DIR = buildbotDir;
+      environment.WORKER_COUNT = builtins.toString cfg.workers;
 
       serviceConfig = {
         # We rather want the CI job to fail on OOM than to have a broken buildbot worker.