Merge pull request #430 from nix-community/flake-restructure

Add a local dev workflow

Author: Mic92

.gitignore

@@ -1,6 +1,8 @@
 result
 result-*
 
+.buildbot-dev
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

buildbot_nix/buildbot_nix/__init__.py

@@ -1079,7 +1079,7 @@ def config_for_project(
     config["builders"].extend(
         [
             # Since all workers run on the same machine, we only assign one of them to do the evaluation.
-            # This should prevent exessive memory usage.
+            # This should prevent excessive memory usage.
             nix_eval_config(
                 project,
                 worker_names,
@@ -1289,6 +1289,10 @@ def configure(self, config: dict[str, Any]) -> None:
         config.setdefault("projects", [])
         config.setdefault("secretsProviders", [])
         config.setdefault("www", {})
+        config.setdefault("workers", [])
+        config.setdefault("services", [])
+        config.setdefault("schedulers", [])
+        config.setdefault("builders", [])
 
         worker_names = []
         for w in self.config.nix_worker_secrets().workers:
@@ -1297,8 +1301,13 @@ def configure(self, config: dict[str, Any]) -> None:
                 config["workers"].append(worker.Worker(worker_name, w.password))
                 worker_names.append(worker_name)
 
+        for i in range(self.config.local_workers):
+            worker_name = f"local-{i}"
+            config["workers"].append(worker.LocalWorker(worker_name))
+            worker_names.append(worker_name)
+
         if worker_names == []:
-            msg = f"No workers configured in {self.config.nix_workers_secret_file}"
+            msg = f"No workers configured in {self.config.nix_workers_secret_file} and {self.config.local_workers} local workers."
             raise BuildbotNixError(msg)
 
         eval_lock = util.MasterLock("nix-eval")
@@ -1362,9 +1371,9 @@ def configure(self, config: dict[str, Any]) -> None:
             config.setdefault("secretsProviders", [])
             config["secretsProviders"].extend(backend.create_secret_providers())
 
-        systemd_secrets = SecretInAFile(
-            dirname=os.environ["CREDENTIALS_DIRECTORY"],
-        )
+        credentials_directory = os.environ.get("CREDENTIALS_DIRECTORY", "./secrets")
+        Path(credentials_directory).mkdir(exist_ok=True, parents=True)
+        systemd_secrets = SecretInAFile(dirname=credentials_directory)
         config["secretsProviders"].append(systemd_secrets)
 
         config["www"].setdefault("plugins", {})

buildbot_nix/buildbot_nix/models.py

@@ -1,7 +1,6 @@
 import json
 import os
 import re
-import sys
 from collections.abc import Callable, Mapping
 from enum import Enum
 from pathlib import Path
@@ -32,8 +31,7 @@ class AuthBackendConfig(str, Enum):
 def read_secret_file(secret_file: Path) -> str:
     directory = os.environ.get("CREDENTIALS_DIRECTORY")
     if directory is None:
-        print("directory not set", file=sys.stderr)
-        sys.exit(1)
+        return secret_file.read_text().rstrip()
     return Path(directory).joinpath(secret_file).read_text().rstrip()
 
 
@@ -92,10 +90,10 @@ class PullBasedRepository(BaseModel):
     name: str
     default_branch: str
     url: str
-    poll_interval: int
+    poll_interval: int = 60
 
-    ssh_private_key_file: Path | None
-    ssh_known_hosts_file: Path | None
+    ssh_private_key_file: Path | None = None
+    ssh_known_hosts_file: Path | None = None
 
     @property
     def ssh_private_key(self) -> str | None:
@@ -112,7 +110,7 @@ def ssh_known_hosts(self) -> str | None:
 
 class PullBasedConfig(BaseModel):
     repositories: dict[str, PullBasedRepository]
-    poll_spread: int | None
+    poll_spread: int | None = None
 
 
 class GitHubLegacyConfig(BaseModel):
@@ -263,28 +261,32 @@ class WorkerConfig(BaseModel):
 
 class BuildbotNixConfig(BaseModel):
     db_url: str
-    auth_backend: AuthBackendConfig
-    gitea: GiteaConfig | None
-    github: GitHubConfig | None
-    pull_based: PullBasedConfig | None
-    admins: list[str]
-    workers_file: Path
     build_systems: list[str]
-    eval_max_memory_size: int
-    eval_worker_count: int | None
-    nix_workers_secret_file: Path = Field(default=Path("buildbot-nix-workers"))
     domain: str
     webhook_base_url: str
-    use_https: bool
-    outputs_path: Path | None
     url: str
-    post_build_steps: list[PostBuildStep]
-    job_report_limit: int | None
-    http_basic_auth_password_file: Path | None
-    effects_per_repo_secrets: dict[str, str]
-    branches: BranchConfigDict
+
+    use_https: bool = False
+    auth_backend: AuthBackendConfig = AuthBackendConfig.none
+    eval_max_memory_size: int = 4096
+    admins: list[str] = []
+    local_workers: int = 0
+    eval_worker_count: int | None = None
+    gitea: GiteaConfig | None = None
+    github: GitHubConfig | None = None
+    pull_based: PullBasedConfig | None
+    outputs_path: Path | None = None
+    post_build_steps: list[PostBuildStep] = []
+    job_report_limit: int | None = None
+    http_basic_auth_password_file: Path | None = None
+    branches: BranchConfigDict = BranchConfigDict({})
+
+    nix_workers_secret_file: Path | None = None
+    effects_per_repo_secrets: dict[str, str] = {}
 
     def nix_worker_secrets(self) -> WorkerConfig:
+        if self.nix_workers_secret_file is None:
+            return WorkerConfig(workers=[])
         try:
             data = json.loads(read_secret_file(self.nix_workers_secret_file))
         except json.JSONDecodeError as e:

buildbot_nix/pyproject.toml

@@ -4,19 +4,18 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "buildbot-nix"
+license = "MIT"
 authors = [
     { name = "Jörg Thalheim", email = "joerg@thalheim.io" },
 ]
 description = "A nixos module to make buildbot a proper Nix-CI."
 readme = "README.rst"
 requires-python = ">=3.9"
-license = {text = "MIT"}
 classifiers = [
     "Programming Language :: Python :: 3",
     "Development Status :: 5 - Production/Stable",
     "Environment :: Console",
     "Topic :: Utilities",
-    "License :: OSI Approved :: MIT License",
     "Operating System :: OS Independent",
     "Programming Language :: Python"
 ]

nixosModules/master.nix

@@ -13,7 +13,7 @@ let
     name = "interpolate";
 
     description = ''
-      A type represnting a Buildbot interpolation string, supports interpolations like `result-%(prop:attr)s`.
+      A type representing a Buildbot interpolation string, supports interpolations like `result-%(prop:attr)s`.
     '';
 
     check = x: x ? "_type" && x._type == "interpolate" && x ? "value";
@@ -453,7 +453,7 @@ in
           type = lib.types.addCheck lib.types.int (x: x > 0);
           default = 60;
           description = ''
-            How often to poll each repository by default expressed in seconds. This value can be overriden
+            How often to poll each repository by default expressed in seconds. This value can be overridden
             per repository.
           '';
         };
@@ -462,7 +462,7 @@ in
           type = lib.types.nullOr lib.types.int;
           default = null;
           description = ''
-            If non-null and non-zero pulls will be randomly spread apart up to the specificied
+            If non-null and non-zero pulls will be randomly spread apart up to the specified
             number of seconds. Can be used to avoid a thundering herd situation.
           '';
         };
@@ -472,7 +472,7 @@ in
           default = null;
           description = ''
             If non-null the specified SSH key will be used to fetch all configured repositories.
-            This option is overriden by the per-repository `sshPrivateKeyFile` option.
+            This option is overridden by the per-repository `sshPrivateKeyFile` option.
           '';
         };
 
@@ -481,7 +481,7 @@ in
           default = null;
           description = ''
             If non-null the specified known hosts file will be matched against when connecting to
-            repositories over SSH. This option is overriden by the per-repository `sshKnownHostsFile`
+            repositories over SSH. This option is overridden by the per-repository `sshKnownHostsFile`
             option.
           '';
         };
@@ -764,7 +764,6 @@ in
                       poll_spread = cfg.pullBased.pollSpread;
                     };
                 admins = cfg.admins;
-                workers_file = cfg.workersFile;
                 build_systems = cfg.buildSystems;
                 eval_max_memory_size = cfg.evalMaxMemorySize;
                 eval_worker_count = cfg.evalWorkerCount;
@@ -781,6 +780,7 @@ in
                   value = "effects-secret__${cleanUpRepoName name}";
                 }) cfg.effects.perRepoSecretFiles;
                 branches = cfg.branches;
+                nix_workers_secret_file = "buildbot-nix-workers";
               }
             }").read_text()))
           )

nixosModules/packages.nix

@@ -18,15 +18,7 @@ in
 
     buildbot = lib.mkOption {
       type = lib.types.package;
-      default = pkgs.buildbot.overrideAttrs (o: {
-        patches = o.patches ++ [
-          (pkgs.fetchpatch {
-            name = "add-Build.skipBuildIf.patch";
-            url = "https://github.com/buildbot/buildbot/commit/f08eeef96e15c686a4f6ad52368ad08246314751.patch";
-            hash = "sha256-ACPYXMbjIfw02gsKwmDKIIZkGSxxLWCaW7ceEcgbtIU=";
-          })
-        ];
-      });
+      default = pkgs.callPackage ../packages/buildbot.nix { };
     };
 
     buildbot-worker = lib.mkOption {

packages/buildbot-dev.nix

@@ -0,0 +1,36 @@
+{
+  writeShellScriptBin,
+  python,
+  buildbot,
+  buildbot-worker,
+  buildbot-nix,
+  buildbot-gitea,
+  buildbot-effects,
+  buildbot-plugins,
+}:
+let
+  pythonEnv = python.withPackages (ps: [
+    ps.twisted
+    (ps.toPythonModule buildbot)
+    (ps.toPythonModule buildbot-worker)
+    buildbot-nix
+    buildbot-gitea
+    buildbot-effects
+    buildbot-plugins.www
+  ]);
+in
+writeShellScriptBin "buildbot-dev" ''
+  set -xeuo pipefail
+  git_root=$(git rev-parse --show-toplevel)
+  mkdir -p "$git_root/.buildbot-dev"
+  cd "$git_root/.buildbot-dev"
+  "${pythonEnv}/bin/buildbot" create-master .
+  #if [ ! -f master.cfg ]; then
+  install -m600 ${./master.cfg.py} master.cfg
+  #fi
+  echo > $git_root/.buildbot-dev/twistd.log
+  tail -f $git_root/.buildbot-dev/twistd.log &
+  tail_pid=$!
+  trap 'kill $tail_pid' EXIT
+  "${pythonEnv}/bin/buildbot" start --nodaemon .
+''

packages/buildbot.nix

@@ -0,0 +1,10 @@
+{ buildbot, fetchpatch }:
+buildbot.overrideAttrs (o: {
+  patches = o.patches ++ [
+    (fetchpatch {
+      name = "add-Build.skipBuildIf.patch";
+      url = "https://github.com/buildbot/buildbot/commit/f08eeef96e15c686a4f6ad52368ad08246314751.patch";
+      hash = "sha256-ACPYXMbjIfw02gsKwmDKIIZkGSxxLWCaW7ceEcgbtIU=";
+    })
+  ];
+})

packages/flake-module.nix

@@ -3,13 +3,23 @@
     {
       lib,
       pkgs,
+      self',
       ...
     }:
     {
       packages =
         {
           # useful for checking what buildbot version is used.
-          buildbot = pkgs.buildbot;
+          buildbot = pkgs.callPackage ./buildbot.nix { };
+          buildbot-dev = pkgs.python3.pkgs.callPackage ./buildbot-dev.nix {
+            inherit (self'.packages)
+              buildbot-gitea
+              buildbot-effects
+              buildbot-nix
+              buildbot
+              ;
+            inherit (pkgs) buildbot-worker buildbot-plugins;
+          };
           buildbot-nix = pkgs.python3.pkgs.callPackage ./buildbot-nix.nix { };
           buildbot-gitea = pkgs.python3.pkgs.callPackage ./buildbot-gitea.nix {
             buildbot = pkgs.buildbot;

packages/master.cfg.py

@@ -0,0 +1,42 @@
+from pathlib import Path
+from buildbot_nix import (
+    NixConfigurator,
+    BuildbotNixConfig,
+)
+from buildbot.process.factory import BuildFactory
+
+factory = BuildFactory()
+
+STATE_DIR = Path(".")
+PORT = 8012
+
+buildbot_nix_config = BuildbotNixConfig(
+    db_url="sqlite:///state.sqlite",
+    pull_based=dict(
+        repositories={
+            "buildbot-nix": dict(
+                name="buildbot-nix",
+                url="https://github.com/nix-community/buildbot-nix",
+                default_branch="main",
+            )
+        },
+        poll_spread=None,
+    ),
+    build_systems=["x86_64-linux"],
+    eval_max_memory_size=4096,
+    eval_worker_count=4,
+    local_workers=4,
+    domain="localhost",
+    webhook_base_url=f"http://localhost:{PORT}",
+    url=f"http://localhost:{PORT}",
+)
+
+c = BuildmasterConfig = dict(
+    title="Hello World CI",
+    titleURL="https://buildbot.github.io/hello-world/",
+    configurators=[
+        NixConfigurator(buildbot_nix_config),
+    ],
+    protocols={"pb": {"port": "tcp:9989:interface=\\:\\:1"}},
+    www=dict(port=PORT, plugins=dict()),
+)