From 9ced02242a48704b830ec0372e4d0bc37b04f7f3 Mon Sep 17 00:00:00 2001 From: Sam Willcocks Date: Mon, 14 Apr 2025 16:59:19 +0100 Subject: [PATCH 1/9] gitlab: add gitlab integration --- buildbot_nix/buildbot_nix/__init__.py | 4 + buildbot_nix/buildbot_nix/gitlab_project.py | 319 ++++++++++++++++++++ buildbot_nix/buildbot_nix/models.py | 28 ++ nixosModules/master.nix | 87 +++++- 4 files changed, 435 insertions(+), 3 deletions(-) create mode 100644 buildbot_nix/buildbot_nix/gitlab_project.py diff --git a/buildbot_nix/buildbot_nix/__init__.py b/buildbot_nix/buildbot_nix/__init__.py index 3292bcf50..23eb9239e 100644 --- a/buildbot_nix/buildbot_nix/__init__.py +++ b/buildbot_nix/buildbot_nix/__init__.py @@ -25,6 +25,7 @@ from buildbot.www.authz import Authz from buildbot.www.authz.endpointmatchers import EndpointMatcherBase, Match +from buildbot_nix.gitlab_project import GitlabBackend from buildbot_nix.pull_based.backend import PullBasedBacked if TYPE_CHECKING: @@ -1270,6 +1271,9 @@ def configure(self, config: dict[str, Any]) -> None: if self.config.gitea is not None: backends["gitea"] = GiteaBackend(self.config.gitea, self.config.url) + if self.config.gitlab is not None: + backends["gitlab"] = GitlabBackend(self.config.gitlab, self.config.url) + if self.config.pull_based is not None: backends["pull_based"] = PullBasedBacked(self.config.pull_based) diff --git a/buildbot_nix/buildbot_nix/gitlab_project.py b/buildbot_nix/buildbot_nix/gitlab_project.py new file mode 100644 index 000000000..41c915588 --- /dev/null +++ b/buildbot_nix/buildbot_nix/gitlab_project.py @@ -0,0 +1,319 @@ +import os +import signal +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +from buildbot.changes.base import ChangeSource +from buildbot.config.builder import BuilderConfig +from buildbot.plugins import util +from buildbot.reporters.base import ReporterBase +from buildbot.reporters.gitlab import GitLabStatusPush +from buildbot.www.auth import AuthBase +from buildbot.www.avatar import AvatarBase +from pydantic import BaseModel +from twisted.logger import Logger +from twisted.python import log + +from buildbot_nix.common import ( + ThreadDeferredBuildStep, + atomic_write_file, + filter_repos_by_topic, + http_request, + model_dump_project_cache, + model_validate_project_cache, + paginated_github_request, + slugify_project_name, +) +from buildbot_nix.models import GitlabConfig, Interpolate +from buildbot_nix.nix_status_generator import BuildNixEvalStatusGenerator +from buildbot_nix.projects import GitBackend, GitProject + +tlog = Logger() + + +class NamespaceData(BaseModel): + path: str + kind: str + + +class RepoData(BaseModel): + id: int + name_with_namespace: str + path: str + path_with_namespace: str + ssh_url_to_repo: str + web_url: str + namespace: NamespaceData + default_branch: str + topics: list[str] + + +class GitlabProject(GitProject): + config: GitlabConfig + data: RepoData + + def __init__(self, config: GitlabConfig, data: RepoData) -> None: + self.config = config + self.data = data + + def get_project_url(self) -> str: + url = urlparse(self.config.instance_url) + return f"{url.scheme}://git:%(secret:{self.config.token_file})s@{url.hostname}/{self.data.path_with_namespace}" + + def create_change_source(self) -> ChangeSource | None: + return None + + @property + def pretty_type(self) -> str: + return "Gitlab" + + @property + def type(self) -> str: + return "gitlab" + + @property + def nix_ref_type(self) -> str: + return "gitlab" + + @property + def repo(self) -> str: + return self.data.path + + @property + def owner(self) -> str: + return self.data.namespace.path + + @property + def name(self) -> str: + return self.data.name_with_namespace + + @property + def url(self) -> str: + return self.data.web_url + + @property + def project_id(self) -> str: + return slugify_project_name(self.data.path_with_namespace) + + @property + def default_branch(self) -> str: + return self.data.default_branch + + @property + def topics(self) -> list[str]: + return self.data.topics + + @property + def belongs_to_org(self) -> bool: + return self.data.namespace.kind == "group" + + @property + def private_key_path(self) -> Path | None: + return None + + @property + def known_hosts_path(self) -> Path | None: + return None + + +class GitlabBackend(GitBackend): + config: GitlabConfig + instance_url: str + + def __init__(self, config: GitlabConfig, instance_url: str) -> None: + self.config = config + self.instance_url = instance_url + + def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig: + factory = util.BuildFactory() + factory.addStep( + ReloadGitlabProjects(self.config, self.config.project_cache_file), + ) + factory.addStep( + CreateGitlabProjectHooks( + self.config, + self.instance_url, + ) + ) + return util.BuilderConfig( + name=self.reload_builder_name, + workernames=worker_names, + factory=factory, + ) + + def create_reporter(self) -> ReporterBase: + return GitLabStatusPush( + token=Interpolate(self.config.token), + context=Interpolate("buildbot/%(prop:status_name)s"), + baseURL=self.config.instance_url, + generators=[ + BuildNixEvalStatusGenerator(), + ], + ) + + def create_change_hook(self) -> dict[str, Any]: + return dict(secret=self.config.webhook_secret) + + def load_projects(self) -> list["GitProject"]: + if not self.config.project_cache_file.exists(): + return [] + + repos: list[RepoData] = filter_repos_by_topic( + self.config.topic, + sorted( + model_validate_project_cache(RepoData, self.config.project_cache_file), + key=lambda repo: repo.path_with_namespace, + ), + lambda repo: repo.topics, + ) + tlog.info(f"Loading {len(repos)} cached repos.") + + return [ + GitlabProject(self.config, RepoData.model_validate(repo)) for repo in repos + ] + + def are_projects_cached(self) -> bool: + return self.config.project_cache_file.exists() + + def create_auth(self) -> AuthBase: + raise NotImplementedError + + def create_avatar_method(self) -> AvatarBase | None: + return None + + @property + def reload_builder_name(self) -> str: + return "reload-gitlab-projects" + + @property + def type(self) -> str: + return "gitlab" + + @property + def pretty_type(self) -> str: + return "Gitlab" + + @property + def change_hook_name(self) -> str: + return "gitlab" + + +class ReloadGitlabProjects(ThreadDeferredBuildStep): + name = "reload_gitlab_projects" + + config: GitlabConfig + project_cache_file: Path + + def __init__( + self, + config: GitlabConfig, + project_cache_file: Path, + **kwargs: Any, + ) -> None: + self.config = config + self.project_cache_file = project_cache_file + super().__init__(**kwargs) + + def run_deferred(self) -> None: + repos: list[RepoData] = filter_repos_by_topic( + self.config.topic, + refresh_projects(self.config, self.project_cache_file), + lambda repo: repo.topics, + ) + atomic_write_file(self.project_cache_file, model_dump_project_cache(repos)) + + def run_post(self) -> Any: + return util.SUCCESS + + +class CreateGitlabProjectHooks(ThreadDeferredBuildStep): + name = "create_gitlab_project_hooks" + + config: GitlabConfig + instance_url: str + + def __init__(self, config: GitlabConfig, instance_url: str, **kwargs: Any) -> None: + self.config = config + self.instance_url = instance_url + super().__init__(**kwargs) + + def run_deferred(self) -> None: + repos = model_validate_project_cache(RepoData, self.config.project_cache_file) + for repo in repos: + create_project_hook( + token=self.config.token, + webhook_secret=self.config.webhook_secret, + project_id=repo.id, + gitlab_url=self.config.instance_url, + instance_url=self.instance_url, + ) + + def run_post(self) -> Any: + os.kill(os.getpid(), signal.SIGHUP) + return util.SUCCESS + + +def refresh_projects(config: GitlabConfig, cache_file: Path) -> list[RepoData]: + # access level 40 == Maintainer. See https://docs.gitlab.com/api/members/#roles + return [ + RepoData.model_validate(repo) + for repo in paginated_github_request( + f"{config.instance_url}/api/v4/projects?min_access_level=40&pagination=keyset&per_page=100&order_by=id&sort=asc", + config.token, + ) + ] + + +def create_project_hook( + token: str, + webhook_secret: str, + project_id: int, + gitlab_url: str, + instance_url: str, +) -> None: + hook_url = instance_url + "change_hook/gitlab" + for hook in paginated_github_request( + f"{gitlab_url}/api/v4/projects/{project_id}/hooks", + token, + ): + if hook["url"] == hook_url: + log.msg(f"hook for gitlab project {project_id} already exists") + return + log.msg(f"creating hook for gitlab project {project_id}") + http_request( + f"{gitlab_url}/api/v4/projects/{project_id}/hooks", + method="POST", + headers={ + "Authorization": f"Bearer {token}", + "Accept": "application/json", + "Content-Type": "application/json", + }, + data=dict( + name="buildbot hook", + url=hook_url, + enable_ssl_verification=True, + token=webhook_secret, + # We don't need to be informed of most events + confidential_issues_events=False, + confidential_note_events=False, + deployment_events=False, + feature_flag_events=False, + issues_events=False, + job_events=False, + merge_requests_events=False, + note_events=False, + pipeline_events=False, + releases_events=False, + wiki_page_events=False, + resource_access_token_events=False, + ), + ) + + +if __name__ == "__main__": + c = GitlabConfig( + topic=None, + ) + + print(refresh_projects(c, Path("deleteme-gitlab-cache"))) diff --git a/buildbot_nix/buildbot_nix/models.py b/buildbot_nix/buildbot_nix/models.py index 0b5913bf5..a237d3a17 100644 --- a/buildbot_nix/buildbot_nix/models.py +++ b/buildbot_nix/buildbot_nix/models.py @@ -165,6 +165,33 @@ def oauth_secret(self) -> str: return read_secret_file(self.oauth_secret_file) +class GitlabConfig(BaseModel): + instance_url: str = Field(default="https://gitlab.com") + topic: str | None + + token_file: Path = Field(default=Path("gitlab-token")) + webhook_secret_file: Path = Field(default=Path("gitlab-webhook-secret")) + + oauth_id: str | None + oauth_secret_file: Path | None + + project_cache_file: Path = Field(default=Path("gitlab-project-cache.json")) + + @property + def token(self) -> str: + return read_secret_file(self.token_file) + + @property + def webhook_secret(self) -> str: + return read_secret_file(self.webhook_secret_file) + + @property + def oauth_secret(self) -> str: + if self.oauth_secret_file is None: + raise InternalError + return read_secret_file(self.oauth_secret_file) + + class PostBuildStep(BaseModel): name: str environment: Mapping[str, str | Interpolate] @@ -273,6 +300,7 @@ class BuildbotNixConfig(BaseModel): eval_worker_count: int | None = None gitea: GiteaConfig | None = None github: GitHubConfig | None = None + gitlab: GitlabConfig | None pull_based: PullBasedConfig | None outputs_path: Path | None = None post_build_steps: list[PostBuildStep] = [] diff --git a/nixosModules/master.nix b/nixosModules/master.nix index 595d982c1..992b4eae4 100644 --- a/nixosModules/master.nix +++ b/nixosModules/master.nix @@ -219,6 +219,7 @@ in type = lib.types.enum [ "gitea" "github" + "gitlab" ]; }; @@ -277,6 +278,42 @@ in }; }; + gitlab = { + enable = lib.mkEnableOption "Enable Gitlab integration"; + tokenFile = lib.mkOption { + type = lib.types.path; + description = "Gitlab token file"; + }; + webhookSecretFile = lib.mkOption { + type = lib.types.path; + description = "Gitlab webhook secret file"; + }; + instanceUrl = lib.mkOption { + type = lib.types.str; + description = "Gitlab instance url"; + example = "https://gitlab.example.com"; + default = "https://gitlab.com"; + }; + topic = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = "build-with-buildbot"; + description = '' + Projects that have this topic will be built by buildbot. + If null, all projects that the buildbot Gitea user has access to, are built. + ''; + }; + oauthId = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = "Gitlab oauth id"; + }; + oauthSecretFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = "Gitlab oauth secret file"; + }; + }; + gitea = { enable = lib.mkEnableOption "Enable Gitea integration" // { default = cfg.authBackend == "gitea"; @@ -665,6 +702,11 @@ in cfg.authBackend == "gitea" -> (cfg.gitea.oauthId != null && cfg.gitea.oauthSecretFile != null); message = ''config.services.buildbot-nix.master.authBackend is set to "gitea", then config.services.buildbot-nix.master.gitea.oauthId and config.services.buildbot-nix.master.gitea.oauthSecretFile have to be set.''; } + { + assertion = + cfg.authBackend == "gitlab" -> (cfg.gitlab.oauthId != null && cfg.gitlab.oauthSecretFile != null); + message = ''config.services.buildbot-nix.master.authBackend is set to "gitlab", then config.services.buildbot-nix.master.gitlab.oauthId and config.services.buildbot-nix.master.gitlab.oauthSecretFile have to be set.''; + } { assertion = cfg.authBackend == "github" -> cfg.github.enable; message = '' @@ -677,6 +719,12 @@ in If `cfg.authBackend` is set to `"gitea"` the GitHub backend must be enabled with `cfg.gitea.enable`; ''; } + { + assertion = cfg.authBackend == "gitlab" -> cfg.gitlab.enable; + message = '' + If `cfg.authBackend is set to `"gitlab"` the Gitlab backend must be enabled with `cfg.gitlab.enable`; + ''; + } ]; services.buildbot-master = { @@ -707,6 +755,19 @@ in (pkgs.formats.json { }).generate "buildbot-nix-config.json" { db_url = cfg.dbUrl; auth_backend = cfg.authBackend; + gitlab = + if !cfg.gitlab.enable then + null + else + { + instance_url = cfg.gitlab.instanceUrl; + topic = cfg.gitea.topic; + oauth_id = cfg.gitlab.oauthId; + oauth_secret_file = "gitlab-oauth-secret"; + token_file = "gitlab-token"; + webhook_secret_file = "gitlab-webhook-secret"; + project_cache_file = "gitlab-project-cache.json"; + }; gitea = if !cfg.gitea.enable then null @@ -821,10 +882,15 @@ in ) ++ lib.optional (cfg.authBackend == "gitea") "gitea-oauth-secret:${cfg.gitea.oauthSecretFile}" ++ lib.optional (cfg.authBackend == "github") "github-oauth-secret:${cfg.github.oauthSecretFile}" + ++ lib.optional (cfg.authBackend == "gitlab") "gitlab-oauth-secret:${cfg.gitlab.oauthSecretFile}" ++ lib.optionals cfg.gitea.enable [ "gitea-token:${cfg.gitea.tokenFile}" "gitea-webhook-secret:${cfg.gitea.webhookSecretFile}" ] + ++ lib.optionals cfg.gitlab.enable [ + "gitlab-token:${cfg.gitlab.tokenFile}" + "gitlab-webhook-secret:${cfg.gitlab.webhookSecretFile}" + ] ++ lib.mapAttrsToList ( repoName: path: "effects-secret__${cleanUpRepoName repoName}:${path}" ) cfg.effects.perRepoSecretFiles @@ -907,15 +973,26 @@ in ]; } (lib.mkIf (cfg.authBackend == "httpbasicauth") { set-basic-auth = true; }) + (lib.mkIf + (lib.elem cfg.accessMode.fullyPrivate.backend [ + "github" + "gitea" + "gitlab" + ]) + { + email-domain = "*"; + } + ) (lib.mkIf (lib.elem cfg.accessMode.fullyPrivate.backend [ "github" "gitea" ]) { + # https://github.com/oauth2-proxy/oauth2-proxy/issues/1724 + scope = "read:user user:email repo"; github-user = lib.concatStringsSep "," (cfg.accessMode.fullyPrivate.users ++ cfg.admins); github-team = cfg.accessMode.fullyPrivate.teams; - email-domain = "*"; } ) (lib.mkIf (cfg.accessMode.fullyPrivate.backend == "github") { provider = "github"; }) @@ -926,6 +1003,12 @@ in redeem-url = "${cfg.gitea.instanceUrl}/login/oauth/access_token"; validate-url = "${cfg.gitea.instanceUrl}/api/v1/user/emails"; }) + (lib.mkIf (cfg.accessMode.fullyPrivate.backend == "gitlab") { + provider = "gitlab"; + provider-display-name = "Gitlab"; + gitlab-groups = cfg.accessMode.fullyPrivate.teams; + oidc-issuer-url = cfg.gitlab.instanceUrl; + }) ]; }; @@ -938,8 +1021,6 @@ in client_secret = "$(cat ${cfg.accessMode.fullyPrivate.clientSecretFile})" cookie_secret = "$(cat ${cfg.accessMode.fullyPrivate.cookieSecretFile})" basic_auth_password = "$(cat ${cfg.httpBasicAuthPasswordFile})" - # https://github.com/oauth2-proxy/oauth2-proxy/issues/1724 - scope = "read:user user:email repo" EOF ''; }; From 35fbd39a8e0fb734b51b0a68e2ab2e77590a9503 Mon Sep 17 00:00:00 2001 From: Sam Willcocks Date: Tue, 15 Apr 2025 13:11:38 +0100 Subject: [PATCH 2/9] update link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4b28e5c1a..4df745aa8 100644 --- a/README.md +++ b/README.md @@ -320,7 +320,7 @@ The following instances run on GitHub: [Configuration](https://github.com/nix-community/infra/tree/master/modules/nixos) | [Instance](https://buildbot.nix-community.org/) - [**Mic92's dotfiles**](https://github.com/Mic92/dotfiles): - [Configuration](https://github.com/Mic92/dotfiles/blob/main/nixos/eve/modules/buildbot.nix) + [Configuration](https://github.com/Mic92/dotfiles/blob/main/machines/eve/modules/buildbot.nix) | [Instance](https://buildbot.thalheim.io/) - [**Technical University Munich**](https://dse.in.tum.de/): [Configuration](https://github.com/TUM-DSE/doctor-cluster-config/tree/master/modules/buildbot) From c1ce284ef2ecc8cb5ad940793f689b2c677e2715 Mon Sep 17 00:00:00 2001 From: Sam Willcocks Date: Wed, 16 Apr 2025 14:36:08 +0100 Subject: [PATCH 3/9] use a proper token when creating statuses --- buildbot_nix/buildbot_nix/gitlab_project.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/buildbot_nix/buildbot_nix/gitlab_project.py b/buildbot_nix/buildbot_nix/gitlab_project.py index 41c915588..a33d1b976 100644 --- a/buildbot_nix/buildbot_nix/gitlab_project.py +++ b/buildbot_nix/buildbot_nix/gitlab_project.py @@ -144,7 +144,7 @@ def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig: def create_reporter(self) -> ReporterBase: return GitLabStatusPush( - token=Interpolate(self.config.token), + token=self.config.token, context=Interpolate("buildbot/%(prop:status_name)s"), baseURL=self.config.instance_url, generators=[ @@ -309,11 +309,3 @@ def create_project_hook( resource_access_token_events=False, ), ) - - -if __name__ == "__main__": - c = GitlabConfig( - topic=None, - ) - - print(refresh_projects(c, Path("deleteme-gitlab-cache"))) From c32c8f331720aaacf17f47c6172e64c38bc8e5bf Mon Sep 17 00:00:00 2001 From: Sam Willcocks Date: Wed, 16 Apr 2025 15:19:34 +0100 Subject: [PATCH 4/9] maybe fix commit triggers --- buildbot_nix/buildbot_nix/gitlab_project.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/buildbot_nix/buildbot_nix/gitlab_project.py b/buildbot_nix/buildbot_nix/gitlab_project.py index a33d1b976..b502e68e3 100644 --- a/buildbot_nix/buildbot_nix/gitlab_project.py +++ b/buildbot_nix/buildbot_nix/gitlab_project.py @@ -39,6 +39,7 @@ class NamespaceData(BaseModel): class RepoData(BaseModel): id: int + name: str name_with_namespace: str path: str path_with_namespace: str @@ -86,7 +87,12 @@ def owner(self) -> str: @property def name(self) -> str: - return self.data.name_with_namespace + # This needs to match what buildbot uses in change hooks to map an incoming change + # to a project: https://github.com/buildbot/buildbot/blob/master/master/buildbot/www/hooks/gitlab.py#L45 + # I suspect this will result in clashes if you have identically-named projects in + # different namespaces, as is totally valid in gitlab. + # Using self.data.name_with_namespace would be more robust. + return self.data.name @property def url(self) -> str: From 9e7ec4573677b539c9df4b6cd386788aa924c968 Mon Sep 17 00:00:00 2001 From: Sam Willcocks Date: Wed, 23 Apr 2025 13:47:14 +0100 Subject: [PATCH 5/9] gitlab: add avatar method --- buildbot_nix/buildbot_nix/gitlab_project.py | 101 +++++++++++++++++++- 1 file changed, 99 insertions(+), 2 deletions(-) diff --git a/buildbot_nix/buildbot_nix/gitlab_project.py b/buildbot_nix/buildbot_nix/gitlab_project.py index b502e68e3..4ef4415f5 100644 --- a/buildbot_nix/buildbot_nix/gitlab_project.py +++ b/buildbot_nix/buildbot_nix/gitlab_project.py @@ -1,17 +1,21 @@ import os import signal +from collections.abc import Generator from pathlib import Path from typing import Any -from urllib.parse import urlparse +from urllib.parse import urlencode, urlparse from buildbot.changes.base import ChangeSource from buildbot.config.builder import BuilderConfig from buildbot.plugins import util from buildbot.reporters.base import ReporterBase from buildbot.reporters.gitlab import GitLabStatusPush +from buildbot.util import httpclientservice +from buildbot.www import resource from buildbot.www.auth import AuthBase from buildbot.www.avatar import AvatarBase from pydantic import BaseModel +from twisted.internet import defer from twisted.logger import Logger from twisted.python import log @@ -186,7 +190,7 @@ def create_auth(self) -> AuthBase: raise NotImplementedError def create_avatar_method(self) -> AvatarBase | None: - return None + return AvatarGitlab(config=self.config) @property def reload_builder_name(self) -> str: @@ -260,6 +264,99 @@ def run_post(self) -> Any: return util.SUCCESS +class AvatarGitlab(AvatarBase): + name = "gitlab" + + config: GitlabConfig + + def __init__( + self, + config: GitlabConfig, + debug: bool = False, + verify: bool = True, + ) -> None: + self.config = config + self.debug = debug + self.verify = verify + + self.master = None + self.client: httpclientservice.HTTPSession | None = None + + def _get_http_client( + self, + ) -> httpclientservice.HTTPSession: + if self.client is not None: + return self.client + + headers = { + "User-Agent": "Buildbot", + "Authorization": f"Bearer {self.config.token}", + } + + self.client = httpclientservice.HTTPSession( + self.master.httpservice, # type: ignore[attr-defined] + self.config.instance_url, + headers=headers, + debug=self.debug, + verify=self.verify, + ) + + return self.client + + @defer.inlineCallbacks + def getUserAvatar( # noqa: N802 + self, + email: str, + username: str | None, + size: str | int, + defaultAvatarUrl: str, # noqa: N803 + ) -> Generator[defer.Deferred, str | None, None]: + if isinstance(size, int): + size = str(size) + avatar_url = None + if username is not None: + avatar_url = yield self._get_avatar_by_username(username) + if avatar_url is None: + avatar_url = yield self._get_avatar_by_email(email, size) + if not avatar_url: + avatar_url = defaultAvatarUrl + raise resource.Redirect(avatar_url) + + @defer.inlineCallbacks + def _get_avatar_by_username( + self, username: str + ) -> Generator[defer.Deferred, Any, str | None]: + qs = urlencode(dict(username=username)) + http = self._get_http_client() + users = yield http.get(f"/api/v4/users?{qs}") + users = yield users.json() + if len(users) == 1: + return users[0]["avatar_url"] + if len(users) > 1: + # TODO: log warning + ... + return None + + @defer.inlineCallbacks + def _get_avatar_by_email( + self, email: str, size: str | None + ) -> Generator[defer.Deferred, Any, str | None]: + http = self._get_http_client() + q = dict(email=email) + if size is not None: + q["size"] = size + qs = urlencode(q) + res = yield http.get(f"/api/v4/avatar?{qs}") + data = yield res.json() + # N.B: Gitlab's public avatar endpoint returns a gravatar url if there isn't an + # account with a matching public email - so it should always return *something*. + # See: https://docs.gitlab.com/api/avatar/#get-details-on-an-account-avatar + if "avatar_url" in data: + return data["avatar_url"] + else: + return None + + def refresh_projects(config: GitlabConfig, cache_file: Path) -> list[RepoData]: # access level 40 == Maintainer. See https://docs.gitlab.com/api/members/#roles return [ From 1db49183a80435ac5b9c32cd77b198d9f502b393 Mon Sep 17 00:00:00 2001 From: Sam Willcocks Date: Wed, 23 Apr 2025 15:40:03 +0100 Subject: [PATCH 6/9] gitlab: use correct url to fix build triggers --- buildbot_nix/buildbot_nix/gitlab_project.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/buildbot_nix/buildbot_nix/gitlab_project.py b/buildbot_nix/buildbot_nix/gitlab_project.py index 4ef4415f5..d289fdc33 100644 --- a/buildbot_nix/buildbot_nix/gitlab_project.py +++ b/buildbot_nix/buildbot_nix/gitlab_project.py @@ -100,7 +100,10 @@ def name(self) -> str: @property def url(self) -> str: - return self.data.web_url + # Not `web_url`: the buildbot gitlab hook dialect uses repository.url which seems + # to be the ssh url in practice. + # See: https://github.com/buildbot/buildbot/blob/master/master/buildbot/www/hooks/gitlab.py#L271 + return self.data.ssh_url_to_repo @property def project_id(self) -> str: From b2d7b40176d105ee198278beecd70ce803cd5386 Mon Sep 17 00:00:00 2001 From: Sam Willcocks Date: Wed, 23 Apr 2025 15:40:28 +0100 Subject: [PATCH 7/9] enable mypy on all unixes --- formatter/flake-module.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/formatter/flake-module.nix b/formatter/flake-module.nix index 6e90da5cd..3b29e8ec6 100644 --- a/formatter/flake-module.nix +++ b/formatter/flake-module.nix @@ -18,7 +18,7 @@ ]; programs.mypy = { - enable = pkgs.stdenv.buildPlatform.isLinux; + enable = pkgs.stdenv.buildPlatform.isUnix; package = pkgs.buildbot.python.pkgs.mypy; directories."." = { modules = [ @@ -37,7 +37,7 @@ # the mypy module adds `./buildbot_nix/**/*.py` which does not appear to work # furthermore, saying `directories.""` will lead to `/buildbot_nix/**/*.py` which # is obviously incorrect... - settings.formatter."mypy-" = lib.mkIf pkgs.stdenv.buildPlatform.isLinux { + settings.formatter."mypy-" = lib.mkIf pkgs.stdenv.buildPlatform.isUnix { includes = [ "buildbot_nix/**/*.py" ]; }; settings.formatter.ruff-check.priority = 1; From bdf18640643ac1c70f351012b359030575ffcd4c Mon Sep 17 00:00:00 2001 From: Sam Willcocks Date: Wed, 23 Apr 2025 16:03:27 +0100 Subject: [PATCH 8/9] gitlab: fix commit status description --- buildbot_nix/buildbot_nix/gitlab_project.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildbot_nix/buildbot_nix/gitlab_project.py b/buildbot_nix/buildbot_nix/gitlab_project.py index d289fdc33..d67085558 100644 --- a/buildbot_nix/buildbot_nix/gitlab_project.py +++ b/buildbot_nix/buildbot_nix/gitlab_project.py @@ -8,6 +8,7 @@ from buildbot.changes.base import ChangeSource from buildbot.config.builder import BuilderConfig from buildbot.plugins import util +from buildbot.process.properties import Interpolate from buildbot.reporters.base import ReporterBase from buildbot.reporters.gitlab import GitLabStatusPush from buildbot.util import httpclientservice @@ -29,7 +30,7 @@ paginated_github_request, slugify_project_name, ) -from buildbot_nix.models import GitlabConfig, Interpolate +from buildbot_nix.models import GitlabConfig from buildbot_nix.nix_status_generator import BuildNixEvalStatusGenerator from buildbot_nix.projects import GitBackend, GitProject From a1167e2a86dd8dc210fc4fd1446ad18224c63178 Mon Sep 17 00:00:00 2001 From: Sam Willcocks Date: Mon, 19 May 2025 14:15:42 +0100 Subject: [PATCH 9/9] fix pyproject license metadata --- buildbot_nix/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildbot_nix/pyproject.toml b/buildbot_nix/pyproject.toml index 8c26fa77e..5e9aa2ff5 100644 --- a/buildbot_nix/pyproject.toml +++ b/buildbot_nix/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "buildbot-nix" -license = "MIT" +license = {file = "../LICENSE.md"} authors = [ { name = "Jörg Thalheim", email = "joerg@thalheim.io" }, ]