Merge pull request #156 from MagicRB/github_app

Add GitHub App support

Author: MagicRB

README.md

@@ -66,16 +66,36 @@ We have the following two roles:
 
 ### Integration with GitHub
 
-To integrate with GitHub:
+#### GitHub App
+
+To integrate with GitHub using app authentication:
+
+1. **GitHub App**: Set up a GitHub app for Buildbot to enable GitHub user
+   authentication on the Buildbot dashboard. Enable the following permissions:
+   - Contents: Read-only
+   - Metadata: Read-only
+   - Commit statuses: Read and write
+   - Webhooks: Read and write
+2. **GitHub App private key**: Get the app private key and app ID from GitHub,
+   configure using the buildbot-nix NixOS module.
+3. **Install App**: Install the app for an organization or specific user.
+4. **Refresh GitHub Projects**: Currently buildbot-nix doesn't respond to
+   changes (new repositories or installations) automatically, it is therefore
+   necessary to manually trigger a reload or wait for the next periodic reload.
+
+#### Legacy Token Auth
+
+To integrate with GitHub using legacy token authentication:
 
 1. **GitHub Token**: Obtain a GitHub token with `admin:repo_hook` and `repo`
    permissions. For GitHub organizations, it's advisable to create a separate
    GitHub user for managing repository webhooks.
 
-#### Optional when using GitHub login
+### Optional when using GitHub login
 
 1. **GitHub App**: Set up a GitHub app for Buildbot to enable GitHub user
-   authentication on the Buildbot dashboard.
+   authentication on the Buildbot dashboard. (can be the same as for GitHub App
+   auth)
 2. **OAuth Credentials**: After installing the app, generate OAuth credentials
    and configure them in the buildbot-nix NixOS module. Set the callback url to
    `https://<your-domain>/auth/login`.

buildbot_nix/__init__.py

@@ -872,6 +872,8 @@ def configure(self, config: dict[str, Any]) -> None:
                 ],
             )
             config["services"].append(backend.create_reporter())
+            config.setdefault("secretProviders", [])
+            config["secretsProviders"].extend(backend.create_secret_providers())
 
         systemd_secrets = SecretInAFile(
             dirname=os.environ["CREDENTIALS_DIRECTORY"],
@@ -887,13 +889,15 @@ def configure(self, config: dict[str, Any]) -> None:
                 backend.create_change_hook()
             )
 
-        if "auth" not in config["www"]:
-            config["www"].setdefault("avatar_methods", [])
+        config["www"].setdefault("avatar_methods", [])
+
+        for backend in backends.values():
+            avatar_method = backend.create_avatar_method()
+            print(avatar_method)
+            if avatar_method is not None:
+                config["www"]["avatar_methods"].append(avatar_method)
 
-            for backend in backends.values():
-                avatar_method = backend.create_avatar_method()
-                if avatar_method is not None:
-                    config["www"]["avatar_methods"].append(avatar_method)
+        if "auth" not in config["www"]:
             # TODO one cannot have multiple auth backends...
             if auth is not None:
                 config["www"]["auth"] = auth

buildbot_nix/common.py

@@ -2,14 +2,18 @@
 import http.client
 import json
 import urllib.request
+from pathlib import Path
+from tempfile import NamedTemporaryFile
 from typing import Any
 
 
 def slugify_project_name(name: str) -> str:
     return name.replace(".", "-").replace("/", "-")
 
 
-def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]:
+def paginated_github_request(
+    url: str, token: str, subkey: None | str = None
+) -> list[dict[str, Any]]:
     next_url: str | None = url
     items = []
     while next_url:
@@ -29,7 +33,10 @@ def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]:
                 link_parts = link.split(";")
                 if link_parts[1].strip() == 'rel="next"':
                     next_url = link_parts[0][1:-1]
-        items += res.json()
+        if subkey is not None:
+            items += res.json()[subkey]
+        else:
+            items += res.json()
     return items
 
 
@@ -78,3 +85,15 @@ def http_request(
         msg = f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}"
         raise HttpError(msg) from e
     return HttpResponse(resp)
+
+
+def atomic_write_file(file: Path, data: str) -> None:
+    with NamedTemporaryFile("w", delete=False, dir=file.parent) as f:
+        path = Path(f.name)
+        try:
+            f.write(data)
+            f.flush()
+            path.rename(file)
+        except OSError:
+            path.unlink()
+            raise

buildbot_nix/github/__init__.py

@@ -0,0 +1 @@
+

buildbot_nix/github/auth/__init__.py

@@ -0,0 +1 @@
+

buildbot_nix/github/auth/_type.py

@@ -0,0 +1,23 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass
+class AuthType:
+    pass
+
+
+@dataclass
+class AuthTypeLegacy(AuthType):
+    token_secret_name: str = "github-token"
+
+
+@dataclass
+class AuthTypeApp(AuthType):
+    app_id: int
+    app_secret_key_name: str = "github-app-secret-key"
+    app_installation_token_map_name: Path = Path(
+        "github-app-installation-token-map.json"
+    )
+    app_project_id_map_name: Path = Path("github-app-project-id-map-name.json")
+    app_jwt_token_name: Path = Path("github-app-jwt-token")

buildbot_nix/github/installation_token.py

@@ -0,0 +1,108 @@
+import json
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+from buildbot_nix.common import (
+    HttpResponse,
+    atomic_write_file,
+    http_request,
+)
+
+from .jwt_token import JWTToken
+from .repo_token import RepoToken
+
+
+class InstallationToken(RepoToken):
+    GITHUB_TOKEN_LIFETIME: timedelta = timedelta(minutes=60)
+
+    jwt_token: JWTToken
+    installation_id: int
+
+    token: str
+    expiration: datetime
+    installations_token_map_name: Path
+
+    @staticmethod
+    def _create_installation_access_token(
+        jwt_token: JWTToken, installation_id: int
+    ) -> HttpResponse:
+        return http_request(
+            f"https://api.github.com/app/installations/{installation_id}/access_tokens",
+            data={},
+            headers={"Authorization": f"Bearer {jwt_token.get()}"},
+            method="POST",
+        )
+
+    @staticmethod
+    def _generate_token(
+        jwt_token: JWTToken, installation_id: int
+    ) -> tuple[str, datetime]:
+        token = InstallationToken._create_installation_access_token(
+            jwt_token, installation_id
+        ).json()["token"]
+        expiration = datetime.now(tz=UTC) + InstallationToken.GITHUB_TOKEN_LIFETIME
+
+        return token, expiration
+
+    def __init__(
+        self,
+        jwt_token: JWTToken,
+        installation_id: int,
+        installations_token_map_name: Path,
+        installation_token: None | tuple[str, datetime] = None,
+    ) -> None:
+        self.jwt_token = jwt_token
+        self.installation_id = installation_id
+        self.installations_token_map_name = installations_token_map_name
+
+        if installation_token is None:
+            self.token, self.expiration = InstallationToken._generate_token(
+                self.jwt_token, self.installation_id
+            )
+            self._save()
+        else:
+            self.token, self.expiration = installation_token
+
+    def get(self) -> str:
+        self.verify()
+        return self.token
+
+    def get_as_secret(self) -> str:
+        return f"%(secret:github-token-{self.installation_id})"
+
+    def verify(self) -> None:
+        if datetime.now(tz=UTC) - self.expiration > self.GITHUB_TOKEN_LIFETIME * 0.8:
+            self.token, self.expiration = InstallationToken._generate_token(
+                self.jwt_token, self.installation_id
+            )
+            self._save()
+
+    def _save(self) -> None:
+        # of format:
+        # {
+        #   123: {
+        #     expiration: <datetime>,
+        #     token: "token"
+        #   }
+        # }
+        installations_token_map: dict[str, Any]
+        if self.installations_token_map_name.exists():
+            installations_token_map = json.loads(
+                self.installations_token_map_name.read_text()
+            )
+        else:
+            installations_token_map = {}
+
+        installations_token_map.update(
+            {
+                str(self.installation_id): {
+                    "expiration": self.expiration.isoformat(),
+                    "token": self.token,
+                }
+            }
+        )
+
+        atomic_write_file(
+            self.installations_token_map_name, json.dumps(installations_token_map)
+        )

buildbot_nix/github/jwt_token.py

@@ -0,0 +1,84 @@
+import base64
+import json
+import os
+import subprocess
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from .repo_token import RepoToken
+
+
+class JWTToken(RepoToken):
+    app_id: int
+    app_private_key: str
+    lifetime: timedelta
+
+    expiration: datetime
+    token: str
+
+    def __init__(
+        self,
+        app_id: int,
+        app_private_key: str,
+        lifetime: timedelta = timedelta(minutes=10),
+    ) -> None:
+        self.app_id = app_id
+        self.app_private_key = app_private_key
+        self.lifetime = lifetime
+
+        self.token, self.expiration = JWTToken.generate_token(
+            self.app_id, self.app_private_key, lifetime
+        )
+
+    @staticmethod
+    def generate_token(
+        app_id: int, app_private_key: str, lifetime: timedelta
+    ) -> tuple[str, datetime]:
+        def build_jwt_payload(
+            app_id: int, lifetime: timedelta
+        ) -> tuple[dict[str, Any], datetime]:
+            jwt_iat_drift: timedelta = timedelta(seconds=60)
+            now: datetime = datetime.now(tz=UTC)
+            iat: datetime = now - jwt_iat_drift
+            exp: datetime = iat + lifetime
+            jwt_payload = {
+                "iat": int(iat.timestamp()),
+                "exp": int(exp.timestamp()),
+                "iss": str(app_id),
+            }
+            return (jwt_payload, exp)
+
+        def rs256_sign(data: str, private_key: str) -> str:
+            signature = subprocess.run(
+                ["openssl", "dgst", "-binary", "-sha256", "-sign", private_key],
+                input=data.encode("utf-8"),
+                stdout=subprocess.PIPE,
+                check=True,
+                cwd=os.environ.get("CREDENTIALS_DIRECTORY"),
+            ).stdout
+            return base64url(signature)
+
+        def base64url(data: bytes) -> str:
+            return base64.urlsafe_b64encode(data).rstrip(b"=").decode("utf-8")
+
+        jwt, expiration = build_jwt_payload(app_id, lifetime)
+        jwt_payload = json.dumps(jwt).encode("utf-8")
+        json_headers = json.dumps({"alg": "RS256", "typ": "JWT"}).encode("utf-8")
+        encoded_jwt_parts = f"{base64url(json_headers)}.{base64url(jwt_payload)}"
+        encoded_mac = rs256_sign(encoded_jwt_parts, app_private_key)
+        return (f"{encoded_jwt_parts}.{encoded_mac}", expiration)
+
+        # installations = paginated_github_request("https://api.github.com/app/installations?per_page=100", generated_jwt)
+
+        # return list(map(lambda installation: create_installation_access_token(installation['id']).json()["token"], installations))
+
+    def get(self) -> str:
+        if datetime.now(tz=UTC) - self.expiration > self.lifetime * 0.8:
+            self.token, self.expiration = JWTToken.generate_token(
+                self.app_id, self.app_private_key, self.lifetime
+            )
+
+        return self.token
+
+    def get_as_secret(self) -> str:
+        return "%(secret:github-jwt-token)"

buildbot_nix/github/legacy_token.py

@@ -0,0 +1,14 @@
+from .repo_token import RepoToken
+
+
+class LegacyToken(RepoToken):
+    token: str
+
+    def __init__(self, token: str) -> None:
+        self.token = token
+
+    def get(self) -> str:
+        return self.token
+
+    def get_as_secret(self) -> str:
+        return "%(secret:github-token)"

buildbot_nix/github/repo_token.py

@@ -0,0 +1,11 @@
+from abc import abstractmethod
+
+
+class RepoToken:
+    @abstractmethod
+    def get(self) -> str:
+        pass
+
+    @abstractmethod
+    def get_as_secret(self) -> str:
+        pass