diff --git a/README.md b/README.md index 77c9cdd..031884e 100644 --- a/README.md +++ b/README.md @@ -485,6 +485,29 @@ allowed_exportvars = ["SKIP_TESTS=yes", "SKIP_TESTS=no"] ``` +``` +clone_git_repo_via = https +``` + +The `clone_git_repo_via` setting specifies via which mechanism the Git repository +should be cloned. This can be either: +* `https` (default): clone repository via HTTPS with `git clone https://github.com//` +* `ssh`: clone repository via SSH with `git clone git@github.com:/.git` +In case of using 'ssh', one may need additional steps to ensure that the bot uses the right SSH key and does not ask for a passphrase (if the key used is protected with one). Here are a few things to consider: +- if the ssh key to be used does not have a standard name (e.g., `id_rsa`), add the following entry to `~/.ssh/config` in the bot's account + ``` + Host github.com + User git + IdentityFile ~/.ssh/NAME_OF_PRIVATE_KEY_FILE + ``` +- if the key is protected by a passphrase (**highly recommended**), run an SSH agent and add the key to it + ``` + eval $(ssh-agent -s) + ssh-add ~/.ssh/NAME_OF_PRIVATE_KEY_FILE + ``` + +Note that the `bot: status` command doesn't work with SSH keys; you'll still need a Github token for that to work. + #### `[bot_control]` section The `[bot_control]` section contains settings for configuring the feature to diff --git a/app.cfg.example b/app.cfg.example index f0a4ec4..e57d9ee 100644 --- a/app.cfg.example +++ b/app.cfg.example @@ -175,6 +175,25 @@ allow_update_submit_opts = false # "SKIP_TESTS=yes". # allowed_exportvars = ["NAME1=value_1a", "NAME1=value_1b", "NAME2=value_2"] +# mechanisn to use to clone Git repository +# 'https' to clone via HTTPS (git clone https://github.com//) +# In case of using 'ssh', one may need additional steps to ensure that the bot +# uses the right ssh key and does not ask for a passphrase (if the key used is +# protected with one). Here are a few things to consider: +# - if the ssh key to be used does not have a standard name (e.g., 'id_rsa'), +# add the following entry to '~/.ssh/config' in the bot's account +# +# Host github.com +# User git +# IdentityFile ~/.ssh/NAME_OF_PRIVATE_KEY_FILE +# +# - if the key is protected by a passphrase (**highly recommended**), run an +# SSH agent and add the key to it (with the following two commands) +# +# eval $(ssh-agent -s) +# ssh-add ~/.ssh/NAME_OF_PRIVATE_KEY_FILE + +clone_git_repo_via = https [deploycfg] # script for uploading built software packages @@ -338,6 +357,8 @@ curl_failure = Unable to download the `.diff` file. curl_tip = _Tip: This could be a connection failure. Try again and if the issue remains check if the address is correct_ git_apply_failure = Unable to download or merge changes between the source branch and the destination branch. git_apply_tip = _Tip: This can usually be resolved by syncing your branch and resolving any merge conflicts._ +pr_diff_failure = Unable to obtain PR diff. +pr_diff_tip = _Tip: This could be a problem with SSH access to the repository._ [clean_up] trash_bin_dir = $HOME/trash_bin diff --git a/eessi_bot_event_handler.py b/eessi_bot_event_handler.py index 8412830..b1d4123 100644 --- a/eessi_bot_event_handler.py +++ b/eessi_bot_event_handler.py @@ -54,6 +54,7 @@ config.BUILDENV_SETTING_BUILD_LOGS_DIR, # optional+recommended config.BUILDENV_SETTING_BUILD_PERMISSION, # optional+recommended config.BUILDENV_SETTING_CONTAINER_CACHEDIR, # optional+recommended + # config.BUILDENV_SETTING_CLONE_GIT_REPO_VIA, # optional # config.BUILDENV_SETTING_CVMFS_CUSTOMIZATIONS, # optional # config.BUILDENV_SETTING_HTTPS_PROXY, # optional # config.BUILDENV_SETTING_HTTP_PROXY, # optional @@ -88,7 +89,9 @@ config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_CHECKOUT_FAILURE, # required config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_CHECKOUT_TIP, # required config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_CLONE_FAILURE, # required - config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_CLONE_TIP], # required + config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_CLONE_TIP, # required + config.DOWNLOAD_PR_COMMENTS_SETTING_PR_DIFF_FAILURE, # required + config.DOWNLOAD_PR_COMMENTS_SETTING_PR_DIFF_TIP], # required config.SECTION_EVENT_HANDLER: [ config.EVENT_HANDLER_SETTING_LOG_PATH], # required config.SECTION_GITHUB: [ diff --git a/tasks/build.py b/tasks/build.py index 517c407..836230c 100644 --- a/tasks/build.py +++ b/tasks/build.py @@ -42,7 +42,8 @@ _ERROR_CURL = "curl" _ERROR_GIT_APPLY = "git apply" _ERROR_GIT_CHECKOUT = "git checkout" -_ERROR_GIT_CLONE = "curl" +_ERROR_GIT_CLONE = "git clone" +_ERROR_PR_DIFF = "pr_diff" _ERROR_NONE = "none" # other constants @@ -171,6 +172,10 @@ def get_build_env_cfg(cfg): log(f"{fn}(): load_modules '{load_modules}'") config_data[config.BUILDENV_SETTING_LOAD_MODULES] = load_modules + clone_git_repo_via = buildenv.get(config.BUILDENV_SETTING_CLONE_GIT_REPO_VIA, None) + log(f"{fn}(): clone_git_repo_via '{clone_git_repo_via}'") + config_data[config.BUILDENV_SETTING_CLONE_GIT_REPO_VIA] = clone_git_repo_via + return config_data @@ -379,7 +384,7 @@ def clone_git_repo(repo, path): return (clone_output, clone_error, clone_exit_code) -def download_pr(repo_name, branch_name, pr, arch_job_dir): +def download_pr(repo_name, branch_name, pr, arch_job_dir, clone_via=None): """ Download pull request to job working directory @@ -388,6 +393,7 @@ def download_pr(repo_name, branch_name, pr, arch_job_dir): branch_name (string): name of the base branch of the pull request pr (github.PullRequest.PullRequest): instance representing the pull request arch_job_dir (string): working directory of the job to be submitted + clone_via (string): mechanism to clone Git repository, should be 'https' (default) or 'ssh' Returns: None (implicitly), in case an error is caught in the git clone, git checkout, curl, @@ -400,7 +406,29 @@ def download_pr(repo_name, branch_name, pr, arch_job_dir): # - 'git checkout' base branch of pull request # - 'curl' diff for pull request # - 'git apply' diff file - clone_output, clone_error, clone_exit_code = clone_git_repo(f'https://github.com/{repo_name}', arch_job_dir) + log(f"Cloning Git repo via: {clone_via}") + if clone_via in (None, 'https'): + repo_url = f'https://github.com/{repo_name}' + pr_diff_cmd = ' '.join([ + 'curl -L', + '-H "Accept: application/vnd.github.diff"', + '-H "X-GitHub-Api-Version: 2022-11-28"', + f'https://api.github.com/repos/{repo_name}/pulls/{pr.number} > {pr.number}.diff', + ]) + elif clone_via == 'ssh': + repo_url = f'git@github.com:{repo_name}.git' + pr_diff_cmd = ' && '.join([ + f"git fetch origin pull/{pr.number}/head:pr{pr.number}", + f"git diff $(git merge-base pr{pr.number} HEAD) pr{pr.number} > {pr.number}.diff", + ]) + else: + clone_output = '' + clone_error = f"Unknown mechanism to clone Git repo: {clone_via}" + clone_exit_code = 1 + error_stage = _ERROR_GIT_CLONE + return clone_output, clone_error, clone_exit_code, error_stage + + clone_output, clone_error, clone_exit_code = clone_git_repo(repo_url, arch_job_dir) if clone_exit_code != 0: error_stage = _ERROR_GIT_CLONE return clone_output, clone_error, clone_exit_code, error_stage @@ -417,24 +445,18 @@ def download_pr(repo_name, branch_name, pr, arch_job_dir): error_stage = _ERROR_GIT_CHECKOUT return checkout_output, checkout_err, checkout_exit_code, error_stage - curl_cmd = ' '.join([ - 'curl -L', - '-H "Accept: application/vnd.github.diff"', - '-H "X-GitHub-Api-Version: 2022-11-28"', - f'https://api.github.com/repos/{repo_name}/pulls/{pr.number} > {pr.number}.diff', - ]) - log(f'curl with command {curl_cmd}') - curl_output, curl_error, curl_exit_code = run_cmd( - curl_cmd, "Obtain patch", arch_job_dir, raise_on_error=False + log(f'obtaining PR diff with command {pr_diff_cmd}') + pr_diff_output, pr_diff_error, pr_diff_exit_code = run_cmd( + pr_diff_cmd, "obtain PR diff", arch_job_dir, raise_on_error=False ) - if curl_exit_code != 0: - error_stage = _ERROR_CURL - return curl_output, curl_error, curl_exit_code, error_stage + if pr_diff_exit_code != 0: + error_stage = _ERROR_PR_DIFF + return pr_diff_output, pr_diff_error, pr_diff_exit_code, error_stage git_apply_cmd = f'git apply {pr.number}.diff' log(f'git apply with command {git_apply_cmd}') git_apply_output, git_apply_error, git_apply_exit_code = run_cmd( - git_apply_cmd, "Apply patch", arch_job_dir, raise_on_error=False + git_apply_cmd, "apply patch", arch_job_dir, raise_on_error=False ) if git_apply_exit_code != 0: error_stage = _ERROR_GIT_APPLY @@ -481,6 +503,12 @@ def comment_download_pr(base_repo_name, pr, download_pr_exit_code, download_pr_e download_comment = (f"```{download_pr_error}```\n" f"{download_pr_comments_cfg[config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_APPLY_FAILURE]}" f"\n{download_pr_comments_cfg[config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_APPLY_TIP]}") + elif error_stage == _ERROR_PR_DIFF: + download_comment = (f"```{download_pr_error}```\n" + f"{download_pr_comments_cfg[config.DOWNLOAD_PR_COMMENTS_SETTING_PR_DIFF_FAILURE]}" + f"\n{download_pr_comments_cfg[config.DOWNLOAD_PR_COMMENTS_SETTING_PR_DIFF_TIP]}") + else: + download_comment = f"```{download_pr_error}```" download_comment = pr_comments.create_comment( repo_name=base_repo_name, pr_number=pr.number, comment=download_comment, req_chatlevel=ChatLevels.MINIMAL @@ -639,8 +667,9 @@ def prepare_jobs(pr, cfg, event_info, action_filter): log(f"{fn}(): job_dir '{job_dir}'") # TODO optimisation? download once, copy and cleanup initial copy? + clone_git_repo_via = build_env_cfg.get(config.BUILDENV_SETTING_CLONE_GIT_REPO_VIA) download_pr_output, download_pr_error, download_pr_exit_code, error_stage = download_pr( - base_repo_name, base_branch_name, pr, job_dir + base_repo_name, base_branch_name, pr, job_dir, clone_via=clone_git_repo_via, ) comment_download_pr(base_repo_name, pr, download_pr_exit_code, download_pr_error, error_stage) # prepare job configuration file 'job.cfg' in directory /cfg diff --git a/tools/config.py b/tools/config.py index 6fe5982..fc790d5 100644 --- a/tools/config.py +++ b/tools/config.py @@ -43,6 +43,7 @@ BUILDENV_SETTING_BUILD_JOB_SCRIPT = 'build_job_script' BUILDENV_SETTING_BUILD_LOGS_DIR = 'build_logs_dir' BUILDENV_SETTING_BUILD_PERMISSION = 'build_permission' +BUILDENV_SETTING_CLONE_GIT_REPO_VIA = 'clone_git_repo_via' BUILDENV_SETTING_CONTAINER_CACHEDIR = 'container_cachedir' BUILDENV_SETTING_CVMFS_CUSTOMIZATIONS = 'cvmfs_customizations' BUILDENV_SETTING_HTTPS_PROXY = 'https_proxy' @@ -82,6 +83,8 @@ DOWNLOAD_PR_COMMENTS_SETTING_GIT_CHECKOUT_TIP = 'git_checkout_tip' DOWNLOAD_PR_COMMENTS_SETTING_GIT_CLONE_FAILURE = 'git_clone_failure' DOWNLOAD_PR_COMMENTS_SETTING_GIT_CLONE_TIP = 'git_clone_tip' +DOWNLOAD_PR_COMMENTS_SETTING_PR_DIFF_FAILURE = 'pr_diff_failure' +DOWNLOAD_PR_COMMENTS_SETTING_PR_DIFF_TIP = 'pr_diff_tip' SECTION_EVENT_HANDLER = 'event_handler' EVENT_HANDLER_SETTING_LOG_PATH = 'log_path'