From a444bccdb9ba70f4462232cf0ca47d94e7ac0192 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 6 Feb 2025 14:54:39 +0100 Subject: [PATCH 01/18] add support for cloning Git repository via SSH rather than HTTPS --- README.md | 9 +++++++++ tasks/build.py | 28 +++++++++++++++++++++++++--- tools/config.py | 1 + 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 157e5e7..66f579f 100644 --- a/README.md +++ b/README.md @@ -443,6 +443,15 @@ variables) that are allowed to be specified in a PR command with the be exported into the build environment before running the bot/build.sh script. +``` +clone_git_repo_via = 'https' +``` + +The `clone_git_repo_via` setting specifies via which mechanism the Git repository +should be cloned. This can be either: +* '`https`' (default): clone repository via HTTPS with `git clone https://github.com//` +* '`ssh`': clone repository via SSH with `git clone git@github.com:/.git` + #### `[bot_control]` section The `[bot_control]` section contains settings for configuring the feature to diff --git a/tasks/build.py b/tasks/build.py index 0ddcf61..9a1f9a8 100644 --- a/tasks/build.py +++ b/tasks/build.py @@ -44,6 +44,7 @@ _ERROR_GIT_APPLY = "git apply" _ERROR_GIT_CHECKOUT = "git checkout" _ERROR_GIT_CLONE = "curl" +_ERROR_GIT_DIFF = "git diff" _ERROR_NONE = "none" # other constants @@ -355,7 +356,7 @@ def clone_git_repo(repo, path): return (clone_output, clone_error, clone_exit_code) -def download_pr(repo_name, branch_name, pr, arch_job_dir): +def download_pr(repo_name, branch_name, pr, arch_job_dir, clone_via=None): """ Download pull request to job working directory @@ -364,6 +365,7 @@ def download_pr(repo_name, branch_name, pr, arch_job_dir): branch_name (string): name of the base branch of the pull request pr (github.PullRequest.PullRequest): instance representing the pull request arch_job_dir (string): working directory of the job to be submitted + clone_via (string): mechanism to clone Git repository, should be 'https' (default) or 'ssh' Returns: None (implicitly), in case an error is caught in the git clone, git checkout, curl, @@ -376,7 +378,15 @@ def download_pr(repo_name, branch_name, pr, arch_job_dir): # - 'git checkout' base branch of pull request # - 'curl' diff for pull request # - 'git apply' diff file - clone_output, clone_error, clone_exit_code = clone_git_repo(f'https://github.com/{repo_name}', arch_job_dir) + if clone_via in (None, 'https'): + repo_url = f'https://github.com/{repo_name}' + elif clone_via == 'ssh': + repo_url = f'git@github.com:{repo_name}.git' + else: + error_stage = _ERROR_GIT_CLONE + return '', f"Unknown mechanism to clone Git repo: {clone_via}", 1, error_stage + + clone_output, clone_error, clone_exit_code = clone_git_repo(repo_url, arch_job_dir) if clone_exit_code != 0: error_stage = _ERROR_GIT_CLONE return clone_output, clone_error, clone_exit_code, error_stage @@ -407,6 +417,17 @@ def download_pr(repo_name, branch_name, pr, arch_job_dir): error_stage = _ERROR_CURL return curl_output, curl_error, curl_exit_code, error_stage + git_diff_cmd = ' '.join([ + f"git fetch origin pull/{pr.number}/head:{pr.number}", + f"git diff HEAD pr{pr.number} > {pr.number}.diff", + ]) + git_diff_output, git_diff_error, git_diff_exit_code = run_cmd( + git_diff_cmd, "Obtain patch", arch_job_dir, raise_on_error=False + ) + if git_diff_exit_code != 0: + error_stage = _ERROR_GIT_DIFF + return git_diff_output, git_diff_error, git_diff_exit_code + git_apply_cmd = f'git apply {pr.number}.diff' log(f'git apply with command {git_apply_cmd}') git_apply_output, git_apply_error, git_apply_exit_code = run_cmd( @@ -615,8 +636,9 @@ def prepare_jobs(pr, cfg, event_info, action_filter): log(f"{fn}(): job_dir '{job_dir}'") # TODO optimisation? download once, copy and cleanup initial copy? + clone_git_repo_via = build_env_cfg.get(BUILDENV_SETTING_CLONE_GIT_REPO_VIA) download_pr_output, download_pr_error, download_pr_exit_code, error_stage = download_pr( - base_repo_name, base_branch_name, pr, job_dir + base_repo_name, base_branch_name, pr, job_dir, clone_via=clone_git_repo_via, ) comment_download_pr(base_repo_name, pr, download_pr_exit_code, download_pr_error, error_stage) # prepare job configuration file 'job.cfg' in directory /cfg diff --git a/tools/config.py b/tools/config.py index 60554be..578d38e 100644 --- a/tools/config.py +++ b/tools/config.py @@ -42,6 +42,7 @@ BUILDENV_SETTING_BUILD_JOB_SCRIPT = 'build_job_script' BUILDENV_SETTING_BUILD_LOGS_DIR = 'build_logs_dir' BUILDENV_SETTING_BUILD_PERMISSION = 'build_permission' +BUILDENV_SETTING_CLONE_GIT_REPO_VIA = 'clone_git_repo_via' BUILDENV_SETTING_CONTAINER_CACHEDIR = 'container_cachedir' BUILDENV_SETTING_CVMFS_CUSTOMIZATIONS = 'cvmfs_customizations' BUILDENV_SETTING_HTTPS_PROXY = 'https_proxy' From 6dcdd8006039804ced279375ab0981a92264a055 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 6 Feb 2025 15:02:44 +0100 Subject: [PATCH 02/18] fix use of config.BUILDENV_SETTING_CLONE_GIT_REPO_VIA --- tasks/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/build.py b/tasks/build.py index 9a1f9a8..6e68ad2 100644 --- a/tasks/build.py +++ b/tasks/build.py @@ -636,7 +636,7 @@ def prepare_jobs(pr, cfg, event_info, action_filter): log(f"{fn}(): job_dir '{job_dir}'") # TODO optimisation? download once, copy and cleanup initial copy? - clone_git_repo_via = build_env_cfg.get(BUILDENV_SETTING_CLONE_GIT_REPO_VIA) + clone_git_repo_via = build_env_cfg.get(config.BUILDENV_SETTING_CLONE_GIT_REPO_VIA) download_pr_output, download_pr_error, download_pr_exit_code, error_stage = download_pr( base_repo_name, base_branch_name, pr, job_dir, clone_via=clone_git_repo_via, ) From 3f2cfcba436d0356eb54af4966db1cad8b2f8ecb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 6 Feb 2025 15:05:29 +0100 Subject: [PATCH 03/18] also use clone_git_repo_via in app.cfg.example --- README.md | 6 +++--- app.cfg.example | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 66f579f..5ed8cd6 100644 --- a/README.md +++ b/README.md @@ -444,13 +444,13 @@ be exported into the build environment before running the bot/build.sh script. ``` -clone_git_repo_via = 'https' +clone_git_repo_via = https ``` The `clone_git_repo_via` setting specifies via which mechanism the Git repository should be cloned. This can be either: -* '`https`' (default): clone repository via HTTPS with `git clone https://github.com//` -* '`ssh`': clone repository via SSH with `git clone git@github.com:/.git` +* `https` (default): clone repository via HTTPS with `git clone https://github.com//` +* `ssh`: clone repository via SSH with `git clone git@github.com:/.git` #### `[bot_control]` section diff --git a/app.cfg.example b/app.cfg.example index 1958121..aa8eaab 100644 --- a/app.cfg.example +++ b/app.cfg.example @@ -136,6 +136,10 @@ allow_update_submit_opts = false # exported into the build environment via `exportvariable` filters allowed_exportvars = ["NAME1=value_1a", "NAME1=value_1b", "NAME2=value_2"] +# mechanisn to use to clone Git repository +# 'https' to clone via HTTPS (git clone https://github.com//) +# 'ssh' to clone via SSH (git clone git@github.com:/.git) +clone_git_repo_via = https [deploycfg] # script for uploading built software packages From 37c06af97816668032d29630684611d1078b0ed8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 6 Feb 2025 15:13:04 +0100 Subject: [PATCH 04/18] mark clone_git_repo_via as optional + add logging --- eessi_bot_event_handler.py | 1 + tasks/build.py | 1 + 2 files changed, 2 insertions(+) diff --git a/eessi_bot_event_handler.py b/eessi_bot_event_handler.py index f5b05d1..c398542 100644 --- a/eessi_bot_event_handler.py +++ b/eessi_bot_event_handler.py @@ -52,6 +52,7 @@ config.BUILDENV_SETTING_BUILD_JOB_SCRIPT, # required config.BUILDENV_SETTING_BUILD_LOGS_DIR, # optional+recommended config.BUILDENV_SETTING_BUILD_PERMISSION, # optional+recommended + config.BUILDENV_SETTING_CLONE_GIT_REPO_VIA, # optional config.BUILDENV_SETTING_CONTAINER_CACHEDIR, # optional+recommended # config.BUILDENV_SETTING_CVMFS_CUSTOMIZATIONS, # optional # config.BUILDENV_SETTING_HTTPS_PROXY, # optional diff --git a/tasks/build.py b/tasks/build.py index 6e68ad2..96980db 100644 --- a/tasks/build.py +++ b/tasks/build.py @@ -637,6 +637,7 @@ def prepare_jobs(pr, cfg, event_info, action_filter): # TODO optimisation? download once, copy and cleanup initial copy? clone_git_repo_via = build_env_cfg.get(config.BUILDENV_SETTING_CLONE_GIT_REPO_VIA) + log(f"Cloning Git repo via: {clone_git_repo_via}") download_pr_output, download_pr_error, download_pr_exit_code, error_stage = download_pr( base_repo_name, base_branch_name, pr, job_dir, clone_via=clone_git_repo_via, ) From 4b046f68eeb4696783988c20cfbb496c30a88d43 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 6 Feb 2025 15:17:09 +0100 Subject: [PATCH 05/18] fix get_build_env_cfg to get clone_git_repo_via value from buildenv in configuration --- tasks/build.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tasks/build.py b/tasks/build.py index 96980db..4821ab0 100644 --- a/tasks/build.py +++ b/tasks/build.py @@ -148,6 +148,10 @@ def get_build_env_cfg(cfg): log(f"{fn}(): load_modules '{load_modules}'") config_data[config.BUILDENV_SETTING_LOAD_MODULES] = load_modules + clone_git_repo_via = buildenv.get(config.BUILDENV_SETTING_CLONE_GIT_REPO_VIA, None) + log(f"{fn}(): clone_git_repo_via '{clone_git_repo_via}'") + config_data[config.BUILDENV_SETTING_CLONE_GIT_REPO_VIA] = clone_git_repo_via + return config_data @@ -378,6 +382,7 @@ def download_pr(repo_name, branch_name, pr, arch_job_dir, clone_via=None): # - 'git checkout' base branch of pull request # - 'curl' diff for pull request # - 'git apply' diff file + log(f"Cloning Git repo via: {clone_via}") if clone_via in (None, 'https'): repo_url = f'https://github.com/{repo_name}' elif clone_via == 'ssh': @@ -637,7 +642,6 @@ def prepare_jobs(pr, cfg, event_info, action_filter): # TODO optimisation? download once, copy and cleanup initial copy? clone_git_repo_via = build_env_cfg.get(config.BUILDENV_SETTING_CLONE_GIT_REPO_VIA) - log(f"Cloning Git repo via: {clone_git_repo_via}") download_pr_output, download_pr_error, download_pr_exit_code, error_stage = download_pr( base_repo_name, base_branch_name, pr, job_dir, clone_via=clone_git_repo_via, ) From 45bd5568b45899f5e6d6229f45b55e6a5226cea1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 6 Feb 2025 16:02:00 +0100 Subject: [PATCH 06/18] remove curl command to obtain PR diff + fix _ERROR_GIT_CLONE constant --- tasks/build.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/tasks/build.py b/tasks/build.py index 4821ab0..062ba13 100644 --- a/tasks/build.py +++ b/tasks/build.py @@ -43,7 +43,7 @@ _ERROR_CURL = "curl" _ERROR_GIT_APPLY = "git apply" _ERROR_GIT_CHECKOUT = "git checkout" -_ERROR_GIT_CLONE = "curl" +_ERROR_GIT_CLONE = "git clone" _ERROR_GIT_DIFF = "git diff" _ERROR_NONE = "none" @@ -408,20 +408,6 @@ def download_pr(repo_name, branch_name, pr, arch_job_dir, clone_via=None): error_stage = _ERROR_GIT_CHECKOUT return checkout_output, checkout_err, checkout_exit_code, error_stage - curl_cmd = ' '.join([ - 'curl -L', - '-H "Accept: application/vnd.github.diff"', - '-H "X-GitHub-Api-Version: 2022-11-28"', - f'https://api.github.com/repos/{repo_name}/pulls/{pr.number} > {pr.number}.diff', - ]) - log(f'curl with command {curl_cmd}') - curl_output, curl_error, curl_exit_code = run_cmd( - curl_cmd, "Obtain patch", arch_job_dir, raise_on_error=False - ) - if curl_exit_code != 0: - error_stage = _ERROR_CURL - return curl_output, curl_error, curl_exit_code, error_stage - git_diff_cmd = ' '.join([ f"git fetch origin pull/{pr.number}/head:{pr.number}", f"git diff HEAD pr{pr.number} > {pr.number}.diff", From b19fe23cb1537d22f9f170c62d3672f70294dd85 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 6 Feb 2025 16:04:33 +0100 Subject: [PATCH 07/18] fix return value for 'git diff' command in download_pr function --- tasks/build.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tasks/build.py b/tasks/build.py index 062ba13..3a440a9 100644 --- a/tasks/build.py +++ b/tasks/build.py @@ -388,8 +388,11 @@ def download_pr(repo_name, branch_name, pr, arch_job_dir, clone_via=None): elif clone_via == 'ssh': repo_url = f'git@github.com:{repo_name}.git' else: + clone_output = '' + clone_error = f"Unknown mechanism to clone Git repo: {clone_via}" + clone_exit_code = 1 error_stage = _ERROR_GIT_CLONE - return '', f"Unknown mechanism to clone Git repo: {clone_via}", 1, error_stage + return clone_output, clone_error, clone_exit_code, error_stage clone_output, clone_error, clone_exit_code = clone_git_repo(repo_url, arch_job_dir) if clone_exit_code != 0: @@ -417,7 +420,7 @@ def download_pr(repo_name, branch_name, pr, arch_job_dir, clone_via=None): ) if git_diff_exit_code != 0: error_stage = _ERROR_GIT_DIFF - return git_diff_output, git_diff_error, git_diff_exit_code + return git_diff_output, git_diff_error, git_diff_exit_code, error_stage git_apply_cmd = f'git apply {pr.number}.diff' log(f'git apply with command {git_apply_cmd}') From 5c51665078dc0b26379512393281b18713f79e7d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 6 Feb 2025 16:13:07 +0100 Subject: [PATCH 08/18] make sure that download_comment is defined in comment_download_pr --- tasks/build.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tasks/build.py b/tasks/build.py index 3a440a9..d047310 100644 --- a/tasks/build.py +++ b/tasks/build.py @@ -472,6 +472,8 @@ def comment_download_pr(base_repo_name, pr, download_pr_exit_code, download_pr_e download_comment = (f"```{download_pr_error}```\n" f"{download_pr_comments_cfg[config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_APPLY_FAILURE]}" f"\n{download_pr_comments_cfg[config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_APPLY_TIP]}") + else: + download_comment = f"```{download_pr_error}```" download_comment = pr_comments.create_comment( repo_name=base_repo_name, pr_number=pr.number, comment=download_comment From a1c26353fcb4a9ee3d8d5e87b313428f25e3898b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 6 Feb 2025 16:13:53 +0100 Subject: [PATCH 09/18] fix 'git diff' command in download_repo --- tasks/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/build.py b/tasks/build.py index d047310..5613c52 100644 --- a/tasks/build.py +++ b/tasks/build.py @@ -411,7 +411,7 @@ def download_pr(repo_name, branch_name, pr, arch_job_dir, clone_via=None): error_stage = _ERROR_GIT_CHECKOUT return checkout_output, checkout_err, checkout_exit_code, error_stage - git_diff_cmd = ' '.join([ + git_diff_cmd = ' && '.join([ f"git fetch origin pull/{pr.number}/head:{pr.number}", f"git diff HEAD pr{pr.number} > {pr.number}.diff", ]) From 02f31257fef744b14931f2f83be38fa72c76a994 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 6 Feb 2025 16:18:53 +0100 Subject: [PATCH 10/18] fix branch name used by 'git diff' command --- tasks/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/build.py b/tasks/build.py index 5613c52..20dccf8 100644 --- a/tasks/build.py +++ b/tasks/build.py @@ -412,7 +412,7 @@ def download_pr(repo_name, branch_name, pr, arch_job_dir, clone_via=None): return checkout_output, checkout_err, checkout_exit_code, error_stage git_diff_cmd = ' && '.join([ - f"git fetch origin pull/{pr.number}/head:{pr.number}", + f"git fetch origin pull/{pr.number}/head:pr{pr.number}", f"git diff HEAD pr{pr.number} > {pr.number}.diff", ]) git_diff_output, git_diff_error, git_diff_exit_code = run_cmd( From c7a98f2541b91a8fef16815513c91045fa13a912 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 21 Feb 2025 14:13:12 +0100 Subject: [PATCH 11/18] comment out clone_git_repo_via setting in REQUIRED_CONFIG to avoid making it required MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Thomas Röblitz --- eessi_bot_event_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi_bot_event_handler.py b/eessi_bot_event_handler.py index c398542..883aed9 100644 --- a/eessi_bot_event_handler.py +++ b/eessi_bot_event_handler.py @@ -52,7 +52,7 @@ config.BUILDENV_SETTING_BUILD_JOB_SCRIPT, # required config.BUILDENV_SETTING_BUILD_LOGS_DIR, # optional+recommended config.BUILDENV_SETTING_BUILD_PERMISSION, # optional+recommended - config.BUILDENV_SETTING_CLONE_GIT_REPO_VIA, # optional + # config.BUILDENV_SETTING_CLONE_GIT_REPO_VIA, # optional config.BUILDENV_SETTING_CONTAINER_CACHEDIR, # optional+recommended # config.BUILDENV_SETTING_CVMFS_CUSTOMIZATIONS, # optional # config.BUILDENV_SETTING_HTTPS_PROXY, # optional From 7da317db383ed43faa8804d0a088f995bd82e8cb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 21 Feb 2025 14:15:29 +0100 Subject: [PATCH 12/18] fix order in REQUIRED_CONFIG (recommended settings before optional ones) --- eessi_bot_event_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi_bot_event_handler.py b/eessi_bot_event_handler.py index 883aed9..93bfa97 100644 --- a/eessi_bot_event_handler.py +++ b/eessi_bot_event_handler.py @@ -52,8 +52,8 @@ config.BUILDENV_SETTING_BUILD_JOB_SCRIPT, # required config.BUILDENV_SETTING_BUILD_LOGS_DIR, # optional+recommended config.BUILDENV_SETTING_BUILD_PERMISSION, # optional+recommended - # config.BUILDENV_SETTING_CLONE_GIT_REPO_VIA, # optional config.BUILDENV_SETTING_CONTAINER_CACHEDIR, # optional+recommended + # config.BUILDENV_SETTING_CLONE_GIT_REPO_VIA, # optional # config.BUILDENV_SETTING_CVMFS_CUSTOMIZATIONS, # optional # config.BUILDENV_SETTING_HTTPS_PROXY, # optional # config.BUILDENV_SETTING_HTTP_PROXY, # optional From 293ce3c5e3cff7c9f463aa4062b5576e5c1faab5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 21 Feb 2025 14:16:37 +0100 Subject: [PATCH 13/18] add more info on use case of clone_git_repo_via set to 'ssh' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Thomas Röblitz --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 5ed8cd6..81e85a8 100644 --- a/README.md +++ b/README.md @@ -451,6 +451,18 @@ The `clone_git_repo_via` setting specifies via which mechanism the Git repositor should be cloned. This can be either: * `https` (default): clone repository via HTTPS with `git clone https://github.com//` * `ssh`: clone repository via SSH with `git clone git@github.com:/.git` +In case of using 'ssh', one may need additional steps to ensure that the bot uses the right ssh key and does not ask for a passphrase (if the key used is protected with one). Here are a few things to consider: +- if the ssh key to be used does not have a standard name (e.g., `id_rsa`), add the following entry to `~/.ssh/config` in the bot's account + ``` + Host github.com + User git + IdentityFile ~/.ssh/NAME_OF_PRIVATE_KEY_FILE + ``` +- if the key is protected by a passphrase (**highly recommended**), run an SSH agent and add the key to it + ``` + eval $(ssh-agent -s) + ssh-add ~/.ssh/NAME_OF_PRIVATE_KEY_FILE + ``` #### `[bot_control]` section From 668915c2b4e004cf6473ad9a30cb45a8a85ce6da Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 21 Feb 2025 14:17:44 +0100 Subject: [PATCH 14/18] also add more info on use case of clone_git_repo_via set to 'ssh' to app.cfg.example MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Thomas Röblitz --- app.cfg.example | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/app.cfg.example b/app.cfg.example index aa8eaab..6ad77f4 100644 --- a/app.cfg.example +++ b/app.cfg.example @@ -138,7 +138,22 @@ allowed_exportvars = ["NAME1=value_1a", "NAME1=value_1b", "NAME2=value_2"] # mechanisn to use to clone Git repository # 'https' to clone via HTTPS (git clone https://github.com//) -# 'ssh' to clone via SSH (git clone git@github.com:/.git) +# In case of using 'ssh', one may need additional steps to ensure that the bot +# uses the right ssh key and does not ask for a passphrase (if the key used is +# protected with one). Here are a few things to consider: +# - if the ssh key to be used does not have a standard name (e.g., 'id_rsa'), +# add the following entry to '~/.ssh/config' in the bot's account +# +# Host github.com +# User git +# IdentityFile ~/.ssh/NAME_OF_PRIVATE_KEY_FILE +# +# - if the key is protected by a passphrase (**highly recommended**), run an +# SSH agent and add the key to it (with the following two commands) +# +# eval $(ssh-agent -s) +# ssh-add ~/.ssh/NAME_OF_PRIVATE_KEY_FILE + clone_git_repo_via = https [deploycfg] From a99ae6015e18a32964a832baceb6b47d12f30d15 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 21 Feb 2025 15:04:26 +0100 Subject: [PATCH 15/18] only use 'git diff' to obtain PR diff when 'ssh' is used to clone repository --- app.cfg.example | 2 ++ tasks/build.py | 33 ++++++++++++++++++++++----------- tools/config.py | 2 ++ 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/app.cfg.example b/app.cfg.example index 6ad77f4..e69dd2e 100644 --- a/app.cfg.example +++ b/app.cfg.example @@ -291,6 +291,8 @@ curl_failure = Unable to download the `.diff` file. curl_tip = _Tip: This could be a connection failure. Try again and if the issue remains check if the address is correct_ git_apply_failure = Unable to download or merge changes between the source branch and the destination branch. git_apply_tip = _Tip: This can usually be resolved by syncing your branch and resolving any merge conflicts._ +pr_diff_failure = Unable to obtain PR diff. +pr_diff_tip = _Tip: This could be a problem with SSH access to the repository._ [clean_up] trash_bin_dir = $HOME/trash_bin diff --git a/tasks/build.py b/tasks/build.py index 20dccf8..28ee5ce 100644 --- a/tasks/build.py +++ b/tasks/build.py @@ -44,7 +44,7 @@ _ERROR_GIT_APPLY = "git apply" _ERROR_GIT_CHECKOUT = "git checkout" _ERROR_GIT_CLONE = "git clone" -_ERROR_GIT_DIFF = "git diff" +_ERROR_PR_DIFF = "pr_diff" _ERROR_NONE = "none" # other constants @@ -385,8 +385,18 @@ def download_pr(repo_name, branch_name, pr, arch_job_dir, clone_via=None): log(f"Cloning Git repo via: {clone_via}") if clone_via in (None, 'https'): repo_url = f'https://github.com/{repo_name}' + pr_diff_cmd = ' '.join([ + 'curl -L', + '-H "Accept: application/vnd.github.diff"', + '-H "X-GitHub-Api-Version: 2022-11-28"', + f'https://api.github.com/repos/{repo_name}/pulls/{pr.number} > {pr.number}.diff', + ]) elif clone_via == 'ssh': repo_url = f'git@github.com:{repo_name}.git' + pr_diff_cmd = ' && '.join([ + f"git fetch origin pull/{pr.number}/head:pr{pr.number}", + f"git diff HEAD pr{pr.number} > {pr.number}.diff", + ]) else: clone_output = '' clone_error = f"Unknown mechanism to clone Git repo: {clone_via}" @@ -411,21 +421,18 @@ def download_pr(repo_name, branch_name, pr, arch_job_dir, clone_via=None): error_stage = _ERROR_GIT_CHECKOUT return checkout_output, checkout_err, checkout_exit_code, error_stage - git_diff_cmd = ' && '.join([ - f"git fetch origin pull/{pr.number}/head:pr{pr.number}", - f"git diff HEAD pr{pr.number} > {pr.number}.diff", - ]) - git_diff_output, git_diff_error, git_diff_exit_code = run_cmd( - git_diff_cmd, "Obtain patch", arch_job_dir, raise_on_error=False + log(f'obtaining PR diff with command {pr_diff_cmd}') + pr_diff_output, pr_diff_error, pr_diff_exit_code = run_cmd( + pr_diff_cmd, "obtain PR diff", arch_job_dir, raise_on_error=False ) - if git_diff_exit_code != 0: - error_stage = _ERROR_GIT_DIFF - return git_diff_output, git_diff_error, git_diff_exit_code, error_stage + if pr_diff_exit_code != 0: + error_stage = _ERROR_PR_DIFF + return pr_diff_output, pr_diff_error, pr_diff_exit_code, error_stage git_apply_cmd = f'git apply {pr.number}.diff' log(f'git apply with command {git_apply_cmd}') git_apply_output, git_apply_error, git_apply_exit_code = run_cmd( - git_apply_cmd, "Apply patch", arch_job_dir, raise_on_error=False + git_apply_cmd, "apply patch", arch_job_dir, raise_on_error=False ) if git_apply_exit_code != 0: error_stage = _ERROR_GIT_APPLY @@ -472,6 +479,10 @@ def comment_download_pr(base_repo_name, pr, download_pr_exit_code, download_pr_e download_comment = (f"```{download_pr_error}```\n" f"{download_pr_comments_cfg[config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_APPLY_FAILURE]}" f"\n{download_pr_comments_cfg[config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_APPLY_TIP]}") + elif error_stage == _ERROR_PR_DIFF: + download_comment = (f"```{download_pr_error}```\n" + f"{download_pr_comments_cfg[config.DOWNLOAD_PR_COMMENTS_SETTING_PR_DIFF_FAILURE]}" + f"\n{download_pr_comments_cfg[config.DOWNLOAD_PR_COMMENTS_SETTING_PR_DIFF_TIP]}") else: download_comment = f"```{download_pr_error}```" diff --git a/tools/config.py b/tools/config.py index 578d38e..6ed98bf 100644 --- a/tools/config.py +++ b/tools/config.py @@ -75,6 +75,8 @@ DOWNLOAD_PR_COMMENTS_SETTING_GIT_CHECKOUT_TIP = 'git_checkout_tip' DOWNLOAD_PR_COMMENTS_SETTING_GIT_CLONE_FAILURE = 'git_clone_failure' DOWNLOAD_PR_COMMENTS_SETTING_GIT_CLONE_TIP = 'git_clone_tip' +DOWNLOAD_PR_COMMENTS_SETTING_PR_DIFF_FAILURE = 'pr_diff_failure' +DOWNLOAD_PR_COMMENTS_SETTING_PR_DIFF_TIP = 'pr_diff_tip' SECTION_EVENT_HANDLER = 'event_handler' EVENT_HANDLER_SETTING_LOG_PATH = 'log_path' From 9b582ab1160eb3a070d4abae9f8ab9e75e9d9e44 Mon Sep 17 00:00:00 2001 From: sam Date: Fri, 4 Apr 2025 16:32:41 +0200 Subject: [PATCH 16/18] git diff from merge-base instead of from HEAD --- tasks/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/build.py b/tasks/build.py index 28ee5ce..2257148 100644 --- a/tasks/build.py +++ b/tasks/build.py @@ -395,7 +395,7 @@ def download_pr(repo_name, branch_name, pr, arch_job_dir, clone_via=None): repo_url = f'git@github.com:{repo_name}.git' pr_diff_cmd = ' && '.join([ f"git fetch origin pull/{pr.number}/head:pr{pr.number}", - f"git diff HEAD pr{pr.number} > {pr.number}.diff", + f"git diff $(git merge-base pr{pr.number} HEAD) pr{pr.number} > {pr.number}.diff", ]) else: clone_output = '' From f4c0940ce79b71dd0940e382196a69b2549d8527 Mon Sep 17 00:00:00 2001 From: sam Date: Thu, 22 May 2025 13:52:52 +0200 Subject: [PATCH 17/18] update README --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ddcbd8e..031884e 100644 --- a/README.md +++ b/README.md @@ -493,7 +493,7 @@ The `clone_git_repo_via` setting specifies via which mechanism the Git repositor should be cloned. This can be either: * `https` (default): clone repository via HTTPS with `git clone https://github.com//` * `ssh`: clone repository via SSH with `git clone git@github.com:/.git` -In case of using 'ssh', one may need additional steps to ensure that the bot uses the right ssh key and does not ask for a passphrase (if the key used is protected with one). Here are a few things to consider: +In case of using 'ssh', one may need additional steps to ensure that the bot uses the right SSH key and does not ask for a passphrase (if the key used is protected with one). Here are a few things to consider: - if the ssh key to be used does not have a standard name (e.g., `id_rsa`), add the following entry to `~/.ssh/config` in the bot's account ``` Host github.com @@ -506,6 +506,8 @@ In case of using 'ssh', one may need additional steps to ensure that the bot use ssh-add ~/.ssh/NAME_OF_PRIVATE_KEY_FILE ``` +Note that the `bot: status` command doesn't work with SSH keys; you'll still need a Github token for that to work. + #### `[bot_control]` section The `[bot_control]` section contains settings for configuring the feature to From 750b8a4b2ad83d075fcd8c05214ad07b967e36f2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 20 Jun 2025 14:19:57 +0200 Subject: [PATCH 18/18] add pr_diff_failure and pr_diff_tip to required configuration settings --- eessi_bot_event_handler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eessi_bot_event_handler.py b/eessi_bot_event_handler.py index 66aced9..b1d4123 100644 --- a/eessi_bot_event_handler.py +++ b/eessi_bot_event_handler.py @@ -89,7 +89,9 @@ config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_CHECKOUT_FAILURE, # required config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_CHECKOUT_TIP, # required config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_CLONE_FAILURE, # required - config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_CLONE_TIP], # required + config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_CLONE_TIP, # required + config.DOWNLOAD_PR_COMMENTS_SETTING_PR_DIFF_FAILURE, # required + config.DOWNLOAD_PR_COMMENTS_SETTING_PR_DIFF_TIP], # required config.SECTION_EVENT_HANDLER: [ config.EVENT_HANDLER_SETTING_LOG_PATH], # required config.SECTION_GITHUB: [