From 1d94f6c1f48c21da7f45852e18cc8e2dcb5ac542 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 27 Apr 2025 18:57:22 +0200 Subject: [PATCH 1/3] {2023.06}[2023a,a64fx] apps originally built with EB 4.9.1 --- .../a64fx/eessi-2023.06-eb-4.9.4-2023a.yml | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023a.yml index 82548a5856..aa90c7abb4 100644 --- a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023a.yml @@ -258,3 +258,39 @@ easyconfigs: - PostgreSQL-16.1-GCCcore-12.3.0.eb - ImageMagick-7.1.1-15-GCCcore-12.3.0.eb - GDAL-3.7.1-foss-2023a.eb +# from here on apps were originally built with EB 4.9.1 (except for +# R-bundle-CRAN-2023.12-foss-2023a, which was originally built with 4.9.4) + - ncdu-1.18-GCC-12.3.0.eb + - SAMtools-1.18-GCC-12.3.0.eb + - R-bundle-CRAN-2023.12-foss-2023a.eb +# while PR 20379 is included since EB 4.9.2, we need to use a commit to avoid +# rebuilding R-bundle-Bioconductor due to the later PR 21948 which was only made +# available with EB 5.0.0 +# - R-bundle-Bioconductor-3.18-foss-2023a-R-4.3.2.eb: +# options: +# from-pr: 20379 + - R-bundle-Bioconductor-3.18-foss-2023a-R-4.3.2.eb: + options: + # (additional extensions have been added) + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21948 + from-commit: f9cfe6ac7d9019970c2be3e8b09db4d846cf005a +# PR 18852 is included since EB 4.9.2 +# - ipympl-0.9.3-gfbf-2023a.eb: +# options: +# # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18852 +# from-pr: 18852 + - ipympl-0.9.3-gfbf-2023a.eb +# PR 20595 is included since EB 4.9.2 +# - ESPResSo-4.2.2-foss-2023a.eb: +# options: +# from-pr: 20595 + - ESPResSo-4.2.2-foss-2023a.eb + - GATK-4.5.0.0-GCCcore-12.3.0-Java-17.eb + - WhatsHap-2.2-foss-2023a.eb +# PR 20784 is included since EB 4.9.2 +# - BLAST+-2.14.1-gompi-2023a.eb: +# options: +# from-pr: 20784 + - BLAST+-2.14.1-gompi-2023a.eb + - Valgrind-3.21.0-gompi-2023a.eb + - OrthoFinder-2.5.5-foss-2023a.eb From bd3d5c79be840577a7af7c024f7afe13a1595092 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 30 Apr 2025 08:36:41 +0200 Subject: [PATCH 2/3] stop after R-bundle-CRAN --- .../a64fx/eessi-2023.06-eb-4.9.4-2023a.yml | 61 ++++++++++--------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023a.yml index aa90c7abb4..a1dfd4a929 100644 --- a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023a.yml @@ -263,34 +263,37 @@ easyconfigs: - ncdu-1.18-GCC-12.3.0.eb - SAMtools-1.18-GCC-12.3.0.eb - R-bundle-CRAN-2023.12-foss-2023a.eb -# while PR 20379 is included since EB 4.9.2, we need to use a commit to avoid -# rebuilding R-bundle-Bioconductor due to the later PR 21948 which was only made -# available with EB 5.0.0 +# R-bundle-CRAN requires a lot of time. We first build that one and afterwards +# continue with the ones below. +# +## while PR 20379 is included since EB 4.9.2, we need to use a commit to avoid +## rebuilding R-bundle-Bioconductor due to the later PR 21948 which was only made +## available with EB 5.0.0 +## - R-bundle-Bioconductor-3.18-foss-2023a-R-4.3.2.eb: +## options: +## from-pr: 20379 # - R-bundle-Bioconductor-3.18-foss-2023a-R-4.3.2.eb: # options: -# from-pr: 20379 - - R-bundle-Bioconductor-3.18-foss-2023a-R-4.3.2.eb: - options: - # (additional extensions have been added) - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21948 - from-commit: f9cfe6ac7d9019970c2be3e8b09db4d846cf005a -# PR 18852 is included since EB 4.9.2 -# - ipympl-0.9.3-gfbf-2023a.eb: -# options: -# # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18852 -# from-pr: 18852 - - ipympl-0.9.3-gfbf-2023a.eb -# PR 20595 is included since EB 4.9.2 -# - ESPResSo-4.2.2-foss-2023a.eb: -# options: -# from-pr: 20595 - - ESPResSo-4.2.2-foss-2023a.eb - - GATK-4.5.0.0-GCCcore-12.3.0-Java-17.eb - - WhatsHap-2.2-foss-2023a.eb -# PR 20784 is included since EB 4.9.2 -# - BLAST+-2.14.1-gompi-2023a.eb: -# options: -# from-pr: 20784 - - BLAST+-2.14.1-gompi-2023a.eb - - Valgrind-3.21.0-gompi-2023a.eb - - OrthoFinder-2.5.5-foss-2023a.eb +# # (additional extensions have been added) +# # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21948 +# from-commit: f9cfe6ac7d9019970c2be3e8b09db4d846cf005a +## PR 18852 is included since EB 4.9.2 +## - ipympl-0.9.3-gfbf-2023a.eb: +## options: +## # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18852 +## from-pr: 18852 +# - ipympl-0.9.3-gfbf-2023a.eb +## PR 20595 is included since EB 4.9.2 +## - ESPResSo-4.2.2-foss-2023a.eb: +## options: +## from-pr: 20595 +# - ESPResSo-4.2.2-foss-2023a.eb +# - GATK-4.5.0.0-GCCcore-12.3.0-Java-17.eb +# - WhatsHap-2.2-foss-2023a.eb +## PR 20784 is included since EB 4.9.2 +## - BLAST+-2.14.1-gompi-2023a.eb: +## options: +## from-pr: 20784 +# - BLAST+-2.14.1-gompi-2023a.eb +# - Valgrind-3.21.0-gompi-2023a.eb +# - OrthoFinder-2.5.5-foss-2023a.eb From e552fcb14f841b981e0c634b121863a9218aa23c Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 30 Apr 2025 08:52:37 +0200 Subject: [PATCH 3/3] increase walltime when building R-bundle-CRAN on a64fx --- det_submit_opts.py | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 det_submit_opts.py diff --git a/det_submit_opts.py b/det_submit_opts.py new file mode 100644 index 0000000000..328d4f3335 --- /dev/null +++ b/det_submit_opts.py @@ -0,0 +1,45 @@ +import os +import subprocess + +from easybuild.framework.easystack import EasyStackParser + +CPU_TARGET_A64FX = 'aarch64/a64fx' + + +def get_orig_easystack(easystack, repo_path): + """ write the original easystack file (before the diff was applied) """ + orig_easystack = f'{easystack}.orig' + git_cmd = f'git -C {repo_path} show HEAD:{easystack}'.split() + with open(os.path.join(repo_path, orig_easystack), 'w', encoding='utf-8') as outfile: + subprocess.run(git_cmd, check=True, stdout=outfile) + return orig_easystack + + +def det_submit_opts(job): + """ + determine submit options from added easyconfigs + Args: + job (Job): namedtuple containing all information about job to be submitted + + Returns: + (string): string containing extra submit options + """ + easystack = 'easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023a.yml' + repo_path = job.working_dir + orig_easystack = get_orig_easystack(easystack, repo_path) + + esp = EasyStackParser() + orig_ecs = {x[0] for x in esp.parse(os.path.join(repo_path, orig_easystack)).ec_opt_tuples} + pr_ecs = {x[0] for x in esp.parse(os.path.join(repo_path, easystack)).ec_opt_tuples} + added_ecs = pr_ecs - orig_ecs + print(f'added easyconfigs: {added_ecs}') + + submit_opts = [job.slurm_opts] + for ec in added_ecs: + # remove OS part from arch_target + arch_name = '/'.join(job.arch_target.split('/')[1:]) + # set walltime limit to 2 days when R-bundle-CRAN should be built on a64fx + if ec.startswith('R-bundle-CRAN-2023.12-foss-2023a') and arch_name == CPU_TARGET_A64FX: + submit_opts.append('--time=2-00:00:00') + + return ' '.join(submit_opts)