From c32f1ecaf6c0424c31d40700d2fd07aa92af19f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 26 Jun 2025 17:19:18 +0200 Subject: [PATCH 1/2] add hooks for improving reproducibility --- eb_hooks.py | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index e20ef2e..b2a33cc 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -1,5 +1,6 @@ # Hooks to customize how EasyBuild installs software in EESSI # see https://docs.easybuild.io/en/latest/Hooks.html +import datetime import glob import os import re @@ -7,9 +8,10 @@ import easybuild.tools.environment as env from easybuild.easyblocks.generic.configuremake import obtain_config_guess from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS +from easybuild.tools import config from easybuild.tools.build_log import EasyBuildError, print_msg -from easybuild.tools.config import build_option, update_build_option -from easybuild.tools.filetools import apply_regex_substitutions, copy_file, remove_file, symlink, which +from easybuild.tools.config import build_option, install_path, update_build_option +from easybuild.tools.filetools import apply_regex_substitutions, copy_dir, copy_file, remove_file, symlink, which from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture, get_cpu_features from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC @@ -46,6 +48,8 @@ # Make sure a single environment variable name is used for this throughout the hooks EESSI_IGNORE_ZEN4_GCC1220_ENVVAR="EESSI_IGNORE_LMOD_ERROR_ZEN4_GCC1220" +STACK_REPROD_SUBDIR = 'reprod' + def is_gcccore_1220_based(**kwargs): # ecname, ecversion, tcname, tcversion): """ @@ -516,6 +520,26 @@ def post_module_hook_zen4_gcccore1220(self, *args, **kwargs): del self.initial_environ[EESSI_IGNORE_ZEN4_GCC1220_ENVVAR] +def post_easyblock_hook_copy_exts_patches_to_easybuild_subdir(self, *args, **kwargs): + """Copy the patches of extensions to the easybuild subdir of the installation directory.""" + + # Temporary fix for https://github.com/easybuilders/easybuild-framework/issues/4864: + # make sure that the patches of extensions are copied to the easybuild subdirectory + for ext in self.exts: + for patch in ext.get('patches', []): + new_log_dir = os.path.join(self.installdir, config.log_path(ec=self.cfg)) + target = os.path.join(new_log_dir, os.path.basename(patch['path'])) + copy_file(patch['path'], target) + + # Now we simply copy the entire "easybuild" subdirectory of the installed application + # to a timestamped subdirectory of the stack's central reprod directory, e.g.: + # + stack_reprod_dir = os.path.join(os.path.dirname(install_path()), STACK_REPROD_SUBDIR) + now_utc_timestamp = datetime.datetime.now(datetime.UTC).isoformat('T', 'seconds').replace('+00:00', 'Z') + app_reprod_dir = os.path.join(stack_reprod_dir, self.install_subdir, now_utc_timestamp) + copy_dir(new_log_dir, app_reprod_dir) + + # Modules for dependencies are loaded in the prepare step. Thus, that's where we need this variable to be set # so that the modules can be succesfully loaded without printing the error (so that we can create a module # _with_ the warning for the current software being installed) @@ -1297,6 +1321,31 @@ def post_module_hook(self, *args, **kwargs): post_module_hook_zen4_gcccore1220(self, *args, **kwargs) +def post_easyblock_hook(self, *args, **kwargs): + """Main post easyblock hook: trigger custom functions based on software name.""" + if self.name in POST_EASYBLOCK_HOOKS: + POST_EASYBLOCK_HOOKS[self.name](self, *args, **kwargs) + + # Always trigger this one, regardless of self.name + post_easyblock_hook_copy_exts_patches_to_easybuild_subdir(self, *args, **kwargs) + + +def post_build_and_install_loop_hook(ecs, *args, **kwargs): + """ + Post build and install loop hook that copies the easybuild subdirectory of every installed application + to a central location in the root of the software stack, e.g.: + /path/to/stack/reprod/20250102T12:34:56Z/MyApp/1.2-foss-2025a + """ + + stack_reprod_dir = os.path.join(os.path.dirname(install_path()), STACK_REPROD_SUBDIR) + for (ec, status) in ecs: + if status['success']: + now_timestamp = datetime.datetime.now(datetime.UTC).isoformat('T', 'seconds').replace('+00:00', 'Z') + app_easybuild_dir = os.path.dirname(status['log_file']) + app_reprod_dir = os.path.join(stack_reprod_dir, ec['short_mod_name'], now_timestamp, 'easybuild') + copy_dir(app_easybuild_dir, app_reprod_dir) + + PARSE_HOOKS = { 'casacore': parse_hook_casacore_disable_vectorize, 'CGAL': parse_hook_cgal_toolchainopts_precise, @@ -1365,6 +1414,8 @@ def post_module_hook(self, *args, **kwargs): POST_MODULE_HOOKS = {} +POST_EASYBLOCK_HOOKS = {} + # Define parallelism limit operations def divide_by_factor(parallel, factor): """Divide parallelism by given factor""" From ce0e0feac5ed5a502686c34a8738ef4c2e277481 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 27 Jun 2025 09:43:36 +0200 Subject: [PATCH 2/2] remove temporary workaround that copies extension patches, clean up, only copy easybuild subdir with post_easyblock_hook --- eb_hooks.py | 44 +++++++++++--------------------------------- 1 file changed, 11 insertions(+), 33 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index b2a33cc..a279729 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -520,24 +520,18 @@ def post_module_hook_zen4_gcccore1220(self, *args, **kwargs): del self.initial_environ[EESSI_IGNORE_ZEN4_GCC1220_ENVVAR] -def post_easyblock_hook_copy_exts_patches_to_easybuild_subdir(self, *args, **kwargs): - """Copy the patches of extensions to the easybuild subdir of the installation directory.""" - - # Temporary fix for https://github.com/easybuilders/easybuild-framework/issues/4864: - # make sure that the patches of extensions are copied to the easybuild subdirectory - for ext in self.exts: - for patch in ext.get('patches', []): - new_log_dir = os.path.join(self.installdir, config.log_path(ec=self.cfg)) - target = os.path.join(new_log_dir, os.path.basename(patch['path'])) - copy_file(patch['path'], target) - - # Now we simply copy the entire "easybuild" subdirectory of the installed application - # to a timestamped subdirectory of the stack's central reprod directory, e.g.: - # +def post_easyblock_hook_copy_easybuild_subdir(self, *args, **kwargs): + """ + Post easyblock hook that copies the easybuild subdirectory of every installed application + to a central and timestamped location in the root of the software stack, e.g.: + /path/to/stack/reprod/20250102T12:34:56Z/MyApp/1.2-foss-2025a + """ + stack_reprod_dir = os.path.join(os.path.dirname(install_path()), STACK_REPROD_SUBDIR) now_utc_timestamp = datetime.datetime.now(datetime.UTC).isoformat('T', 'seconds').replace('+00:00', 'Z') - app_reprod_dir = os.path.join(stack_reprod_dir, self.install_subdir, now_utc_timestamp) - copy_dir(new_log_dir, app_reprod_dir) + app_easybuild_dir = os.path.join(self.installdir, config.log_path(ec=self.cfg)) + app_reprod_dir = os.path.join(stack_reprod_dir, self.install_subdir, now_utc_timestamp, 'easybuild') + copy_dir(app_easybuild_dir, app_reprod_dir) # Modules for dependencies are loaded in the prepare step. Thus, that's where we need this variable to be set @@ -1327,23 +1321,7 @@ def post_easyblock_hook(self, *args, **kwargs): POST_EASYBLOCK_HOOKS[self.name](self, *args, **kwargs) # Always trigger this one, regardless of self.name - post_easyblock_hook_copy_exts_patches_to_easybuild_subdir(self, *args, **kwargs) - - -def post_build_and_install_loop_hook(ecs, *args, **kwargs): - """ - Post build and install loop hook that copies the easybuild subdirectory of every installed application - to a central location in the root of the software stack, e.g.: - /path/to/stack/reprod/20250102T12:34:56Z/MyApp/1.2-foss-2025a - """ - - stack_reprod_dir = os.path.join(os.path.dirname(install_path()), STACK_REPROD_SUBDIR) - for (ec, status) in ecs: - if status['success']: - now_timestamp = datetime.datetime.now(datetime.UTC).isoformat('T', 'seconds').replace('+00:00', 'Z') - app_easybuild_dir = os.path.dirname(status['log_file']) - app_reprod_dir = os.path.join(stack_reprod_dir, ec['short_mod_name'], now_timestamp, 'easybuild') - copy_dir(app_easybuild_dir, app_reprod_dir) + post_easyblock_hook_copy_easybuild_subdir(self, *args, **kwargs) PARSE_HOOKS = {