Skip to content

WIP: hooks for improving reproducibility #24

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 31 additions & 2 deletions eb_hooks.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,17 @@
# Hooks to customize how EasyBuild installs software in EESSI
# see https://docs.easybuild.io/en/latest/Hooks.html
import datetime
import glob
import os
import re

import easybuild.tools.environment as env
from easybuild.easyblocks.generic.configuremake import obtain_config_guess
from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS
from easybuild.tools import config
from easybuild.tools.build_log import EasyBuildError, print_msg
from easybuild.tools.config import build_option, update_build_option
from easybuild.tools.filetools import apply_regex_substitutions, copy_file, remove_file, symlink, which
from easybuild.tools.config import build_option, install_path, update_build_option
from easybuild.tools.filetools import apply_regex_substitutions, copy_dir, copy_file, remove_file, symlink, which
from easybuild.tools.run import run_cmd
from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture, get_cpu_features
from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC
Expand Down Expand Up @@ -46,6 +48,8 @@
# Make sure a single environment variable name is used for this throughout the hooks
EESSI_IGNORE_ZEN4_GCC1220_ENVVAR="EESSI_IGNORE_LMOD_ERROR_ZEN4_GCC1220"

STACK_REPROD_SUBDIR = 'reprod'

def is_gcccore_1220_based(**kwargs):
# ecname, ecversion, tcname, tcversion):
"""
Expand Down Expand Up @@ -516,6 +520,20 @@ def post_module_hook_zen4_gcccore1220(self, *args, **kwargs):
del self.initial_environ[EESSI_IGNORE_ZEN4_GCC1220_ENVVAR]


def post_easyblock_hook_copy_easybuild_subdir(self, *args, **kwargs):
"""
Post easyblock hook that copies the easybuild subdirectory of every installed application
to a central and timestamped location in the root of the software stack, e.g.:
/path/to/stack/reprod/20250102T12:34:56Z/MyApp/1.2-foss-2025a
"""

stack_reprod_dir = os.path.join(os.path.dirname(install_path()), STACK_REPROD_SUBDIR)
now_utc_timestamp = datetime.datetime.now(datetime.UTC).isoformat('T', 'seconds').replace('+00:00', 'Z')
app_easybuild_dir = os.path.join(self.installdir, config.log_path(ec=self.cfg))
app_reprod_dir = os.path.join(stack_reprod_dir, self.install_subdir, now_utc_timestamp, 'easybuild')
copy_dir(app_easybuild_dir, app_reprod_dir)


# Modules for dependencies are loaded in the prepare step. Thus, that's where we need this variable to be set
# so that the modules can be succesfully loaded without printing the error (so that we can create a module
# _with_ the warning for the current software being installed)
Expand Down Expand Up @@ -1297,6 +1315,15 @@ def post_module_hook(self, *args, **kwargs):
post_module_hook_zen4_gcccore1220(self, *args, **kwargs)


def post_easyblock_hook(self, *args, **kwargs):
"""Main post easyblock hook: trigger custom functions based on software name."""
if self.name in POST_EASYBLOCK_HOOKS:
POST_EASYBLOCK_HOOKS[self.name](self, *args, **kwargs)

# Always trigger this one, regardless of self.name
post_easyblock_hook_copy_easybuild_subdir(self, *args, **kwargs)


PARSE_HOOKS = {
'casacore': parse_hook_casacore_disable_vectorize,
'CGAL': parse_hook_cgal_toolchainopts_precise,
Expand Down Expand Up @@ -1365,6 +1392,8 @@ def post_module_hook(self, *args, **kwargs):

POST_MODULE_HOOKS = {}

POST_EASYBLOCK_HOOKS = {}

# Define parallelism limit operations
def divide_by_factor(parallel, factor):
"""Divide parallelism by given factor"""
Expand Down
Loading