diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index f48433e23c..fd22e384e5 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1969,7 +1969,7 @@ def set_tmpdir(tmpdir=None, raise_error=False): os.close(fd) os.chmod(tmptest_file, 0o700) if not run_cmd(tmptest_file, simple=True, log_ok=False, regexp=False, force_in_dry_run=True, trace=False, - stream_output=False, with_hooks=False): + stream_output=False, with_hooks=False, with_sysroot=False): msg = "The temporary directory (%s) does not allow to execute files. " % tempfile.gettempdir() msg += "This can cause problems in the build process, consider using --tmpdir." if raise_error: diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 30f413be55..f8d034f981 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -134,7 +134,7 @@ def get_output_from_process(proc, read_size=None, asynchronous=False): @run_cmd_cache def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None, force_in_dry_run=False, verbose=True, shell=None, trace=True, stream_output=None, asynchronous=False, - with_hooks=True): + with_hooks=True, with_sysroot=True): """ Run specified command (in a subshell) :param cmd: command to run @@ -152,6 +152,7 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True :param stream_output: enable streaming command output to stdout :param asynchronous: run command asynchronously (returns subprocess.Popen instance if set to True) :param with_hooks: trigger pre/post run_shell_cmd hooks (if defined) + :param with_sysroot: prepend sysroot to exec_cmd (if defined) """ cwd = os.getcwd() @@ -228,6 +229,16 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True exec_cmd = "/bin/bash" + # if EasyBuild is configured to use an alternate sysroot, + # we should also run shell commands using the bash shell provided in there, + # since /bin/bash may not be compatible with the alternate sysroot + if with_sysroot: + sysroot = build_option('sysroot') + if sysroot: + sysroot_bin_bash = os.path.join(sysroot, 'bin', 'bash') + if os.path.exists(sysroot_bin_bash): + exec_cmd = sysroot_bin_bash + if not shell: if isinstance(cmd, list): exec_cmd = None @@ -237,6 +248,8 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True else: raise EasyBuildError("Don't know how to prefix with /usr/bin/env for commands of type %s", type(cmd)) + _log.info("Using %s as shell for running cmd: %s", exec_cmd, cmd) + if with_hooks: hooks = load_hooks(build_option('hooks')) hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs={'work_dir': os.getcwd()}) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 24e910fb94..68c42fe416 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -275,7 +275,8 @@ def get_avail_core_count(): core_cnt = int(sum(sched_getaffinity())) else: # BSD-type systems - out, _ = run_cmd('sysctl -n hw.ncpu', force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False) + out, _ = run_cmd('sysctl -n hw.ncpu', force_in_dry_run=True, trace=False, stream_output=False, + with_hooks=False, with_sysroot=False) try: if int(out) > 0: core_cnt = int(out) @@ -312,7 +313,8 @@ def get_total_memory(): elif os_type == DARWIN: cmd = "sysctl -n hw.memsize" _log.debug("Trying to determine total memory size on Darwin via cmd '%s'", cmd) - out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False) + out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False, + with_sysroot=False) if ec == 0: memtotal = int(out.strip()) // (1024**2) @@ -394,7 +396,8 @@ def get_cpu_vendor(): elif os_type == DARWIN: cmd = "sysctl -n machdep.cpu.vendor" - out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False, with_hooks=False) + out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False, + with_hooks=False, with_sysroot=False) out = out.strip() if ec == 0 and out in VENDOR_IDS: vendor = VENDOR_IDS[out] @@ -402,7 +405,7 @@ def get_cpu_vendor(): else: cmd = "sysctl -n machdep.cpu.brand_string" out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False, - with_hooks=False) + with_hooks=False, with_sysroot=False) out = out.strip().split(' ')[0] if ec == 0 and out in CPU_VENDORS: vendor = out @@ -505,7 +508,8 @@ def get_cpu_model(): elif os_type == DARWIN: cmd = "sysctl -n machdep.cpu.brand_string" - out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False) + out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False, + with_sysroot=False) if ec == 0: model = out.strip() _log.debug("Determined CPU model on Darwin using cmd '%s': %s" % (cmd, model)) @@ -550,7 +554,8 @@ def get_cpu_speed(): elif os_type == DARWIN: cmd = "sysctl -n hw.cpufrequency_max" _log.debug("Trying to determine CPU frequency on Darwin via cmd '%s'" % cmd) - out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False) + out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False, + with_sysroot=False) out = out.strip() cpu_freq = None if ec == 0 and out: @@ -599,7 +604,7 @@ def get_cpu_features(): cmd = "sysctl -n machdep.cpu.%s" % feature_set _log.debug("Trying to determine CPU features on Darwin via cmd '%s'", cmd) out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False, - with_hooks=False) + with_hooks=False, with_sysroot=False) if ec == 0: cpu_feat.extend(out.strip().lower().split()) @@ -626,8 +631,8 @@ def get_gpu_info(): try: cmd = "nvidia-smi --query-gpu=gpu_name,driver_version --format=csv,noheader" _log.debug("Trying to determine NVIDIA GPU info on Linux via cmd '%s'", cmd) - out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False, - force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False) + out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False, force_in_dry_run=True, + trace=False, stream_output=False, with_hooks=False, with_sysroot=False) if ec == 0: for line in out.strip().split('\n'): nvidia_gpu_info = gpu_info.setdefault('NVIDIA', {}) @@ -645,15 +650,15 @@ def get_gpu_info(): try: cmd = "rocm-smi --showdriverversion --csv" _log.debug("Trying to determine AMD GPU driver on Linux via cmd '%s'", cmd) - out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False, - force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False) + out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False, force_in_dry_run=True, + trace=False, stream_output=False, with_hooks=False, with_sysroot=False) if ec == 0: amd_driver = out.strip().split('\n')[1].split(',')[1] cmd = "rocm-smi --showproductname --csv" _log.debug("Trying to determine AMD GPU info on Linux via cmd '%s'", cmd) - out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False, - force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False) + out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False, force_in_dry_run=True, + trace=False, stream_output=False, with_hooks=False, with_sysroot=False) if ec == 0: for line in out.strip().split('\n')[1:]: amd_card_series = line.split(',')[1] @@ -900,7 +905,7 @@ def get_tool_version(tool, version_option='--version', ignore_ec=False): Output is returned as a single-line string (newlines are replaced by '; '). """ out, ec = run_cmd(' '.join([tool, version_option]), simple=False, log_ok=False, force_in_dry_run=True, - trace=False, stream_output=False, with_hooks=False) + trace=False, stream_output=False, with_hooks=False, with_sysroot=False) if not ignore_ec and ec: _log.warning("Failed to determine version of %s using '%s %s': %s" % (tool, tool, version_option, out)) return UNKNOWN diff --git a/test/framework/run.py b/test/framework/run.py index bab4391cf6..bb2e5c0558 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -796,6 +796,31 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs): ]) self.assertEqual(stdout, expected_stdout) + def test_run_cmd_sysroot(self): + """Test with_sysroot option of run_cmd function.""" + + # put fake /bin/bash in place that will be picked up when using run_cmd with with_sysroot=True + bin_bash = os.path.join(self.test_prefix, 'bin', 'bash') + bin_bash_txt = '\n'.join([ + "#!/bin/bash", + "echo 'Hi there I am a fake /bin/bash in %s'" % self.test_prefix, + '/bin/bash "$@"', + ]) + write_file(bin_bash, bin_bash_txt) + adjust_permissions(bin_bash, stat.S_IXUSR) + + update_build_option('sysroot', self.test_prefix) + + (out, ec) = run_cmd("echo hello") + self.assertEqual(ec, 0) + self.assertTrue(out.startswith("Hi there I am a fake /bin/bash in")) + self.assertTrue(out.endswith("\nhello\n")) + + # picking up on alternate sysroot is enabled by default, but can be disabled via with_sysroot=False + (out, ec) = run_cmd("echo hello", with_sysroot=False) + self.assertEqual(ec, 0) + self.assertEqual(out, "hello\n") + def suite(): """ returns all the testcases in this module """