Skip to content

Commit bad829e

Browse files
boegelboegel
committed
Merge branch 'main-software-layer-scripts' of /tmp/software-layer
2 parents 7f31c3b + 6f10355 commit bad829e

File tree

149 files changed

+11661
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

149 files changed

+11661
-0
lines changed

‎.github/workflows/modules/fake_module.lua

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
setenv("INSIDE_GITHUB_ACTIONS", "true")
2+
-- Interfere with PATH so Lmod keeps a record
3+
prepend_path("PATH", "/snap/bin")

‎.github/workflows/scorecards.yml

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
# This workflow uses actions that are not certified by GitHub. They are provided
2+
# by a third-party and are governed by separate terms of service, privacy
3+
# policy, and support documentation.
4+
5+
name: Scorecards supply-chain security
6+
on:
7+
# To guarantee Maintained check is occasionally updated. See
8+
# https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained
9+
schedule:
10+
- cron: '25 15 * * 3'
11+
push:
12+
branches:
13+
- '*-software.eessi.io'
14+
15+
# Declare default permissions as read only.
16+
permissions: read-all
17+
18+
jobs:
19+
analysis:
20+
if: github.repository_owner == 'EESSI' # Prevent running on forks
21+
name: Scorecards analysis
22+
runs-on: ubuntu-24.04
23+
permissions:
24+
# Needed to upload the results to code-scanning dashboard.
25+
security-events: write
26+
# Needed to publish results and get a badge (see publish_results below).
27+
id-token: write
28+
# Uncomment the permissions below if installing in a private repository.
29+
# contents: read
30+
# actions: read
31+
32+
steps:
33+
- name: "Checkout code"
34+
uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
35+
with:
36+
persist-credentials: false
37+
38+
- name: "Run analysis"
39+
uses: ossf/scorecard-action@dc50aa9510b46c811795eb24b2f1ba02a914e534 # v2.3.3
40+
with:
41+
results_file: results.sarif
42+
results_format: sarif
43+
# (Optional) "write" PAT token. Uncomment the `repo_token` line below if:
44+
# - you want to enable the Branch-Protection check on a *public* repository, or
45+
# - you are installing Scorecards on a *private* repository
46+
# To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat.
47+
# repo_token: ${{ secrets.SCORECARD_TOKEN }}
48+
49+
# Public repositories:
50+
# - Publish results to OpenSSF REST API for easy access by consumers
51+
# - Allows the repository to include the Scorecard badge.
52+
# - See https://github.com/ossf/scorecard-action#publishing-results.
53+
# For private repositories:
54+
# - `publish_results` will always be set to `false`, regardless
55+
# of the value entered here.
56+
publish_results: true
57+
58+
# Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
59+
# format to the repository Actions tab.
60+
- name: "Upload artifact"
61+
uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
62+
with:
63+
name: SARIF file
64+
path: results.sarif
65+
retention-days: 5
66+
67+
# Upload the results to GitHub's code scanning dashboard.
68+
- name: "Upload to code-scanning"
69+
uses: github/codeql-action/upload-sarif@9fdb3e49720b44c48891d036bb502feb25684276 # v3.25.6
70+
with:
71+
sarif_file: results.sarif

‎.github/workflows/scripts/compare_stacks.py

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
import argparse
2+
import os
3+
import re
4+
import glob
5+
import json
6+
7+
8+
def parse_module_file(module_file_path):
9+
"""
10+
Extracts module name, version, and extensions from a module file.
11+
"""
12+
module_name = os.path.basename(os.path.dirname(module_file_path))
13+
version = os.path.basename(module_file_path)
14+
15+
try:
16+
with open(module_file_path, "r") as file:
17+
content = file.read()
18+
19+
# Extract extensions from content using regex
20+
match = re.search(r'extensions\("(.+)"\)', content)
21+
extensions = []
22+
23+
if match:
24+
# Split the list of packages by commas
25+
packages = match.group(1)
26+
for pkg in packages.split(","):
27+
parts = pkg.split("/")
28+
29+
# Check if the package is in the name/version format
30+
if len(parts) == 2:
31+
extensions.append((parts[0], parts[1]))
32+
elif len(parts) == 1:
33+
extensions.append((parts[0], "none"))
34+
else:
35+
print(f"Warning: Skipping invalid package format: {pkg}")
36+
37+
return {(module_name, version): tuple(extensions)}
38+
39+
except Exception as e:
40+
print(f"Error parsing module file {module_file_path}: {e}")
41+
return {(module_name, version): ()}
42+
43+
44+
def get_available_modules(base_dir):
45+
"""
46+
Get the list of modules from all subdirectories inside the specified base directory.
47+
"""
48+
try:
49+
modules = {}
50+
# Only look for .lua files
51+
for module_path in glob.glob(os.path.join(base_dir, "*/*.lua")):
52+
modules.update(parse_module_file(module_path))
53+
return modules
54+
55+
except Exception as e:
56+
print(f"Error retrieving modules from {base_dir}: {e}")
57+
return {}
58+
59+
60+
def compare_stacks(dir1, dir2):
61+
"""
62+
Compare two sets of Lmod module files, including versions and extensions.
63+
"""
64+
modules1 = get_available_modules(dir1)
65+
modules2 = get_available_modules(dir2)
66+
67+
# Find differences between the two dictionaries
68+
modules_removed = set(modules1.keys()) - set(modules2.keys())
69+
modules_added = set(modules2.keys()) - set(modules1.keys())
70+
matching_keys = set(modules1.keys()) & set(modules2.keys())
71+
72+
diff_results = {
73+
"module_differences": {
74+
"missing": list("/".join(module) for module in modules_removed),
75+
"added": list("/".join(module) for module in modules_added),
76+
},
77+
"extension_differences": [],
78+
}
79+
80+
# Compare extensions for matching keys
81+
for key in matching_keys:
82+
if modules1[key] != modules2[key]:
83+
diff_results["extension_differences"].append(
84+
{
85+
"/".join(key): {
86+
"missing": list(
87+
"/".join(key)
88+
for key in list(set(modules1[key]) - set(modules2[key]))
89+
),
90+
"added": list(
91+
"/".join(key)
92+
for key in list(set(modules2[key]) - set(modules1[key]))
93+
),
94+
}
95+
}
96+
)
97+
98+
return diff_results
99+
100+
101+
def main():
102+
# Set up argument parser
103+
parser = argparse.ArgumentParser(description="Compare two Lmod module directories")
104+
parser.add_argument("path1", type=str, help="The first directory path")
105+
parser.add_argument("path2", type=str, help="The second directory path")
106+
107+
# Parse the arguments
108+
args = parser.parse_args()
109+
110+
# Validate the paths
111+
for path in [args.path1, args.path2]:
112+
if not os.path.exists(path):
113+
print(f"Warning: Path does not exist: {path}")
114+
115+
# Compare the stacks
116+
diff_results = compare_stacks(args.path1, args.path2)
117+
118+
# Print the differences
119+
if any(
120+
[
121+
diff_results["module_differences"]["missing"],
122+
diff_results["module_differences"]["added"],
123+
diff_results["extension_differences"],
124+
]
125+
):
126+
print(json.dumps(diff_results, indent=2))
127+
exit(1)
128+
129+
130+
if __name__ == "__main__":
131+
main()

‎.github/workflows/scripts/compare_to_generic.sh

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/usr/bin/env bash
2+
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
3+
# Take the arguments
4+
base_dir=$1
5+
target_arch=$2
6+
modules_subdir="modules/all"
7+
# Decide if we want x86_64 or aarch64
8+
arch=$(echo $target_arch | cut -d"/" -f1)
9+
# Get the generic directory
10+
source_of_truth="$arch/generic"
11+
case $arch in
12+
"x86_64")
13+
echo "Using $source_of_truth as source of truth"
14+
;;
15+
"aarch64")
16+
echo "Using $source_of_truth as source of truth"
17+
;;
18+
*)
19+
echo "I don't understand the base architecture: $arch"
20+
exit 1
21+
;;
22+
esac
23+
source_of_truth_modules="$base_dir/$source_of_truth/$modules_subdir"
24+
arch_modules="$base_dir/$target_arch/$modules_subdir"
25+
echo "Comparing $arch_modules to $source_of_truth_modules"
26+
python3 $script_dir/compare_stacks.py $source_of_truth_modules $arch_modules

‎.github/workflows/scripts/only_latest_easystacks.sh

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#!/bin/bash
2+
#
3+
# This script figures out the latest version of EasyBuild being used for the installation of easystack
4+
# files.
5+
#
6+
# This file is part of the EESSI software layer, see
7+
# https://github.com/EESSI/software-layer.git
8+
#
9+
# author: Alan O'Cais (CECAM)
10+
#
11+
# license: GPLv2
12+
#
13+
14+
EESSI_VERSION=${EESSI_VERSION:-"2023.06"}
15+
16+
directory="easystacks/software.eessi.io/${EESSI_VERSION}"
17+
# List of example filenames
18+
files=($(find "$directory" -name "*.yml" | grep -e '-eb-'))
19+
[ -n "$DEBUG" ] && echo "${files[@]}"
20+
21+
versions=()
22+
# Loop over each filename
23+
for filename in "${files[@]}"; do
24+
# Extract the semantic version using grep
25+
version=$(echo "$filename" | grep -oP '(?<=eb-)\d+\.\d+\.\d+?(?=-)')
26+
27+
# Output the result
28+
[ -n "$DEBUG" ] && echo "Filename: $filename"
29+
[ -n "$DEBUG" ] && echo "Extracted version: $version"
30+
[ -n "$DEBUG" ] && echo
31+
versions+=("$version")
32+
done
33+
highest_version=$(printf "%s\n" "${versions[@]}" | sort -V | tail -n 1)
34+
35+
[ -n "$DEBUG" ] && echo "Highest version: $highest_version"
36+
[ -n "$DEBUG" ] && echo
37+
[ -n "$DEBUG" ] && echo "Matching files:"
38+
all_latest_easystacks=($(find $directory -type f -name "*eb-$highest_version*.yml"))
39+
40+
accel_latest_easystacks=()
41+
cpu_latest_easystacks=()
42+
43+
# Loop through the array and split based on partial matching of string
44+
accel="/accel/"
45+
for item in "${all_latest_easystacks[@]}"; do
46+
if [[ "$item" == *"$accel"* ]]; then
47+
accel_latest_easystacks+=("$item")
48+
else
49+
cpu_latest_easystacks+=("$item")
50+
fi
51+
done
52+
53+
# Output the results
54+
if [ -n "$ACCEL_EASYSTACKS" ]; then
55+
echo "${accel_latest_easystacks[@]}"
56+
else
57+
echo "${cpu_latest_easystacks[@]}"
58+
fi

‎.github/workflows/scripts/test_init_scripts.sh

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#!/bin/bash
2+
EESSI_VERSION="2023.06"
3+
export LMOD_PAGER=cat
4+
5+
# initialize assert framework
6+
if [ ! -d assert.sh ]; then
7+
echo "assert.sh not cloned."
8+
echo ""
9+
echo "run \`git clone https://github.com/lehmannro/assert.sh.git\`"
10+
exit 1
11+
fi
12+
. assert.sh/assert.sh
13+
14+
TEST_SHELLS=("bash" "zsh" "fish" "ksh")
15+
SHELLS=$@
16+
17+
for shell in ${SHELLS[@]}; do
18+
echo = | awk 'NF += (OFS = $_) + 100'
19+
echo RUNNING TESTS FOR SHELL: $shell
20+
echo = | awk 'NF += (OFS = $_) + 100'
21+
if [[ ! " ${TEST_SHELLS[*]} " =~ [[:space:]]${shell}[[:space:]] ]]; then
22+
### EXCEPTION FOR CSH ###
23+
echo -e "\033[33mWe don't now how to test the shell '$shell', PRs are Welcome.\033[0m"
24+
else
25+
# TEST 1: Source Script and check Module Output
26+
assert "$shell -c 'source init/lmod/$shell' 2>&1 " "EESSI/$EESSI_VERSION loaded successfully"
27+
# TEST 2: Check if module overviews first section is the loaded EESSI module
28+
MODULE_SECTIONS=($($shell -c "source init/lmod/$shell 2>/dev/null; module ov 2>&1 | grep -e '---'"))
29+
PATTERN="/cvmfs/software\.eessi\.io/versions/$EESSI_VERSION/software/linux/x86_64/(intel/haswell|amd/zen3)/modules/all"
30+
assert_raises 'echo "${MODULE_SECTIONS[1]}" | grep -E "$PATTERN"'
31+
# TEST 3: Check if module overviews second section is the EESSI init module
32+
assert "echo ${MODULE_SECTIONS[4]}" "/cvmfs/software.eessi.io/versions/$EESSI_VERSION/init/modules"
33+
# Test 4: Load Python module and check version
34+
command="$shell -c 'source init/lmod/$shell 2>/dev/null; module load Python/3.10.8-GCCcore-12.2.0; python --version'"
35+
expected="Python 3.10.8"
36+
assert "$command" "$expected"
37+
# Test 5: Load Python module and check path
38+
PYTHON_PATH=$($shell -c "source init/lmod/$shell 2>/dev/null; module load Python/3.10.8-GCCcore-12.2.0; which python")
39+
PATTERN="/cvmfs/software\.eessi\.io/versions/$EESSI_VERSION/software/linux/x86_64/(intel/haswell|amd/zen3)/software/Python/3\.10\.8-GCCcore-12\.2\.0/bin/python"
40+
echo "$PYTHON_PATH" | grep -E "$PATTERN"
41+
assert_raises 'echo "$PYTHON_PATH" | grep -E "$PATTERN"'
42+
43+
#End Test Suite
44+
assert_end "source_eessi_$shell"
45+
fi
46+
done
47+
48+
49+
# RESET PAGER
50+
export LMOD_PAGER=

0 commit comments

Comments
 (0)