Skip to content

Commit fe8d312

Browse files
q10spcyppt
q10
authored and
spcyppt
committed
Change scripts to accommodate conda prefix to work with Nova (#2022)
Summary: Pull Request resolved: #2022 Pull Request resolved: #2021 Nova creates conda environments with `--prefix` where the environments are outside of the default `envs` folder. Current script only works with `--name`. This diff changes the scripts to accommodate conda prefix. https://github.com/pytorch/FBGEMM/actions/runs/6189731547 Reviewed By: spcyppt Differential Revision: D49306286 fbshipit-source-id: 2f2ca00645526639369de3f555dbab30da56739e
1 parent 1f8938e commit fe8d312

12 files changed

+220
-63
lines changed

‎.github/scripts/fbgemm_gpu_build.bash

Lines changed: 30 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -38,10 +38,16 @@ prepare_fbgemm_gpu_build () {
3838
git submodule sync
3939
git submodule update --init --recursive
4040

41+
# shellcheck disable=SC2155
42+
local env_prefix=$(env_name_or_prefix "${env_name}")
43+
4144
echo "[BUILD] Installing other build dependencies ..."
42-
(exec_with_retries conda run --no-capture-output -n "${env_name}" python -m pip install -r requirements.txt) || return 1
45+
# shellcheck disable=SC2086
46+
(exec_with_retries conda run --no-capture-output ${env_prefix} python -m pip install -r requirements.txt) || return 1
4347

48+
# shellcheck disable=SC2086
4449
(test_python_import_package "${env_name}" numpy) || return 1
50+
# shellcheck disable=SC2086
4551
(test_python_import_package "${env_name}" skbuild) || return 1
4652

4753
echo "[BUILD] Successfully ran git submodules update"
@@ -76,7 +82,8 @@ __configure_fbgemm_gpu_build_rocm () {
7682
fi
7783

7884
echo "[BUILD] Setting the following ROCm targets: ${arch_list}"
79-
print_exec conda env config vars set -n "${env_name}" PYTORCH_ROCM_ARCH="${arch_list}"
85+
# shellcheck disable=SC2086
86+
print_exec conda env config vars set ${env_prefix} PYTORCH_ROCM_ARCH="${arch_list}"
8087

8188
echo "[BUILD] Setting ROCm build args ..."
8289
build_args=()
@@ -121,8 +128,8 @@ __configure_fbgemm_gpu_build_cuda () {
121128

122129
# Build only CUDA 7.0 and 8.0 (i.e. V100 and A100) because of 100 MB binary size limits from PyPI.
123130
echo "[BUILD] Setting CUDA build args ..."
124-
# shellcheck disable=SC2155
125-
local nvml_lib_path=$(conda run --no-capture-output -n "${env_name}" printenv NVML_LIB_PATH)
131+
# shellcheck disable=SC2155,SC2086
132+
local nvml_lib_path=$(conda run --no-capture-output ${env_prefix} printenv NVML_LIB_PATH)
126133
build_args=(
127134
--nvml_lib_path="${nvml_lib_path}"
128135
-DTORCH_CUDA_ARCH_LIST="'${arch_list}'"
@@ -192,16 +199,17 @@ __build_fbgemm_gpu_common_pre_steps () {
192199
echo "[BUILD] Determined Python package name to use: ${package_name}"
193200

194201
# Extract the Python tag
195-
# shellcheck disable=SC2207
196-
python_version=($(conda run --no-capture-output -n "${env_name}" python --version))
202+
# shellcheck disable=SC2207,SC2086
203+
python_version=($(conda run --no-capture-output ${env_prefix} python --version))
197204
# shellcheck disable=SC2206
198205
python_version_arr=(${python_version[1]//./ })
199206
python_tag="py${python_version_arr[0]}${python_version_arr[1]}"
200207
echo "[BUILD] Extracted Python tag: ${python_tag}"
201208

202209
echo "[BUILD] Running pre-build cleanups ..."
203210
print_exec rm -rf dist
204-
print_exec conda run --no-capture-output -n "${env_name}" python setup.py clean
211+
# shellcheck disable=SC2086
212+
print_exec conda run --no-capture-output ${env_prefix} python setup.py clean
205213

206214
echo "[BUILD] Printing git status ..."
207215
print_exec git status
@@ -290,6 +298,9 @@ build_fbgemm_gpu_package () {
290298
return 1
291299
fi
292300

301+
# shellcheck disable=SC2155
302+
local env_prefix=$(env_name_or_prefix "${env_name}")
303+
293304
# Set up and configure the build
294305
__build_fbgemm_gpu_common_pre_steps || return 1
295306
__configure_fbgemm_gpu_build "${fbgemm_variant}" "${fbgemm_variant_targets}" || return 1
@@ -320,7 +331,8 @@ build_fbgemm_gpu_package () {
320331

321332
# Distribute Python extensions as wheels on Linux
322333
echo "[BUILD] Building FBGEMM-GPU wheel (VARIANT=${fbgemm_variant}) ..."
323-
print_exec conda run --no-capture-output -n "${env_name}" \
334+
# shellcheck disable=SC2086
335+
print_exec conda run --no-capture-output ${env_prefix} \
324336
python setup.py "${run_multicore}" bdist_wheel \
325337
--package_name="${package_name}" \
326338
--python-tag="${python_tag}" \
@@ -356,6 +368,9 @@ build_fbgemm_gpu_install () {
356368
return 1
357369
fi
358370

371+
# shellcheck disable=SC2155
372+
local env_prefix=$(env_name_or_prefix "${env_name}")
373+
359374
# Set up and configure the build
360375
__build_fbgemm_gpu_common_pre_steps || return 1
361376
__configure_fbgemm_gpu_build "${fbgemm_variant}" "${fbgemm_variant_targets}" || return 1
@@ -370,7 +385,8 @@ build_fbgemm_gpu_install () {
370385
# Parallelism may need to be limited to prevent the build from being
371386
# canceled for going over ulimits
372387
echo "[BUILD] Building + installing FBGEMM-GPU (VARIANT=${fbgemm_variant}) ..."
373-
print_exec conda run --no-capture-output -n "${env_name}" \
388+
# shellcheck disable=SC2086
389+
print_exec conda run --no-capture-output ${env_prefix} \
374390
python setup.py install "${build_args[@]}"
375391

376392
# Run checks on the built libraries
@@ -400,6 +416,9 @@ build_fbgemm_gpu_develop () {
400416
return 1
401417
fi
402418

419+
# shellcheck disable=SC2155
420+
local env_prefix=$(env_name_or_prefix "${env_name}")
421+
403422
# Set up and configure the build
404423
__build_fbgemm_gpu_common_pre_steps || return 1
405424
__configure_fbgemm_gpu_build "${fbgemm_variant}" "${fbgemm_variant_targets}" || return 1
@@ -414,7 +433,8 @@ build_fbgemm_gpu_develop () {
414433
# Parallelism may need to be limited to prevent the build from being
415434
# canceled for going over ulimits
416435
echo "[BUILD] Building (develop) FBGEMM-GPU (VARIANT=${fbgemm_variant}) ..."
417-
print_exec conda run --no-capture-output -n "${env_name}" \
436+
# shellcheck disable=SC2086
437+
print_exec conda run --no-capture-output ${env_prefix} \
418438
python setup.py build develop "${build_args[@]}"
419439

420440
# Run checks on the built libraries

‎.github/scripts/fbgemm_gpu_docs.bash

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,17 @@ install_docs_tools () {
2929
echo ""
3030
fi
3131

32+
test_network_connection || return 1
33+
34+
# shellcheck disable=SC2155
35+
local env_prefix=$(env_name_or_prefix "${env_name}")
36+
3237
echo "[INSTALL] Installing docs tools ..."
33-
(exec_with_retries conda install -n "${env_name}" -c conda-forge -y \
38+
# shellcheck disable=SC2086
39+
(exec_with_retries conda install ${env_prefix} -c conda-forge -y \
3440
doxygen) || return 1
3541

36-
# Check binaries are visible in the PAATH
42+
# Check binaries are visible in the PATH
3743
(test_binpath "${env_name}" doxygen) || return 1
3844

3945
echo "[INSTALL] Successfully installed all the docs tools"
@@ -60,14 +66,20 @@ build_fbgemm_gpu_docs () {
6066
echo ""
6167
fi
6268

69+
# shellcheck disable=SC2155
70+
local env_prefix=$(env_name_or_prefix "${env_name}")
71+
6372
echo "[BUILD] Installing docs-build dependencies ..."
64-
(exec_with_retries conda run -n "${env_name}" python -m pip install -r requirements.txt) || return 1
73+
# shellcheck disable=SC2086
74+
(exec_with_retries conda run ${env_prefix} python -m pip install -r requirements.txt) || return 1
6575

6676
echo "[BUILD] Running Doxygen build ..."
67-
(exec_with_retries conda run -n "${env_name}" doxygen Doxyfile.in) || return 1
77+
# shellcheck disable=SC2086
78+
(exec_with_retries conda run ${env_prefix} doxygen Doxyfile.in) || return 1
6879

6980
echo "[BUILD] Building HTML pages ..."
70-
(exec_with_retries conda run -n "${env_name}" make html) || return 1
81+
# shellcheck disable=SC2086
82+
(exec_with_retries conda run ${env_prefix} make html) || return 1
7183

7284
echo "[INSTALL] FBGEMM-GPU documentation build completed"
7385
}

‎.github/scripts/fbgemm_gpu_install.bash

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,8 @@ __fbgemm_gpu_post_install_checks () {
2020
(test_python_import_symbol "${env_name}" fbgemm_gpu __version__) || return 1
2121

2222
echo "[CHECK] Printing out the FBGEMM-GPU version ..."
23-
installed_fbgemm_gpu_version=$(conda run -n "${env_name}" python -c "import fbgemm_gpu; print(fbgemm_gpu.__version__)")
23+
# shellcheck disable=SC2086
24+
installed_fbgemm_gpu_version=$(conda run ${env_prefix} python -c "import fbgemm_gpu; print(fbgemm_gpu.__version__)")
2425
echo "[CHECK] The installed version is: ${installed_fbgemm_gpu_version}"
2526
}
2627

@@ -46,8 +47,12 @@ install_fbgemm_gpu_wheel () {
4647
print_exec sha256sum "${wheel_path}"
4748
print_exec md5sum "${wheel_path}"
4849

50+
# shellcheck disable=SC2155
51+
local env_prefix=$(env_name_or_prefix "${env_name}")
52+
4953
echo "[INSTALL] Installing FBGEMM-GPU wheel: ${wheel_path} ..."
50-
(exec_with_retries conda run -n "${env_name}" python -m pip install "${wheel_path}") || return 1
54+
# shellcheck disable=SC2086
55+
(exec_with_retries conda run ${env_prefix} python -m pip install "${wheel_path}") || return 1
5156

5257
__fbgemm_gpu_post_install_checks || return 1
5358

@@ -131,9 +136,12 @@ install_fbgemm_gpu_pip () {
131136
fi
132137
fi
133138

139+
# shellcheck disable=SC2155
140+
local env_prefix=$(env_name_or_prefix "${env_name}")
141+
134142
echo "[INSTALL] Attempting to install FBGEMM-GPU ${fbgemm_gpu_version}+${fbgemm_gpu_variant} through PIP ..."
135143
# shellcheck disable=SC2086
136-
(exec_with_retries conda run -n "${env_name}" pip install ${fbgemm_gpu_package}) || return 1
144+
(exec_with_retries conda run ${env_prefix} pip install ${fbgemm_gpu_package}) || return 1
137145

138146
__fbgemm_gpu_post_install_checks || return 1
139147

‎.github/scripts/fbgemm_gpu_lint.bash

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,14 @@ install_lint_tools () {
2929
echo ""
3030
fi
3131

32+
test_network_connection || return 1
33+
34+
# shellcheck disable=SC2155
35+
local env_prefix=$(env_name_or_prefix "${env_name}")
36+
3237
echo "[INSTALL] Installing lint tools ..."
33-
(exec_with_retries conda install -n "${env_name}" -c conda-forge -y \
38+
# shellcheck disable=SC2086
39+
(exec_with_retries conda install ${env_prefix} -c conda-forge -y \
3440
click \
3541
flake8 \
3642
ufmt) || return 1
@@ -70,10 +76,14 @@ lint_fbgemm_gpu_flake8 () {
7076

7177
echo "::add-matcher::fbgemm_gpu/test/lint/flake8_problem_matcher.json"
7278

79+
# shellcheck disable=SC2155
80+
local env_prefix=$(env_name_or_prefix "${env_name}")
81+
7382
# E501 = line too long
7483
# W503 = line break before binary operator (deprecated)
7584
# E203 = whitespace before ":"
76-
(print_exec conda run -n "${env_name}" flake8 --ignore=E501,W503,E203 .) || return 1
85+
# shellcheck disable=SC2086
86+
(print_exec conda run ${env_prefix} flake8 --ignore=E501,W503,E203 .) || return 1
7787

7888
echo "[TEST] Finished running flake8 lint checks"
7989
}
@@ -100,8 +110,12 @@ lint_fbgemm_gpu_ufmt () {
100110
fbgemm_gpu/bench
101111
)
102112

113+
# shellcheck disable=SC2155
114+
local env_prefix=$(env_name_or_prefix "${env_name}")
115+
103116
for p in "${lint_paths[@]}"; do
104-
(print_exec conda run -n "${env_name}" ufmt diff "${p}") || return 1
117+
# shellcheck disable=SC2086
118+
(print_exec conda run ${env_prefix} ufmt diff "${p}") || return 1
105119
done
106120

107121
echo "[TEST] Finished running ufmt lint checks"
@@ -129,8 +143,12 @@ lint_fbgemm_gpu_copyright () {
129143
fbgemm_gpu/bench
130144
)
131145

146+
# shellcheck disable=SC2155
147+
local env_prefix=$(env_name_or_prefix "${env_name}")
148+
132149
for p in "${lint_paths[@]}"; do
133-
(print_exec conda run -n "${env_name}" python fbgemm_gpu/test/lint/check_meta_header.py --path="${p}" --fixit=False) || return 1
150+
# shellcheck disable=SC2086
151+
(print_exec conda run ${env_prefix} python fbgemm_gpu/test/lint/check_meta_header.py --path="${p}" --fixit=False) || return 1
134152
done
135153

136154
echo "[TEST] Finished running Meta Copyright Header checks"

‎.github/scripts/fbgemm_gpu_test.bash

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,11 @@ run_python_test () {
2828
echo "################################################################################"
2929
fi
3030

31-
if print_exec conda run --no-capture-output -n "${env_name}" python -m pytest -v -rsx -s -W ignore::pytest.PytestCollectionWarning "${python_test_file}"; then
31+
# shellcheck disable=SC2155
32+
local env_prefix=$(env_name_or_prefix "${env_name}")
33+
34+
# shellcheck disable=SC2086
35+
if print_exec conda run --no-capture-output ${env_prefix} python -m pytest -v -rsx -s -W ignore::pytest.PytestCollectionWarning "${python_test_file}"; then
3236
echo "[TEST] Python test suite PASSED: ${python_test_file}"
3337
echo ""
3438
else
@@ -62,10 +66,14 @@ run_fbgemm_gpu_tests () {
6266
echo ""
6367
fi
6468

69+
# shellcheck disable=SC2155
70+
local env_prefix=$(env_name_or_prefix "${env_name}")
71+
6572
# Enable ROCM testing if specified
6673
if [ "$fbgemm_variant" == "rocm" ]; then
6774
echo "[TEST] Set environment variable FBGEMM_TEST_WITH_ROCM to enable ROCm tests ..."
68-
print_exec conda env config vars set -n "${env_name}" FBGEMM_TEST_WITH_ROCM=1
75+
# shellcheck disable=SC2086
76+
print_exec conda env config vars set ${env_prefix} FBGEMM_TEST_WITH_ROCM=1
6977
fi
7078

7179
# These are either non-tests or currently-broken tests in both FBGEMM_GPU and FBGEMM_GPU-CPU
@@ -90,7 +98,8 @@ run_fbgemm_gpu_tests () {
9098
fi
9199

92100
echo "[TEST] Installing pytest ..."
93-
print_exec conda install -n "${env_name}" -y pytest
101+
# shellcheck disable=SC2086
102+
print_exec conda install ${env_prefix} -y pytest
94103

95104
echo "[TEST] Checking imports ..."
96105
(test_python_import_package "${env_name}" fbgemm_gpu) || return 1

‎.github/scripts/nova_postscript.bash

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
echo "Current working directory: $(pwd)"
99
cd "${FBGEMM_REPO}" || echo "Failed to cd to ${FBGEMM_REPO}"
1010
PRELUDE="${FBGEMM_REPO}/.github/scripts/setup_env.bash"
11-
BUILD_ENV_NAME=base
11+
BUILD_ENV_NAME=${CONDA_ENV}
1212
GITHUB_ENV=TRUE
1313
export GITHUB_ENV
1414

‎.github/scripts/nova_prescript.bash

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
echo "Current working directory: $(pwd)"
99
cd "${FBGEMM_REPO}" || echo "Failed to cd to ${FBGEMM_REPO}"
1010
PRELUDE="${FBGEMM_REPO}/.github/scripts/setup_env.bash"
11-
BUILD_ENV_NAME=base
11+
BUILD_ENV_NAME=${CONDA_ENV}
1212
echo "--------------------------"
1313
echo "----- conda env list -----"
1414
conda env list

0 commit comments

Comments
 (0)