Skip to content

Commit 5b1f551

Browse files
q10q10
committed
Fix <crypt.h> issue (#2430)
Summary: - Fix issue with missing <crypt.h> for Python 3.8 and below - Clean up release testing scripts Pull Request resolved: #2430 Reviewed By: spcyppt Differential Revision: D54933067 Pulled By: q10 fbshipit-source-id: fee087d6306594b733703422ca8a941c0c942228
1 parent 7dd9fc6 commit 5b1f551

File tree

2 files changed

+67
-31
lines changed

2 files changed

+67
-31
lines changed

‎.github/scripts/fbgemm_gpu_test.bash

Lines changed: 15 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -240,31 +240,23 @@ test_setup_conda_environment () {
240240
fi
241241

242242
echo "Creating the Build Environment: ${env_name} ..."
243-
create_conda_environment "${env_name}" "${python_version}" || return 1
243+
create_conda_environment "${env_name}" "${python_version}" || return 1
244244

245-
# Set up the build tools and/or GPU runtimes
246-
if [ "$pytorch_variant_type" == "cuda" ]; then
247-
if [ "$compiler" != "" ]; then
248-
install_cxx_compiler "${env_name}" "${compiler}" || return 1
249-
fi
250-
install_build_tools "${env_name}" || return 1
251-
install_cuda "${env_name}" "${pytorch_variant_version}" || return 1
252-
install_cudnn "${env_name}" "${HOME}/cudnn-${pytorch_variant_version}" "${pytorch_variant_version}" || return 1
253-
254-
elif [ "$pytorch_variant_type" == "rocm" ]; then
255-
install_rocm_ubuntu "${env_name}" "${pytorch_variant_version}" || return 1
256-
if [ "$compiler" != "" ]; then
257-
install_cxx_compiler "${env_name}" "${compiler}" || return 1
258-
fi
259-
install_build_tools "${env_name}" || return 1
260-
return 1
245+
# Install ROCm tools and runtime
246+
if [ "$pytorch_variant_type" == "rocm" ]; then
247+
install_rocm_ubuntu "${env_name}" "${pytorch_variant_version}" || return 1
261248

262-
else
263-
if [ "$compiler" != "" ]; then
264-
install_cxx_compiler "${env_name}" "${compiler}" || return 1
265-
fi
266-
install_build_tools "${env_name}" || return 1
249+
# Install CUDA tools and runtime
250+
elif [ "$pytorch_variant_type" == "cuda" ]; then
251+
install_cuda "${env_name}" "${pytorch_variant_version}" || return 1
252+
install_cudnn "${env_name}" "${HOME}/cudnn-${pytorch_variant_version}" "${pytorch_variant_version}" || return 1
253+
fi
254+
255+
# Install C++ compiler and build tools (all FBGEMM_GPU variants)
256+
if [ "$compiler" == "gcc" ] || [ "$compiler" == "clang" ]; then
257+
install_cxx_compiler "${env_name}" "${compiler}" || return 1
267258
fi
259+
install_build_tools "${env_name}" || return 1
268260

269261
# Install PyTorch
270262
if [ "$pytorch_installer" == "conda" ]; then
@@ -332,7 +324,7 @@ test_fbgemm_gpu_setup_and_pip_install () {
332324

333325
local env_name="test_py${py_version}_pytorch_${pytorch_channel_version}_fbgemm_${fbgemm_gpu_channel_version}_${variant_type}/${variant_version}"
334326
local env_name="${env_name//\//_}"
335-
test_setup_conda_environment "${env_name}" '' "${py_version}" pip "${pytorch_channel_version}" "${variant_type}" "${variant_version}" || return 1
327+
test_setup_conda_environment "${env_name}" 'no-compiler' "${py_version}" pip "${pytorch_channel_version}" "${variant_type}" "${variant_version}" || return 1
336328
install_fbgemm_gpu_pip "${env_name}" "${fbgemm_gpu_channel_version}" "${variant_type}/${variant_version}" || return 1
337329
cd ~/FBGEMM/fbgemm_gpu/test || return 1
338330

‎.github/scripts/utils_conda.bash

Lines changed: 52 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,54 @@ setup_miniconda () {
7777
echo "[SETUP] Successfully set up Miniconda at ${miniconda_prefix}"
7878
}
7979

80+
__handle_pyopenssl_version_issue () {
81+
# NOTE: pyOpenSSL needs to be above a certain version for PyPI publishing to
82+
# work correctly.
83+
local env_name="$1"
84+
85+
# shellcheck disable=SC2155
86+
local env_prefix=$(env_name_or_prefix "${env_name}")
87+
88+
# The pyOpenSSL and cryptography packages versions need to line up for PyPI publishing to work
89+
# https://stackoverflow.com/questions/74981558/error-updating-python3-pip-attributeerror-module-lib-has-no-attribute-openss
90+
echo "[SETUP] Upgrading pyOpenSSL ..."
91+
# shellcheck disable=SC2086
92+
(exec_with_retries 3 conda run ${env_prefix} python -m pip install "pyOpenSSL>22.1.0") || return 1
93+
94+
# This test fails with load errors if the pyOpenSSL and cryptography package versions don't align
95+
echo "[SETUP] Testing pyOpenSSL import ..."
96+
(test_python_import_package "${env_name}" OpenSSL) || return 1
97+
98+
}
99+
100+
__handle_libcrypt_header_issue () {
101+
# NOTE: <crypt.h> appears to be missing, especially in Python 3.8 and under,
102+
# which results in runtime errors when `torch.compile()` is called.
103+
local env_name="$1"
104+
105+
# shellcheck disable=SC2155
106+
local env_prefix=$(env_name_or_prefix "${env_name}")
107+
108+
# https://git.sr.ht/~andir/nixpkgs/commit/4ace88d63b14ef62f24d26c984775edc2ab1737c
109+
echo "[SETUP] Installing libxcrypt ..."
110+
# shellcheck disable=SC2086
111+
(exec_with_retries 3 conda install ${env_prefix} -c conda-forge -y libxcrypt) || return 1
112+
113+
# shellcheck disable=SC2155,SC2086
114+
local conda_prefix=$(conda run ${env_prefix} printenv CONDA_PREFIX)
115+
# shellcheck disable=SC2207,SC2086
116+
local python_version=($(conda run --no-capture-output ${env_prefix} python --version))
117+
# shellcheck disable=SC2206
118+
local python_version_arr=(${python_version[1]//./ })
119+
120+
# Copy the header file from include/ to include/python3.X/
121+
# https://github.com/stanford-futuredata/ColBERT/issues/309
122+
echo "[SETUP] Copying <crypt.h> over ..."
123+
# shellcheck disable=SC2206
124+
local dst_file="${conda_prefix}/include/python${python_version_arr[0]}.${python_version_arr[1]}/crypt.h"
125+
print_exec cp "${conda_prefix}/include/crypt.h" "${dst_file}"
126+
}
127+
80128
create_conda_environment () {
81129
local env_name="$1"
82130
local python_version="$2"
@@ -119,15 +167,11 @@ create_conda_environment () {
119167
# shellcheck disable=SC2086
120168
(exec_with_retries 3 conda run ${env_prefix} pip install --upgrade pip) || return 1
121169

122-
# The pyOpenSSL and cryptography packages versions need to line up for PyPI publishing to work
123-
# https://stackoverflow.com/questions/74981558/error-updating-python3-pip-attributeerror-module-lib-has-no-attribute-openss
124-
echo "[SETUP] Upgrading pyOpenSSL ..."
125-
# shellcheck disable=SC2086
126-
(exec_with_retries 3 conda run ${env_prefix} python -m pip install "pyOpenSSL>22.1.0") || return 1
170+
# Handle pyOpenSSL version issue
171+
__handle_pyopenssl_version_issue "${env_name}"
127172

128-
# This test fails with load errors if the pyOpenSSL and cryptography package versions don't align
129-
echo "[SETUP] Testing pyOpenSSL import ..."
130-
(test_python_import_package "${env_name}" OpenSSL) || return 1
173+
# Handle missing <crypt.h> issue
174+
__handle_libcrypt_header_issue "${env_name}"
131175

132176
# shellcheck disable=SC2086
133177
echo "[SETUP] Installed Python version: $(conda run ${env_prefix} python --version)"

0 commit comments

Comments
 (0)