Skip to content

Commit 2cbcb6e

Browse files
q10spcyppt
q10
authored and
spcyppt
committed
Auto-generate the version file (#2011)
Summary: - Auto-generate the version file in OSS, so that the `__version__` symbol is available in the package Pull Request resolved: #2011 Reviewed By: shintaro-iwasaki Differential Revision: D49174509 Pulled By: q10 fbshipit-source-id: d0dfd1ab0a2912016ad6e003bf88feeab696fa4a
1 parent 7060f15 commit 2cbcb6e

File tree

10 files changed

+208
-16
lines changed

10 files changed

+208
-16
lines changed

‎.github/scripts/fbgemm_gpu_build.bash

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,8 @@ prepare_fbgemm_gpu_build () {
4141
echo "[BUILD] Installing other build dependencies ..."
4242
(exec_with_retries conda run -n "${env_name}" python -m pip install -r requirements.txt) || return 1
4343

44-
(test_python_import "${env_name}" numpy) || return 1
45-
(test_python_import "${env_name}" skbuild) || return 1
44+
(test_python_import_package "${env_name}" numpy) || return 1
45+
(test_python_import_package "${env_name}" skbuild) || return 1
4646

4747
echo "[BUILD] Successfully ran git submodules update"
4848
}
@@ -367,7 +367,7 @@ build_fbgemm_gpu_install () {
367367
# Exit this directory to prevent import clashing, since there is an
368368
# fbgemm_gpu/ subdirectory present
369369
cd - || return 1
370-
(test_python_import "${env_name}" fbgemm_gpu) || return 1
370+
(test_python_import_package "${env_name}" fbgemm_gpu) || return 1
371371

372372
echo "[BUILD] FBGEMM-GPU build + install completed"
373373
}

‎.github/scripts/fbgemm_gpu_install.bash

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
9+
# shellcheck disable=SC1091,SC2128
10+
. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash"
11+
12+
################################################################################
13+
# FBGEMM_GPU Install Functions
14+
################################################################################
15+
16+
__fbgemm_gpu_post_install_checks () {
17+
echo "[INSTALL] Checking imports and symbols ..."
18+
(test_python_import_package "${env_name}" fbgemm_gpu) || return 1
19+
(test_python_import_package "${env_name}" fbgemm_gpu.split_embedding_codegen_lookup_invokers) || return 1
20+
(test_python_import_symbol "${env_name}" fbgemm_gpu __version__) || return 1
21+
22+
echo "[CHECK] Printing out the FBGEMM-GPU version ..."
23+
installed_fbgemm_gpu_version=$(conda run -n "${env_name}" python -c "import fbgemm_gpu; print(fbgemm_gpu.__version__)")
24+
echo "[CHECK] The installed version is: ${installed_fbgemm_gpu_version}"
25+
}
26+
27+
install_fbgemm_gpu_wheel () {
28+
local env_name="$1"
29+
local wheel_path="$2"
30+
if [ "$wheel_path" == "" ]; then
31+
echo "Usage: ${FUNCNAME[0]} ENV_NAME WHEEL_NAME"
32+
echo "Example(s):"
33+
echo " ${FUNCNAME[0]} build_env fbgemm_gpu.whl # Install the package (wheel)"
34+
return 1
35+
else
36+
echo "################################################################################"
37+
echo "# Install FBGEMM-GPU from Wheel"
38+
echo "#"
39+
echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
40+
echo "################################################################################"
41+
echo ""
42+
fi
43+
44+
echo "[INSTALL] Printing out FBGEMM-GPU wheel SHA: ${wheel_path}"
45+
print_exec sha1sum "${wheel_path}"
46+
print_exec sha256sum "${wheel_path}"
47+
print_exec md5sum "${wheel_path}"
48+
49+
echo "[INSTALL] Installing FBGEMM-GPU wheel: ${wheel_path} ..."
50+
(exec_with_retries conda run -n "${env_name}" python -m pip install "${wheel_path}") || return 1
51+
52+
__fbgemm_gpu_post_install_checks || return 1
53+
54+
echo "[INSTALL] FBGEMM-GPU installation through wheel completed ..."
55+
}
56+
57+
58+
59+
install_fbgemm_gpu_pip () {
60+
local env_name="$1"
61+
local fbgemm_gpu_version="$2"
62+
local fbgemm_gpu_variant_type="$3"
63+
local fbgemm_gpu_variant_version="$4"
64+
if [ "$fbgemm_gpu_variant_type" == "" ]; then
65+
echo "Usage: ${FUNCNAME[0]} ENV_NAME FBGEMM_GPU_VERSION FBGEMM_GPU_VARIANT_TYPE [FBGEMM_GPU_VARIANT_VERSION]"
66+
echo "Example(s):"
67+
echo " ${FUNCNAME[0]} build_env 0.5.0rc2 cuda 12.1.1 # Install a specific version of the package (PyPI)"
68+
return 1
69+
else
70+
echo "################################################################################"
71+
echo "# Install FBGEMM-GPU Package from PIP"
72+
echo "#"
73+
echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
74+
echo "################################################################################"
75+
echo ""
76+
fi
77+
78+
# Set the package variant
79+
if [ "$fbgemm_gpu_variant_type" == "cuda" ]; then
80+
# Extract the CUDA version or default to 11.8.0
81+
local cuda_version="${fbgemm_gpu_variant_version:-11.8.0}"
82+
# shellcheck disable=SC2206
83+
local cuda_version_arr=(${cuda_version//./ })
84+
# Convert, i.e. cuda 11.7.1 => cu117
85+
local fbgemm_gpu_variant="cu${cuda_version_arr[0]}${cuda_version_arr[1]}"
86+
elif [ "$fbgemm_gpu_variant_type" == "rocm" ]; then
87+
# Extract the ROCM version or default to 5.5.1
88+
local rocm_version="${fbgemm_gpu_variant_version:-5.5.1}"
89+
# shellcheck disable=SC2206
90+
local rocm_version_arr=(${rocm_version//./ })
91+
# Convert, i.e. rocm 5.5.1 => rocm5.5
92+
local fbgemm_gpu_variant="rocm${rocm_version_arr[0]}.${rocm_version_arr[1]}"
93+
else
94+
local fbgemm_gpu_variant_type="cpu"
95+
local fbgemm_gpu_variant="cpu"
96+
fi
97+
echo "[INSTALL] Extracted FBGEMM-GPU variant: ${fbgemm_gpu_variant}"
98+
99+
# Set the package name and installation channel
100+
# if [ "$fbgemm_gpu_version" == "nightly" ] || [ "$fbgemm_gpu_version" == "test" ]; then
101+
# local fbgemm_gpu_package="--pre fbgemm-gpu"
102+
# local fbgemm_gpu_channel="https://download.pytorch.org/whl/${fbgemm_gpu_version}/${fbgemm_gpu_variant}/"
103+
# elif [ "$fbgemm_gpu_version" == "latest" ]; then
104+
# local fbgemm_gpu_package="fbgemm-gpu"
105+
# local fbgemm_gpu_channel="https://download.pytorch.org/whl/${fbgemm_gpu_variant}/"
106+
# else
107+
# local fbgemm_gpu_package="fbgemm-gpu==${fbgemm_gpu_version}+${fbgemm_gpu_variant}"
108+
# local fbgemm_gpu_channel="https://download.pytorch.org/whl/${fbgemm_gpu_variant}/"
109+
# fi
110+
111+
if [ "$fbgemm_gpu_variant_type" == "cuda" ]; then
112+
if [ "$fbgemm_gpu_version" == "nightly" ]; then
113+
local fbgemm_gpu_package="fbgemm-gpu-nightly"
114+
elif [ "$fbgemm_gpu_version" == "latest" ]; then
115+
local fbgemm_gpu_package="fbgemm-gpu"
116+
else
117+
local fbgemm_gpu_package="fbgemm-gpu==${fbgemm_gpu_version}"
118+
fi
119+
120+
elif [ "$fbgemm_gpu_variant_type" == "rocm" ]; then
121+
echo "ROCm is currently not supported in PyPI!"
122+
return 1
123+
124+
else
125+
if [ "$fbgemm_gpu_version" == "nightly" ]; then
126+
local fbgemm_gpu_package="fbgemm-gpu-nightly-cpu"
127+
elif [ "$fbgemm_gpu_version" == "latest" ]; then
128+
local fbgemm_gpu_package="fbgemm-gpu-cpu"
129+
else
130+
local fbgemm_gpu_package="fbgemm-gpu-cpu==${fbgemm_gpu_version}"
131+
fi
132+
fi
133+
134+
echo "[INSTALL] Attempting to install FBGEMM-GPU ${fbgemm_gpu_version}+${fbgemm_gpu_variant} through PIP ..."
135+
# shellcheck disable=SC2086
136+
(exec_with_retries conda run -n "${env_name}" pip install ${fbgemm_gpu_package}) || return 1
137+
138+
__fbgemm_gpu_post_install_checks || return 1
139+
140+
echo "[INSTALL] FBGEMM-GPU installation through PIP completed ..."
141+
}

‎.github/scripts/fbgemm_gpu_lint.bash

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ install_lint_tools () {
4242
# Check Python packages are importable
4343
local import_tests=( click )
4444
for p in "${import_tests[@]}"; do
45-
(test_python_import "${env_name}" "${p}") || return 1
45+
(test_python_import_package "${env_name}" "${p}") || return 1
4646
done
4747

4848
echo "[INSTALL] Successfully installed all the lint tools"

‎.github/scripts/fbgemm_gpu_test.bash

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,8 +93,8 @@ run_fbgemm_gpu_tests () {
9393
print_exec conda install -n "${env_name}" -y pytest
9494

9595
echo "[TEST] Checking imports ..."
96-
(test_python_import "${env_name}" fbgemm_gpu) || return 1
97-
(test_python_import "${env_name}" fbgemm_gpu.split_embedding_codegen_lookup_invokers) || return 1
96+
(test_python_import_package "${env_name}" fbgemm_gpu) || return 1
97+
(test_python_import_package "${env_name}" fbgemm_gpu.split_embedding_codegen_lookup_invokers) || return 1
9898

9999
echo "[TEST] Enumerating test files ..."
100100
print_exec ls -lth ./*.py

‎.github/scripts/setup_env.bash

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,7 @@ install_build_tools () {
177177
# Check Python packages are importable
178178
local import_tests=( click hypothesis jinja2 numpy skbuild wheel )
179179
for p in "${import_tests[@]}"; do
180-
(test_python_import "${env_name}" "${p}") || return 1
180+
(test_python_import_package "${env_name}" "${p}") || return 1
181181
done
182182

183183
echo "[INSTALL] Successfully installed all the build tools"
@@ -210,8 +210,8 @@ publish_to_pypi () {
210210

211211
echo "[INSTALL] Installing twine ..."
212212
print_exec conda install -n "${env_name}" -y twine
213-
(test_python_import "${env_name}" twine) || return 1
214-
(test_python_import "${env_name}" OpenSSL) || return 1
213+
(test_python_import_package "${env_name}" twine) || return 1
214+
(test_python_import_package "${env_name}" OpenSSL) || return 1
215215

216216
echo "[PUBLISH] Uploading package(s) to PyPI: ${package_name} ..."
217217
conda run -n "${env_name}" \

‎.github/scripts/utils_base.bash

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,26 @@ exec_with_retries () {
7070
# Assert Functions
7171
################################################################################
7272

73-
test_python_import () {
73+
test_python_import_symbol () {
74+
local env_name="$1"
75+
local package_name="$2"
76+
local target_symbol="$3"
77+
if [ "$target_symbol" == "" ]; then
78+
echo "Usage: ${FUNCNAME[0]} ENV_NAME PACKAGE_NAME SYMBOL"
79+
echo "Example(s):"
80+
echo " ${FUNCNAME[0]} build_env numpy __version__"
81+
return 1
82+
fi
83+
84+
if conda run -n "${env_name}" python -c "from ${package_name} import ${target_symbol}"; then
85+
echo "[CHECK] Found symbol '${target_symbol}' in Python package '${package_name}'."
86+
else
87+
echo "[CHECK] Could not find symbol '${target_symbol}' in Python package '${package_name}'; the package might be missing or broken."
88+
return 1
89+
fi
90+
}
91+
92+
test_python_import_package () {
7493
local env_name="$1"
7594
local python_import="$2"
7695
if [ "$python_import" == "" ]; then
@@ -81,9 +100,9 @@ test_python_import () {
81100
fi
82101

83102
if conda run -n "${env_name}" python -c "import ${python_import}"; then
84-
echo "[CHECK] Python package ${python_import} found."
103+
echo "[CHECK] Python package '${python_import}' found."
85104
else
86-
echo "[CHECK] Python package ${python_import} was not found or is broken!"
105+
echo "[CHECK] Python package '${python_import}' was not found, or the package is broken!"
87106
return 1
88107
fi
89108
}

‎.github/scripts/utils_conda.bash

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ create_conda_environment () {
116116

117117
# This test fails with load errors if the pyOpenSSL and cryptography package versions don't align
118118
echo "[SETUP] Testing pyOpenSSL import ..."
119-
(test_python_import "${env_name}" OpenSSL) || return 1
119+
(test_python_import_package "${env_name}" OpenSSL) || return 1
120120

121121
echo "[SETUP] Installed Python version: $(conda run -n "${env_name}" python --version)"
122122
echo "[SETUP] Successfully created Conda environment: ${env_name}"

‎.github/scripts/utils_pytorch.bash

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ install_pytorch_conda () {
6565
(exec_with_retries conda install --force-reinstall -n "${env_name}" -y ${pytorch_package} -c "${pytorch_channel}") || return 1
6666

6767
# Check that PyTorch is importable
68-
(test_python_import "${env_name}" torch.distributed) || return 1
68+
(test_python_import_package "${env_name}" torch.distributed) || return 1
6969

7070
# Print out the actual installed PyTorch version
7171
installed_pytorch_version=$(conda run -n "${env_name}" python -c "import torch; print(torch.__version__)")
@@ -154,7 +154,7 @@ install_pytorch_pip () {
154154
(exec_with_retries conda run -n "${env_name}" pip install ${pytorch_package} --extra-index-url ${pytorch_channel}) || return 1
155155

156156
# Check that PyTorch is importable
157-
(test_python_import "${env_name}" torch.distributed) || return 1
157+
(test_python_import_package "${env_name}" torch.distributed) || return 1
158158

159159
# Print out the actual installed PyTorch version
160160
installed_pytorch_version=$(conda run -n "${env_name}" python -c "import torch; print(torch.__version__)")

‎fbgemm_gpu/fbgemm_gpu/__init__.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,8 @@
1818
# Use existence to check if fbgemm_gpu_py.so has already been loaded
1919
open_source: bool = True
2020

21+
# Re-export docs
2122
from . import _fbgemm_gpu_docs # noqa: F401, E402
23+
24+
# Re-export the version string from the auto-generated version file
25+
from ._fbgemm_gpu_version import __version__ # noqa: F401, E402

‎fbgemm_gpu/setup.py

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,15 @@
44
# This source code is licensed under the BSD-style license found in the
55
# LICENSE file in the root directory of this source tree.
66

7+
# @licenselint-loose-mode
78

89
import argparse
910
import os
1011
import random
1112
import re
1213
import subprocess
1314
import sys
15+
import textwrap
1416

1517
from datetime import date
1618
from typing import List, Optional
@@ -182,6 +184,26 @@ def _get_cxx11_abi():
182184
class FbgemmGpuInstaller(PipInstall):
183185
"""FBGEMM_GPU PIP Installer"""
184186

187+
@classmethod
188+
def generate_version_file(cls, package_version: str) -> None:
189+
with open("fbgemm_gpu/_fbgemm_gpu_version.py", "w") as file:
190+
print(
191+
f"[SETUP.PY] Generating version file at: {os.path.realpath(file.name)}"
192+
)
193+
text = textwrap.dedent(
194+
f"""
195+
#!/usr/bin/env python3
196+
# Copyright (c) Meta Platforms, Inc. and affiliates.
197+
# All rights reserved.
198+
#
199+
# This source code is licensed under the BSD-style license found in the
200+
# LICENSE file in the root directory of this source tree.
201+
202+
__version__: str = "{package_version}"
203+
"""
204+
)
205+
file.write(text)
206+
185207
@classmethod
186208
def description(cls) -> str:
187209
# Get the long description from the relevant file
@@ -253,9 +275,15 @@ def main(argv: List[str]) -> None:
253275
# Repair command line args for setup.
254276
sys.argv = [sys.argv[0]] + unknown
255277

278+
# Determine the package version
279+
package_version = generate_package_version(args.package_name)
280+
281+
# Generate the version file
282+
FbgemmGpuInstaller.generate_version_file(package_version)
283+
256284
setup(
257285
name=args.package_name,
258-
version=generate_package_version(args.package_name),
286+
version=package_version,
259287
author="FBGEMM Team",
260288
author_email="packages@pytorch.org",
261289
long_description=FbgemmGpuInstaller.description(),

0 commit comments

Comments
 (0)