Skip to content

Commit 1d5ff21

Browse files
charrischarris
authored
Merge pull request numpy#25117 from charris/backport-25055
BLD: improve detection of Netlib libblas/libcblas/liblapack
2 parents 6d974a5 + 2a057fd commit 1d5ff21

File tree

2 files changed

+315
-9
lines changed

2 files changed

+315
-9
lines changed

‎.github/workflows/linux_blas.yml

Lines changed: 309 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,32 @@ name: BLAS tests (Linux)
1212
# `numpy.distutils`-based build. It can be removed once we remove
1313
# support for those builds.
1414
# - openblas32_stable_nightly:
15-
# Uses the 32-bit OpenBLAS builds, both the latest stable release and a
16-
# nightly build.
17-
#
18-
# TODO: coverage here is limited, we should add non-OpenBLAS libraries and
19-
# exercise the BLAS-related build options (see `meson_options.txt`).
15+
# Uses the 32-bit OpenBLAS builds, both the latest stable release
16+
# and a nightly build.
17+
# - openblas_no_pkgconfig_fedora:
18+
# Test OpenBLAS on Fedora. Fedora doesn't ship .pc files for OpenBLAS,
19+
# hence this exercises the "system dependency" detection method.
20+
# - flexiblas_fedora:
21+
# Tests FlexiBLAS (the default on Fedora for its own packages), via
22+
# pkg-config. FlexiBLAS allows runtime switching of BLAS/LAPACK
23+
# libraries, which is a useful capability (not tested in this job).
24+
# - openblas_cmake:
25+
# Tests whether OpenBLAS LP64 is detected correctly when only CMake
26+
# and not pkg-config is installed.
27+
# - netlib-debian:
28+
# Installs libblas/liblapack, which in Debian contains libcblas within
29+
# libblas.
30+
# - netlib-split:
31+
# Installs vanilla Netlib blas/lapack with separate libcblas, which is
32+
# the last option tried in auto-detection.
33+
# - mkl:
34+
# Tests MKL installed from PyPI (because easiest/fastest, if broken) in
35+
# 3 ways: both LP64 and ILP64 via pkg-config, and then using the
36+
# Single Dynamic Library (SDL, or `libmkl_rt`).
37+
# - blis:
38+
# Simple test for LP64 via pkg-config
39+
# - atlas:
40+
# Simple test for LP64 via pkg-config
2041

2142
on:
2243
pull_request:
@@ -46,11 +67,11 @@ jobs:
4667
USE_NIGHTLY_OPENBLAS: ${{ matrix.USE_NIGHTLY_OPENBLAS }}
4768
name: "Test Linux (${{ matrix.USE_NIGHTLY_OPENBLAS && 'nightly' || 'stable' }} OpenBLAS)"
4869
steps:
49-
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
70+
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
5071
with:
5172
submodules: recursive
5273
fetch-depth: 0
53-
- uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
74+
- uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1
5475
with:
5576
python-version: '3.11'
5677

@@ -91,3 +112,284 @@ jobs:
91112
run: |
92113
pip install pytest pytest-xdist hypothesis typing_extensions
93114
spin test -j auto
115+
116+
117+
openblas_no_pkgconfig_fedora:
118+
if: "github.repository == 'numpy/numpy'"
119+
runs-on: ubuntu-latest
120+
container: fedora:39
121+
name: "OpenBLAS (Fedora, no pkg-config, LP64/ILP64)"
122+
steps:
123+
- name: Install system dependencies
124+
run: |
125+
dnf install git gcc-gfortran g++ python3-devel openblas-devel -y
126+
127+
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
128+
with:
129+
submodules: recursive
130+
fetch-depth: 0
131+
132+
- name: Install dependencies
133+
run: |
134+
pip install -r build_requirements.txt
135+
pip install pytest hypothesis typing_extensions
136+
137+
- name: Build (LP64)
138+
run: spin build -- -Dblas=openblas -Dlapack=openblas -Ddisable-optimization=true
139+
140+
- name: Test
141+
run: spin test -- numpy/linalg
142+
143+
- name: Build (ILP64)
144+
run: |
145+
rm -rf build
146+
spin build -- -Duse-ilp64=true -Ddisable-optimization=true
147+
148+
- name: Test
149+
run: spin test -- numpy/linalg
150+
151+
152+
flexiblas_fedora:
153+
if: "github.repository == 'numpy/numpy'"
154+
runs-on: ubuntu-latest
155+
container: fedora:39
156+
name: "FlexiBLAS (LP64, ILP64 on Fedora)"
157+
steps:
158+
- name: Install system dependencies
159+
run: |
160+
dnf install git gcc-gfortran g++ python3-devel flexiblas-devel -y
161+
162+
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
163+
with:
164+
submodules: recursive
165+
fetch-depth: 0
166+
167+
- name: Install dependencies
168+
run: |
169+
pip install -r build_requirements.txt
170+
pip install pytest hypothesis typing_extensions
171+
172+
- name: Build
173+
run: spin build -- -Ddisable-optimization=true
174+
175+
- name: Test
176+
run: spin test -- numpy/linalg
177+
178+
- name: Build (ILP64)
179+
run: |
180+
rm -rf build
181+
spin build -- -Ddisable-optimization=true -Duse-ilp64=true
182+
183+
- name: Test (ILP64)
184+
run: spin test -- numpy/linalg
185+
186+
187+
openblas_cmake:
188+
if: "github.repository == 'numpy/numpy'"
189+
runs-on: ubuntu-latest
190+
name: "OpenBLAS with CMake"
191+
steps:
192+
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
193+
with:
194+
submodules: recursive
195+
fetch-depth: 0
196+
- uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1
197+
with:
198+
python-version: '3.11'
199+
200+
- name: Install dependencies
201+
run: |
202+
pip install -r build_requirements.txt
203+
pip install pytest pytest-xdist hypothesis typing_extensions
204+
sudo apt-get install libopenblas-dev cmake
205+
sudo apt-get remove pkg-config
206+
207+
- name: Build
208+
run: spin build -- -Ddisable-optimization=true
209+
210+
- name: Test
211+
run: spin test -j auto -- numpy/linalg
212+
213+
214+
netlib-debian:
215+
if: "github.repository == 'numpy/numpy'"
216+
runs-on: ubuntu-latest
217+
name: "Debian libblas/liblapack"
218+
steps:
219+
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
220+
with:
221+
submodules: recursive
222+
fetch-depth: 0
223+
- uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1
224+
with:
225+
python-version: '3.11'
226+
227+
- name: Install dependencies
228+
run: |
229+
pip install -r build_requirements.txt
230+
sudo apt-get install liblapack-dev pkg-config
231+
232+
- name: Build
233+
run: |
234+
spin build -- -Ddisable-optimization=true
235+
236+
- name: Test
237+
run: |
238+
pip install pytest pytest-xdist hypothesis typing_extensions
239+
spin test -j auto -- numpy/linalg
240+
241+
242+
netlib-split:
243+
if: "github.repository == 'numpy/numpy'"
244+
runs-on: ubuntu-latest
245+
container: opensuse/tumbleweed
246+
name: "OpenSUSE Netlib BLAS/LAPACK"
247+
steps:
248+
- name: Install system dependencies
249+
run: |
250+
# No blas.pc on OpenSUSE as of Nov 2023, so no need to install pkg-config.
251+
# If it is needed in the future, use install name `pkgconf-pkg-config`
252+
zypper install -y git gcc-c++ python3-pip python3-devel blas cblas lapack
253+
254+
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
255+
with:
256+
submodules: recursive
257+
fetch-depth: 0
258+
259+
- name: Install PyPI dependencies
260+
run: |
261+
pip install --break-system-packages -r build_requirements.txt
262+
263+
- name: Build
264+
run: |
265+
spin build -- -Dblas=blas -Dlapack=lapack -Ddisable-optimization=true -Dallow-noblas=false
266+
267+
- name: Test
268+
run: |
269+
pip install --break-system-packages pytest pytest-xdist hypothesis typing_extensions
270+
spin test -j auto -- numpy/linalg
271+
272+
273+
mkl:
274+
if: "github.repository == 'numpy/numpy'"
275+
runs-on: ubuntu-latest
276+
name: "MKL (LP64, ILP64, SDL)"
277+
steps:
278+
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
279+
with:
280+
submodules: recursive
281+
fetch-depth: 0
282+
- uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1
283+
with:
284+
python-version: '3.11'
285+
286+
- name: Install dependencies
287+
run: |
288+
pip install -r build_requirements.txt
289+
pip install pytest pytest-xdist hypothesis typing_extensions
290+
pip install mkl mkl-devel
291+
292+
- name: Repair MKL pkg-config files and symlinks
293+
run: |
294+
# MKL 2023.2 works when installed from conda-forge (except for `-iomp`
295+
# and `-tbb` pkg-config files), Spack, or with the standalone Intel
296+
# installer. The standalone installer is the worst option, since it's
297+
# large and clumsy to install and requires running a setvars.sh script
298+
# before things work. The PyPI MKL packages are broken and need the
299+
# fixes in this step. For details, see
300+
# https://github.com/conda-forge/intel_repack-feedstock/issues/34
301+
cd $Python3_ROOT_DIR/lib/pkgconfig
302+
sed -i 's/\/intel64//g' mkl*.pc
303+
# add the expected .so -> .so.2 symlinks to fix linking
304+
cd ..
305+
for i in $( ls libmkl*.so.2 ); do ln -s $i ${i%.*}; done
306+
307+
- name: Build with defaults (LP64)
308+
run: |
309+
pkg-config --libs mkl-dynamic-lp64-seq # check link flags
310+
spin build -- -Ddisable-optimization=true
311+
312+
- name: Test
313+
run: spin test -- numpy/linalg
314+
315+
- name: Build with ILP64
316+
run: |
317+
git clean -xdf > /dev/null
318+
pkg-config --libs mkl-dynamic-ilp64-seq
319+
spin build -- -Duse-ilp64=true -Ddisable-optimization=true
320+
321+
- name: Test
322+
run: spin test -- numpy/linalg
323+
324+
- name: Build without pkg-config (default options, SDL)
325+
run: |
326+
git clean -xdf > /dev/null
327+
pushd $Python3_ROOT_DIR/lib/pkgconfig
328+
rm mkl*.pc
329+
popd
330+
export MKLROOT=$Python3_ROOT_DIR
331+
spin build -- -Ddisable-optimization=true
332+
333+
- name: Test
334+
run: spin test -- numpy/linalg
335+
336+
blis:
337+
if: "github.repository == 'numpy/numpy'"
338+
runs-on: ubuntu-latest
339+
name: "BLIS"
340+
steps:
341+
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
342+
with:
343+
submodules: recursive
344+
fetch-depth: 0
345+
- uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1
346+
with:
347+
python-version: '3.11'
348+
349+
- name: Install dependencies
350+
run: |
351+
pip install -r build_requirements.txt
352+
pip install pytest pytest-xdist hypothesis typing_extensions
353+
sudo apt-get install libblis-dev libopenblas-dev pkg-config
354+
355+
- name: Add BLIS pkg-config file
356+
run: |
357+
# Needed because blis.pc missing in Debian:
358+
# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=989076
359+
# The alternative here would be to use another distro or Miniforge
360+
sudo cp tools/ci/_blis_debian.pc /usr/lib/x86_64-linux-gnu/pkgconfig/blis.pc
361+
# Check if the patch works:
362+
pkg-config --libs blis
363+
pkg-config --cflags blis
364+
365+
- name: Build
366+
run: spin build -- -Dblas=blis -Ddisable-optimization=true
367+
368+
- name: Test
369+
run: spin test -- numpy/linalg
370+
371+
atlas:
372+
if: "github.repository == 'numpy/numpy'"
373+
runs-on: ubuntu-latest
374+
name: "ATLAS"
375+
steps:
376+
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
377+
with:
378+
submodules: recursive
379+
fetch-depth: 0
380+
- uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1
381+
with:
382+
python-version: '3.11'
383+
384+
- name: Install dependencies
385+
run: |
386+
pip install -r build_requirements.txt
387+
pip install pytest pytest-xdist hypothesis typing_extensions
388+
sudo apt-get install libatlas-base-dev pkg-config
389+
390+
- name: Build
391+
run: spin build -- -Dblas=blas-atlas -Dlapack=lapack-atlas -Ddisable-optimization=true
392+
393+
- name: Test
394+
run: spin test -- numpy/linalg
395+

‎tools/ci/cirrus_arm.yml

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ freebsd_test_task:
127127
memory: 4G
128128

129129
install_devtools_script: |
130-
pkg install -y git bash ninja ccache
130+
pkg install -y git bash ninja ccache blas cblas lapack pkgconf
131131
132132
<<: *MODIFIED_CLONE
133133

@@ -149,11 +149,15 @@ freebsd_test_task:
149149
build_script: |
150150
chsh -s /usr/local/bin/bash
151151
source .venv/bin/activate
152-
python -m pip install . --no-build-isolation -v -Csetup-args="-Dallow-noblas=true"
152+
python -m pip install . --no-build-isolation -v -Csetup-args="-Dallow-noblas=false"
153153
154154
test_script: |
155155
chsh -s /usr/local/bin/bash
156156
source .venv/bin/activate
157157
cd tools
158158
python -m pytest --pyargs numpy -m "not slow"
159159
ccache -s
160+
161+
on_failure:
162+
debug_script: |
163+
cat build/meson-logs/meson-log.txt

0 commit comments

Comments
 (0)