Merge pull request #7 from SCIInstitute/support-build-and-publish-as-vtk-python-wheels

ENH: Add infrastructure for building the python package using GitHub Actions

Author: jcfr

.github/scripts/cibw_before_build.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -ev
+
+if [[ $RUNNER_OS == "Linux" ]]; then
+    # Some of the VTK modules use this library, and auditwheel will
+    # complain if it can't find it. We will remove it from the wheel
+    # after the repair is complete.
+    yum install libXcursor-devel -y
+
+    # Make sure this is removed before every build, so we always
+    # get the correct one.
+    rm -rf $VTK_WHEEL_SDK_INSTALL_PATH
+fi

.github/scripts/install.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -ev
+
+#### Install cibuildwheel ####
+pip install cibuildwheel
+
+if [[ $RUNNER_OS == "macOS" ]]; then
+    # VTK is expecting the xcode path to be slightly different.
+    # Specifically, the VTK::RenderingOpenGL2 imported target seems
+    # to expect the absolute path to the OpenGL library to match.
+    ln -s /Applications/Xcode_13.1.app/ /Applications/Xcode-13.1.app
+fi

.github/scripts/linux_repair_wheel.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+
+import glob
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+
+from utils import change_dir, unzip_file
+
+"""
+This script does the following:
+
+1. Copy all vtk modules into the wheel
+2. Run auditwheel repair
+3. Remove anything that wasn't there before the repair
+
+This is done so that we can get proper tags on the wheel, without the
+RPATHs or contents of the wheel being affected.
+
+Note: if we make the VTK modules visible to auditwheel by modifying the
+LD_LIBRARY_PATH, auditwheel copies them into the project, but modifies
+the RPATHs of our libraries. We do not want auditwheel to modify the
+RPATHs, so we copy the VTK modules in ourselves.
+
+It would be great if auditwheel, at some point, allowed us to add tags
+to the wheel *without* repairing it...
+"""
+
+if len(sys.argv) < 3:
+    sys.exit('Usage: <script> <wheel_path> <output_dir>')
+
+wheel_path = sys.argv[1]
+output_dir = sys.argv[2]
+
+wheel_sdk_path = os.getenv('VTK_WHEEL_SDK_INSTALL_PATH')
+if wheel_sdk_path is None:
+    raise Exception('VTK_WHEEL_SDK_INSTALL_PATH must be set')
+
+vtkmodules_glob_pattern = Path(wheel_sdk_path) / 'build/*/vtkmodules/*.so'
+vtkmodules = glob.glob(str(vtkmodules_glob_pattern))
+if not vtkmodules:
+    raise Exception(f'Failed to find vtkmodules at: {vtkmodules_glob_pattern}')
+
+with unzip_file(wheel_path) as unpacked_path:
+    # Take a snapshot of the directory contents. After the wheel repair, we
+    # will remove all files except for these.
+    with change_dir(unpacked_path):
+        contents_snapshot = list(Path().rglob('*'))
+
+        # Find the path to the RECORD file
+        record_glob = list(Path().glob('*.dist-info/RECORD'))
+        if len(record_glob) != 1:
+            raise Exception('Failed to find RECORD file')
+
+        record_path = record_glob[0]
+
+        # Read in the RECORD file. We will write it back out after the repair.
+        with open(record_path, 'r') as rf:
+            record_text = rf.read()
+
+    # Copy the vtkmodules in
+    destination = Path(unpacked_path) / 'vtkmodules'
+    for f in vtkmodules:
+        output = destination / Path(f).name
+        if output.exists():
+            # Don't overwrite any files
+            continue
+
+        shutil.copyfile(f, output)
+
+# Now, perform the auditwheel repair
+cmd = ['auditwheel', 'repair', wheel_path, '-w', output_dir]
+subprocess.check_call(cmd)
+
+# There should now be a wheel in the wheelhouse
+output = glob.glob(f'{output_dir}/*.whl')
+if len(output) != 1:
+    raise Exception(f'Failed to find wheel in `{output_dir}`')
+
+output_wheel = output[0]
+
+# Now, remove all newly added files
+with unzip_file(output_wheel) as unpacked_path:
+    with change_dir(unpacked_path):
+        for item in Path().rglob('*'):
+            if item not in contents_snapshot:
+                if item.is_dir():
+                    shutil.rmtree(item)
+                else:
+                    item.unlink()
+
+        # Replace the RECORD file with the old one
+        with open(record_path, 'w') as wf:
+            wf.write(record_text)

.github/scripts/utils.py

@@ -0,0 +1,38 @@
+from contextlib import contextmanager
+import os
+from pathlib import Path
+import shutil
+import zipfile
+
+
+@contextmanager
+def unzip_file(zip_path, unpack_path=None):
+    # This will unzip the file while inside the context, and re-zip it
+    # afterwards, which allows the caller to modify the contents
+
+    if unpack_path is None:
+        unpack_path = str(Path('__tmp_unzip_file_unpack').resolve())
+
+    with zipfile.ZipFile(zip_path, 'r') as zip_ref:
+        zip_ref.extractall(unpack_path)
+
+    try:
+        yield unpack_path
+    finally:
+        name = shutil.make_archive(zip_path, 'zip', unpack_path)
+        shutil.rmtree(unpack_path)
+        if name.endswith('.zip'):
+            Path(name).rename(name[:-4])
+
+
+@contextmanager
+def change_dir(path):
+    # Temporarily change the directory to the path. When the context ends,
+    # the path will be changed back.
+
+    origin = Path.cwd()
+    try:
+        os.chdir(path)
+        yield
+    finally:
+        os.chdir(origin)

.github/workflows/build_wheels.yml

@@ -0,0 +1,113 @@
+name: Build Wheels
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+       - '*'
+  pull_request:
+    branches:
+      - main
+
+  workflow_dispatch:
+
+env:
+  # Only support 64-bit CPython >= 3.7
+  # VTK does not currently build python 3.8 arm64 wheels, so skip it too
+  CIBW_SKIP: "cp27-* cp35-* cp36-* cp37-* pp* *-manylinux_i686 *-musllinux_* *-win32 cp38-macosx_arm64"
+
+  # Need to match the version used by VTK
+  CIBW_ENVIRONMENT_MACOS: MACOSX_DEPLOYMENT_TARGET=10.10
+
+  # In the Linux docker container, install the wheel SDKs to this location
+  CIBW_ENVIRONMENT_LINUX: VTK_WHEEL_SDK_INSTALL_PATH=/vtk-wheel-sdk
+
+  # NOTE: cross-compilation is not currently working for us for arm64.
+  # We are going to turn it off and build them manually until GitHub Actions
+  # makes arm64 runners available.
+  # Build both x86_64 and arm64 (through cross-compilation) wheels on Mac
+  # CIBW_ARCHS_MACOS: x86_64 arm64
+
+  # VTK already fixes the rpaths, so we can skip this step for MacOS
+  CIBW_REPAIR_WHEEL_COMMAND_MACOS:
+
+  # On Linux, we only need auditwheel to add the tags to the wheel.
+  # Unfortunately, auditwheel currently requires us to repair the wheel to
+  # add the tags, even though we do not need to repair the wheel.
+  # Thus, we need to set everything up for a wheel repair (including placing
+  # the VTK libraries in `vtkmodules`, where they are expected to be at
+  # runtime), perform the wheel repair, and then remove the added libraries.
+  # Then the tags will have been added.
+  CIBW_REPAIR_WHEEL_COMMAND_LINUX: .github/scripts/linux_repair_wheel.py {wheel} {dest_dir}
+
+  # Pass these variables into the Linux docker containers
+  CIBW_ENVIRONMENT_PASS_LINUX: RUNNER_OS VTK_WHEEL_SDK_INSTALL_PATH
+
+  # Run this before every build
+  CIBW_BEFORE_BUILD: bash .github/scripts/cibw_before_build.sh
+
+  CIBW_TEST_COMMAND: >
+    pip install -r {package}/Testing/Python/requirements.txt &&
+    pytest -v {package}/Testing/Python
+
+  CIBW_TEST_COMMAND_WINDOWS: >
+    pip install -r {package}/Testing/Python/requirements.txt &&
+    pytest -v {package}/Testing/Python
+
+
+# Use bash by default for the run command
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+
+    steps:
+      - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+
+      - uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b # v4.6.0
+        name: Install Python
+        with:
+          python-version: '3.9'
+
+      - name: Install dependencies
+        run: bash .github/scripts/install.sh
+
+      - name: Build wheels
+        run: cibuildwheel --output-dir wheelhouse
+
+      - name: Upload skbuild if an error occurred (for debugging)
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        with:
+          name: skbuild
+          path: ${{ github.workspace }}/_skbuild
+
+      - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        with:
+          path: ./wheelhouse/*.whl
+
+  upload_pypi:
+    needs: build_wheels
+    name: Upload wheels to PyPI
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    # upload to PyPI on every tag push
+    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/')
+    steps:
+      - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+        with:
+          name: artifact
+          path: dist
+
+      - uses: pypa/gh-action-pypi-publish@0bf742be3ebe032c25dd15117957dc15d0cfc38d # v1.8.5

CMakeLists.txt

@@ -17,9 +17,9 @@ if(NOT DEFINED CLEAVER_JsonCpp_LIBRARY)
   set(CLEAVER_JsonCpp_LIBRARY ${CLEAVER_LIBRARY})
   cmake_path(REMOVE_FILENAME CLEAVER_JsonCpp_LIBRARY)
   if(WIN32)
-    cmake_path(APPEND CLEAVER_JsonCpp_LIBRARY "jsoncpp" "jsoncpp.lib")
+    cmake_path(APPEND CLEAVER_JsonCpp_LIBRARY "jsoncpp.lib")
   elseif(APPLE)
-    # TODO
+    cmake_path(APPEND CLEAVER_JsonCpp_LIBRARY "libjsoncpp.a")
   else()
     cmake_path(APPEND CLEAVER_JsonCpp_LIBRARY "libjsoncpp.a")
   endif()

FetchFromUrl.cmake

@@ -0,0 +1,14 @@
+include(FetchContent)
+
+# Before calling this, the following variables must be defined:
+# FETCH_FROM_URL_PROJECT_NAME - a unique name for the project
+# FETCH_FROM_URL_URL - the URL from which to download the project
+# FETCH_FROM_URL_INSTALL_LOCATION - the location to unpack the project
+
+FetchContent_Populate(${FETCH_FROM_URL_PROJECT_NAME}
+  URL         ${FETCH_FROM_URL_URL}
+  SOURCE_DIR  ${FETCH_FROM_URL_INSTALL_LOCATION}
+  QUIET
+  )
+
+message(STATUS "Remote - FETCH_FROM_URL ${FETCH_FROM_URL_PROJECT_NAME} [OK]")

FetchVTKExternalModule.cmake

@@ -0,0 +1,21 @@
+include(FetchContent)
+
+set(proj VTKExternalModule)
+if (FETCH_${proj}_INSTALL_LOCATION)
+  # The install location can be specified
+  set(EP_SOURCE_DIR "${FETCH_${proj}_INSTALL_LOCATION}")
+else()
+  set(EP_SOURCE_DIR ${CMAKE_BINARY_DIR}/${proj})
+endif()
+
+FetchContent_Populate(${proj}
+  SOURCE_DIR     ${EP_SOURCE_DIR}
+  GIT_REPOSITORY https://github.com/KitwareMedical/VTKExternalModule.git
+  GIT_TAG        c1906cf121e34b6391a91c2fffc448eca402a6cc
+  QUIET
+  )
+
+message(STATUS "Remote - ${proj} [OK]")
+
+set(VTKExternalModule_SOURCE_DIR ${EP_SOURCE_DIR})
+message(STATUS "Remote - VTKExternalModule_SOURCE_DIR:${VTKExternalModule_SOURCE_DIR}")