diff --git a/.circleci/config.yml b/.circleci/config.yml index 5b207874..af135db3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,7 +20,7 @@ jobs: - run: name: Build the Linux aarch64 wheels. command: | - python3 -m pip install --user cibuildwheel==2.17.0 + python3 -m pip install --user cibuildwheel==2.20.0 python3 -m cibuildwheel --output-dir wheelhouse - run: name: Test the wheels diff --git a/.github/workflows/docker-gdal.yml b/.github/workflows/docker-gdal.yml index 601edbe4..1a474b15 100644 --- a/.github/workflows/docker-gdal.yml +++ b/.github/workflows/docker-gdal.yml @@ -20,8 +20,9 @@ jobs: fail-fast: false matrix: container: - - "ghcr.io/osgeo/gdal:ubuntu-small-latest" # >= python 3.10.6 - - "ghcr.io/osgeo/gdal:ubuntu-small-3.8.3" # python 3.10.12 + - "ghcr.io/osgeo/gdal:ubuntu-small-latest" # >= python 3.12.3 + - "ghcr.io/osgeo/gdal:ubuntu-small-3.9.2" # python 3.12.3 + - "ghcr.io/osgeo/gdal:ubuntu-small-3.8.5" # python 3.10.12 - "ghcr.io/osgeo/gdal:ubuntu-small-3.7.3" # python 3.10.12 - "ghcr.io/osgeo/gdal:ubuntu-small-3.6.4" # python 3.10.6 - "osgeo/gdal:ubuntu-small-3.5.3" # python 3.8.10 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 131d78e7..740e565b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -54,36 +54,42 @@ jobs: needs: [build-sdist] runs-on: ubuntu-latest container: - image: "ghcr.io/osgeo/gdal:ubuntu-small-3.8.5" + image: "ghcr.io/osgeo/gdal:ubuntu-small-3.9.2" steps: + - name: Install packages + run: | + apt-get update && apt-get install -y build-essential python3-dev + + - name: Create virtual environment + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + . $HOME/.cargo/env + uv venv .venv + - name: Download sdist from artifacts uses: actions/download-artifact@v4 with: name: pyogrio-sdist path: wheelhouse - - name: Install pip - run: | - apt-get update && apt-get install -y python3-pip - - name: Build from sdist and install test dependencies shell: bash run: | - python3 -m pip install --no-cache-dir wheelhouse/*.tar.gz - python3 -m pip install --no-cache-dir pytest pandas pyproj shapely>=2 - python3 -m pip install --no-cache-dir --no-deps geopandas - python3 -m pip list + uv pip install --no-cache wheelhouse/*.tar.gz + uv pip install --no-cache pytest pandas pyproj shapely>=2 + uv pip install --no-cache --no-deps geopandas + uv pip list - name: Run tests shell: bash run: | cd .. - python3 -m pytest --pyargs pyogrio.tests -v + uv run python -m pytest --pyargs pyogrio.tests -v build-wheels-linux: name: Build wheels on Linux - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-latest" steps: - name: Checkout @@ -103,7 +109,7 @@ jobs: uses: docker/build-push-action@v6 with: context: . - file: ci/manylinux2014_x86_64-vcpkg-gdal.Dockerfile + file: ci/manylinux_2_28_x86_64-vcpkg-gdal.Dockerfile tags: manylinux-vcpkg-gdal:latest push: false load: true @@ -129,7 +135,7 @@ jobs: matrix: include: - os: "macos-12" - triplet: "x64-osx-dynamic" + triplet: "x64-osx-dynamic-release" arch: x86_64 vcpkg_cache: "/Users/runner/.cache/vcpkg/archives" vcpkg_logs: "/usr/local/share/vcpkg/buildtrees/**/*.log" @@ -160,7 +166,7 @@ jobs: path: | ${{ matrix.vcpkg_cache }} # bump the last digit to avoid using previous build cache - key: ${{ matrix.os }}-${{ matrix.arch }}-vcpkg-gdal3.8.5-cache0 + key: ${{ matrix.os }}-${{ matrix.arch }}-vcpkg-gdal3.9.2-cache0 # MacOS build requires aclocal, which is part of automake, but appears # to be missing in default image @@ -176,9 +182,9 @@ jobs: cd $VCPKG_INSTALLATION_ROOT # on mac the clone is not clean, otherwise git pull fails git reset --hard - # pull latest version to ensure the required commit with GDAL 3.8 is available + # pull latest version to ensure the required commit with GDAL 3.9 is available git pull - git checkout 4f4a1821b2e8c7a2863e4df65a4d514f84144049 + git checkout 73794ce5f63fd138fab999a22959ca7c6305d93c - name: Install GDAL env: @@ -215,17 +221,26 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-20.04", "windows-latest", "macos-12", "macos-latest"] + os: + [ + "ubuntu-latest", + "ubuntu-20.04", + "windows-latest", + "macos-12", + "macos-latest", + ] python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] include: + - os: "ubuntu-latest" + artifact: pyogrio-wheel-linux - os: "ubuntu-20.04" artifact: pyogrio-wheel-linux - os: "windows-latest" artifact: pyogrio-wheel-x64-windows-dynamic-release - os: "macos-12" - artifact: pyogrio-wheel-x64-osx-dynamic + artifact: pyogrio-wheel-x64-osx-dynamic-release - os: "macos-latest" - artifact: pyogrio-wheel-x64-osx-dynamic + artifact: pyogrio-wheel-arm64-osx-dynamic-release steps: - name: Checkout @@ -236,8 +251,20 @@ jobs: with: python-version: ${{ matrix.python-version }} allow-prereleases: true - cache: "pip" - cache-dependency-path: "ci/requirements-wheel-test.txt" + + - name: Create virtual environment (Linux / MacOS) + if: ${{ runner.os != 'Windows' }} + shell: bash + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + . $HOME/.cargo/env + uv venv .venv + + - name: Create virtual environment (Windows) + if: ${{ runner.os == 'Windows' }} + run: | + irm https://astral.sh/uv/install.ps1 | iex + uv venv .venv - name: Download wheels from artifacts uses: actions/download-artifact@v4 @@ -248,19 +275,19 @@ jobs: - name: Install dependencies and pyogrio wheel shell: bash run: | - python -m pip install -r ci/requirements-wheel-test.txt + uv pip install --no-cache -r ci/requirements-wheel-test.txt if [ ${{ matrix.python-version }} != "3.12" ]; then - python -m pip install --no-deps geopandas + uv pip install --no-cache --no-deps geopandas fi - python -m pip install --pre --find-links wheelhouse pyogrio - python -m pip list + uv pip install --no-cache --pre --no-index --find-links wheelhouse pyogrio + uv pip list - name: Run tests shell: bash run: | cd .. - python -c "import pyogrio; print(f'GDAL version: {pyogrio.__gdal_version__}\nGEOS version: {pyogrio.__gdal_geos_version__}')" - python -m pytest --pyargs pyogrio.tests -v + uv run python -c "import pyogrio; print(f'GDAL version: {pyogrio.__gdal_version__}\nGEOS version: {pyogrio.__gdal_geos_version__}')" + uv run python -m pytest --pyargs pyogrio.tests -v publish: name: Publish pyogrio to GitHub / PyPI diff --git a/CHANGES.md b/CHANGES.md index 626f3eb2..214ab946 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,13 @@ # CHANGELOG -## 0.9.1 (yyyy-mm-dd) +## 0.10.0 (yyyy-mm-dd) + +### Bug fixes + +- Silence warning from `write_dataframe` with `GeoSeries.notna()` (#435). +- Enable mask & bbox filter when geometry column not read (#431). +- Raise NotImplmentedError when user attempts to write to an open file handle (#442). +- Prevent seek on read from compressed inputs (#443). ### Packaging @@ -10,13 +17,10 @@ packages if needed: more info [here](https://gdal.org/download.html#conda). This also leads to `pyproj` becoming an optional dependency; you will need to install `pyproj` in order to support spatial reference systems (#452). - -### Bug fixes - -- Silence warning from `write_dataframe` with `GeoSeries.notna()` (#435). -- Enable mask & bbox filter when geometry column not read (#431). -- Raise NotImplmentedError when user attempts to write to an open file handle (#442). -- Prevent seek on read from compressed inputs (#443). +- The GDAL library included in the wheels is updated from 3.8.5 to GDAL 3.9.2 + and includes the LIBKML driver (#589). +- Updates to manylinux_2_28 for building Linux x64 wheels; these will require + Linux versions with glibc >= 2.28 (#589). ## 0.9.0 (2024-06-17) diff --git a/ci/custom-triplets/x64-linux-dynamic.cmake b/ci/custom-triplets/x64-linux-dynamic-release.cmake similarity index 100% rename from ci/custom-triplets/x64-linux-dynamic.cmake rename to ci/custom-triplets/x64-linux-dynamic-release.cmake diff --git a/ci/custom-triplets/x64-osx-dynamic.cmake b/ci/custom-triplets/x64-osx-dynamic-release.cmake similarity index 100% rename from ci/custom-triplets/x64-osx-dynamic.cmake rename to ci/custom-triplets/x64-osx-dynamic-release.cmake diff --git a/ci/manylinux_2_28_aarch64-vcpkg-gdal.Dockerfile b/ci/manylinux_2_28_aarch64-vcpkg-gdal.Dockerfile index 568143e4..174b37a1 100644 --- a/ci/manylinux_2_28_aarch64-vcpkg-gdal.Dockerfile +++ b/ci/manylinux_2_28_aarch64-vcpkg-gdal.Dockerfile @@ -4,7 +4,7 @@ FROM quay.io/pypa/manylinux_2_28_aarch64:2024-08-12-7fde9b1 RUN dnf -y install curl zip unzip tar ninja-build perl-IPC-Cmd RUN git clone https://github.com/Microsoft/vcpkg.git /opt/vcpkg && \ - git -C /opt/vcpkg checkout 4f4a1821b2e8c7a2863e4df65a4d514f84144049 + git -C /opt/vcpkg checkout 73794ce5f63fd138fab999a22959ca7c6305d93c ENV VCPKG_INSTALLATION_ROOT="/opt/vcpkg" ENV PATH="${PATH}:/opt/vcpkg" @@ -24,7 +24,6 @@ RUN bootstrap-vcpkg.sh && \ vcpkg integrate bash COPY ci/custom-triplets/arm64-linux-dynamic-release.cmake opt/vcpkg/custom-triplets/arm64-linux-dynamic-release.cmake -COPY ci/vcpkg-custom-ports/ opt/vcpkg/custom-ports/ COPY ci/vcpkg.json opt/vcpkg/ # temporary workaround for https://github.com/microsoft/vcpkg/issues/36094 @@ -33,7 +32,6 @@ RUN git -C /opt/vcpkg apply vcpkg_linux_crosscompiling.patch ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/vcpkg/installed/arm64-linux-dynamic-release/lib" RUN vcpkg install --overlay-triplets=opt/vcpkg/custom-triplets \ - --overlay-ports=opt/vcpkg/custom-ports \ --feature-flags="versions,manifests" \ --x-manifest-root=opt/vcpkg \ --x-install-root=opt/vcpkg/installed && \ diff --git a/ci/manylinux2014_x86_64-vcpkg-gdal.Dockerfile b/ci/manylinux_2_28_x86_64-vcpkg-gdal.Dockerfile similarity index 62% rename from ci/manylinux2014_x86_64-vcpkg-gdal.Dockerfile rename to ci/manylinux_2_28_x86_64-vcpkg-gdal.Dockerfile index 99485463..915050d4 100644 --- a/ci/manylinux2014_x86_64-vcpkg-gdal.Dockerfile +++ b/ci/manylinux_2_28_x86_64-vcpkg-gdal.Dockerfile @@ -1,18 +1,15 @@ -FROM quay.io/pypa/manylinux2014_x86_64:2024-08-12-7fde9b1 +FROM quay.io/pypa/manylinux_2_28_x86_64:2024-07-15-c746fd8 # building openssl needs IPC-Cmd (https://github.com/microsoft/vcpkg/issues/24988) -RUN yum install -y curl unzip zip tar perl-IPC-Cmd - -# require python >= 3.7 (python 3.6 is default on base image) for meson -RUN ln -s /opt/python/cp38-cp38/bin/python3 /usr/bin/python3 +RUN dnf -y install curl zip unzip tar ninja-build perl-IPC-Cmd RUN git clone https://github.com/Microsoft/vcpkg.git /opt/vcpkg && \ - git -C /opt/vcpkg checkout 4f4a1821b2e8c7a2863e4df65a4d514f84144049 + git -C /opt/vcpkg checkout 73794ce5f63fd138fab999a22959ca7c6305d93c ENV VCPKG_INSTALLATION_ROOT="/opt/vcpkg" ENV PATH="${PATH}:/opt/vcpkg" -ENV VCPKG_DEFAULT_TRIPLET="x64-linux-dynamic" +ENV VCPKG_DEFAULT_TRIPLET="x64-linux-dynamic-release" # mkdir & touch -> workaround for https://github.com/microsoft/vcpkg/issues/27786 RUN bootstrap-vcpkg.sh && \ @@ -21,13 +18,11 @@ RUN bootstrap-vcpkg.sh && \ vcpkg integrate install && \ vcpkg integrate bash -COPY ci/custom-triplets/x64-linux-dynamic.cmake opt/vcpkg/custom-triplets/x64-linux-dynamic.cmake -COPY ci/vcpkg-custom-ports/ opt/vcpkg/custom-ports/ +COPY ci/custom-triplets/x64-linux-dynamic-release.cmake opt/vcpkg/custom-triplets/x64-linux-dynamic-release.cmake COPY ci/vcpkg.json opt/vcpkg/ -ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/vcpkg/installed/x64-linux-dynamic/lib" +ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/vcpkg/installed/x64-linux-dynamic-release/lib" RUN vcpkg install --overlay-triplets=opt/vcpkg/custom-triplets \ - --overlay-ports=opt/vcpkg/custom-ports \ --feature-flags="versions,manifests" \ --x-manifest-root=opt/vcpkg \ --x-install-root=opt/vcpkg/installed && \ diff --git a/ci/vcpkg-custom-ports/zlib/0002-skip-building-examples.patch b/ci/vcpkg-custom-ports/zlib/0002-skip-building-examples.patch deleted file mode 100644 index d18885a8..00000000 --- a/ci/vcpkg-custom-ports/zlib/0002-skip-building-examples.patch +++ /dev/null @@ -1,17 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index ee25365..40a1936 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -171,6 +171,7 @@ endif() - # Example binaries - #============================================================================ - -+if (0) - add_executable(example test/example.c) - target_link_libraries(example zlib) - add_test(example example) -@@ -188,3 +189,4 @@ if(HAVE_OFF64_T) - target_link_libraries(minigzip64 zlib) - set_target_properties(minigzip64 PROPERTIES COMPILE_FLAGS "-D_FILE_OFFSET_BITS=64") - endif() -+endif() diff --git a/ci/vcpkg-custom-ports/zlib/LICENSE b/ci/vcpkg-custom-ports/zlib/LICENSE deleted file mode 100644 index ca5fddfe..00000000 --- a/ci/vcpkg-custom-ports/zlib/LICENSE +++ /dev/null @@ -1,20 +0,0 @@ - Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler - - This software is provided 'as-is', without any express or implied - warranty. In no event will the authors be held liable for any damages - arising from the use of this software. - - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it - freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - 3. This notice may not be removed or altered from any source distribution. - - Jean-loup Gailly Mark Adler - jloup@gzip.org madler@alumni.caltech.edu \ No newline at end of file diff --git a/ci/vcpkg-custom-ports/zlib/portfile.cmake b/ci/vcpkg-custom-ports/zlib/portfile.cmake deleted file mode 100644 index bed8dce4..00000000 --- a/ci/vcpkg-custom-ports/zlib/portfile.cmake +++ /dev/null @@ -1,38 +0,0 @@ -set(VERSION 1.2.5.2) - -vcpkg_download_distfile(ARCHIVE_FILE - URLS "http://zlib.net/fossils/zlib-1.2.5.2.tar.gz" - FILENAME "zlib-1.2.5.2.tar.gz" - SHA512 d4bd29ebfd5642253cecb9b8364ee6de87442d192229a9080cc306b819745e80c0791bd0a8abefd0c5e11c958bc85485d5d5d051b4770e45f6f479f3bb16e867 -) - -vcpkg_extract_source_archive_ex( - OUT_SOURCE_PATH SOURCE_PATH - ARCHIVE ${ARCHIVE_FILE} - REF ${VERSION} - PATCHES - "0002-skip-building-examples.patch" -) - -# This is generated during the cmake build -file(REMOVE ${SOURCE_PATH}/zconf.h) - -vcpkg_configure_cmake( - SOURCE_PATH ${SOURCE_PATH} - PREFER_NINJA - OPTIONS - -DSKIP_INSTALL_FILES=ON - OPTIONS_DEBUG - -DSKIP_INSTALL_HEADERS=ON -) - -vcpkg_install_cmake() -file(INSTALL ${CMAKE_CURRENT_LIST_DIR}/vcpkg-cmake-wrapper.cmake DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT}) - -vcpkg_fixup_pkgconfig() - -file(INSTALL ${CMAKE_CURRENT_LIST_DIR}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT} RENAME copyright) - -vcpkg_copy_pdbs() - -file(COPY ${CMAKE_CURRENT_LIST_DIR}/usage DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT}) diff --git a/ci/vcpkg-custom-ports/zlib/usage b/ci/vcpkg-custom-ports/zlib/usage deleted file mode 100644 index 39d86182..00000000 --- a/ci/vcpkg-custom-ports/zlib/usage +++ /dev/null @@ -1,4 +0,0 @@ -The package zlib is compatible with built-in CMake targets: - - find_package(ZLIB REQUIRED) - target_link_libraries(main PRIVATE ZLIB::ZLIB) diff --git a/ci/vcpkg-custom-ports/zlib/vcpkg-cmake-wrapper.cmake b/ci/vcpkg-custom-ports/zlib/vcpkg-cmake-wrapper.cmake deleted file mode 100644 index 8624b708..00000000 --- a/ci/vcpkg-custom-ports/zlib/vcpkg-cmake-wrapper.cmake +++ /dev/null @@ -1,12 +0,0 @@ -find_path(ZLIB_INCLUDE_DIR NAMES zlib.h PATHS "${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/include" NO_DEFAULT_PATH) -find_library(ZLIB_LIBRARY_RELEASE NAMES zlib z PATHS "${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib" NO_DEFAULT_PATH) -find_library(ZLIB_LIBRARY_DEBUG NAMES zlibd z PATHS "${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/debug/lib" NO_DEFAULT_PATH) -if(NOT ZLIB_INCLUDE_DIR OR NOT (ZLIB_LIBRARY_RELEASE OR ZLIB_LIBRARY_DEBUG)) - message(FATAL_ERROR "Broken installation of vcpkg port zlib") -endif() -if(CMAKE_VERSION VERSION_LESS 3.4) - include(SelectLibraryConfigurations) - select_library_configurations(ZLIB) - unset(ZLIB_FOUND) -endif() -_find_package(${ARGS}) diff --git a/ci/vcpkg-custom-ports/zlib/vcpkg.json b/ci/vcpkg-custom-ports/zlib/vcpkg.json deleted file mode 100644 index c9ccf36b..00000000 --- a/ci/vcpkg-custom-ports/zlib/vcpkg.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "name": "zlib", - "version": "1.2.5.2", - "port-version": 0, - "description": "A compression library", - "homepage": "https://www.zlib.net/" -} diff --git a/ci/vcpkg.json b/ci/vcpkg.json index 95ef73bf..9fc30fbc 100644 --- a/ci/vcpkg.json +++ b/ci/vcpkg.json @@ -1,12 +1,18 @@ { "name": "pyogrio", - "version": "0.8.0", + "version": "0.10.0", "dependencies": [ { "name": "gdal", "default-features": false, - "features": ["recommended-features", "curl", "geos", "iconv"] + "features": [ + "recommended-features", + "curl", + "geos", + "iconv", + "libkml" + ] } ], - "builtin-baseline": "4f4a1821b2e8c7a2863e4df65a4d514f84144049" + "builtin-baseline": "73794ce5f63fd138fab999a22959ca7c6305d93c" } diff --git a/pyproject.toml b/pyproject.toml index 3ed6f129..999588b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ build-verbosity = 3 VCPKG_INSTALL = "$VCPKG_INSTALLATION_ROOT/installed/$VCPKG_DEFAULT_TRIPLET" GDAL_INCLUDE_PATH = "$VCPKG_INSTALL/include" GDAL_LIBRARY_PATH = "$VCPKG_INSTALL/lib" -GDAL_VERSION = "3.8.5" +GDAL_VERSION = "3.9.2" PYOGRIO_PACKAGE_DATA = 1 GDAL_DATA = "$VCPKG_INSTALL/share/gdal" PROJ_LIB = "$VCPKG_INSTALL/share/proj" @@ -33,7 +33,7 @@ repair-wheel-command = [ VCPKG_INSTALL = "$VCPKG_INSTALLATION_ROOT/installed/$VCPKG_DEFAULT_TRIPLET" GDAL_INCLUDE_PATH = "$VCPKG_INSTALL/include" GDAL_LIBRARY_PATH = "$VCPKG_INSTALL/lib" -GDAL_VERSION = "3.8.5" +GDAL_VERSION = "3.9.2" PYOGRIO_PACKAGE_DATA = 1 GDAL_DATA = "$VCPKG_INSTALL/share/gdal" PROJ_LIB = "$VCPKG_INSTALL/share/proj" @@ -47,7 +47,7 @@ repair-wheel-command = "delvewheel repair --add-path C:/vcpkg/installed/x64-wind VCPKG_INSTALL = "$VCPKG_INSTALLATION_ROOT/installed/x64-windows-dynamic-release" GDAL_INCLUDE_PATH = "$VCPKG_INSTALL/include" GDAL_LIBRARY_PATH = "$VCPKG_INSTALL/lib" -GDAL_VERSION = "3.8.5" +GDAL_VERSION = "3.9.2" PYOGRIO_PACKAGE_DATA = 1 GDAL_DATA = "$VCPKG_INSTALL/share/gdal" PROJ_LIB = "$VCPKG_INSTALL/share/proj"