From f24577cf76052be819f9c40e6d68d91974d7c468 Mon Sep 17 00:00:00 2001 From: Yuan Date: Mon, 1 Sep 2025 15:17:25 +0100 Subject: [PATCH 1/9] [VL] fix centos7 docker image build Signed-off-by: Yuan --- .github/workflows/docker_image.yml | 1 - dev/vcpkg/setup-build-depends.sh | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml index 40aefa6c18b6..0a665fde0bd5 100644 --- a/.github/workflows/docker_image.yml +++ b/.github/workflows/docker_image.yml @@ -36,7 +36,6 @@ env: jobs: build-vcpkg-centos-7: - if: ${{ startsWith(github.repository, 'apache/') }} runs-on: ubuntu-latest steps: diff --git a/dev/vcpkg/setup-build-depends.sh b/dev/vcpkg/setup-build-depends.sh index 6d4695fb00f8..95132c9bdd46 100755 --- a/dev/vcpkg/setup-build-depends.sh +++ b/dev/vcpkg/setup-build-depends.sh @@ -99,9 +99,9 @@ install_centos_7() { # Requires git >= 2.7.4 if [[ "$(git --version)" != "git version 2."* ]]; then yum -y remove git - # Requires 'centos-release-scl' package to be installed. - yum -y install rh-git227 - source /opt/rh/rh-git227/enable + yum install -y https://mirrors.ibiblio.org/IUS/ius-release-el7.rpm + sed -i -e "s|https://repo.ius.io/7|https://vault.ius.io/el7|g" /etc/yum.repos.d/ius*repo + yum install -y git236 fi # flex>=2.6.0 From d7205e05076f07cad23fdf595d7eb04678867ad6 Mon Sep 17 00:00:00 2001 From: Yuan Date: Mon, 1 Sep 2025 23:15:36 +0100 Subject: [PATCH 2/9] fix build arrow Signed-off-by: Yuan --- dev/builddeps-veloxbe.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh index 6688818b7388..0493ad15800a 100755 --- a/dev/builddeps-veloxbe.sh +++ b/dev/builddeps-veloxbe.sh @@ -208,6 +208,8 @@ fi concat_velox_param function build_arrow { + get_velox + setup_dependencies cd $GLUTEN_DIR/dev source ./build_arrow.sh } @@ -261,10 +263,8 @@ function build_velox_backend { } function get_velox { - ( - cd $GLUTEN_DIR/ep/build-velox/src - ./get_velox.sh $VELOX_PARAMETER - ) + cd $GLUTEN_DIR/ep/build-velox/src + ./get_velox.sh $VELOX_PARAMETER } function setup_dependencies { From 48c2500d7fb953289faf15e4fe6b9dd38c7702d3 Mon Sep 17 00:00:00 2001 From: Yuan Date: Mon, 1 Sep 2025 23:19:07 +0100 Subject: [PATCH 3/9] test Signed-off-by: Yuan --- .github/workflows/docker_image.yml | 37 +++++++++++---------- dev/docker/Dockerfile.centos7-static-build | 2 +- dev/docker/Dockerfile.centos8-dynamic-build | 2 +- dev/docker/Dockerfile.centos9-dynamic-build | 2 +- 4 files changed, 23 insertions(+), 20 deletions(-) diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml index 0a665fde0bd5..74a9083d40dc 100644 --- a/.github/workflows/docker_image.yml +++ b/.github/workflows/docker_image.yml @@ -16,6 +16,9 @@ name: Build and Push Docker Image on: + pull_request: + paths: + - .github/workflows/docker_image.yml push: branches: - main @@ -45,11 +48,11 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + # - name: Login to Docker Hub + # uses: docker/login-action@v2 + # with: + # username: ${{ secrets.DOCKERHUB_USER }} + # password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push Docker image uses: docker/build-push-action@v6 @@ -191,7 +194,7 @@ jobs: retention-days: 1 build-centos-8: - if: ${{ startsWith(github.repository, 'apache/') }} + # if: ${{ startsWith(github.repository, 'apache/') }} runs-on: ${{ matrix.os }} strategy: matrix: @@ -223,11 +226,11 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + # - name: Login to Docker Hub + # uses: docker/login-action@v2 + # with: + # username: ${{ secrets.DOCKERHUB_USER }} + # password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push by digest id: build @@ -254,7 +257,7 @@ jobs: retention-days: 1 build-centos-9: - if: ${{ startsWith(github.repository, 'apache/') }} + # if: ${{ startsWith(github.repository, 'apache/') }} runs-on: ${{ matrix.os }} strategy: matrix: @@ -286,11 +289,11 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + # - name: Login to Docker Hub + # uses: docker/login-action@v2 + # with: + # username: ${{ secrets.DOCKERHUB_USER }} + # password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push by digest id: build diff --git a/dev/docker/Dockerfile.centos7-static-build b/dev/docker/Dockerfile.centos7-static-build index dfd2499e58e8..c2faf49cf9cd 100644 --- a/dev/docker/Dockerfile.centos7-static-build +++ b/dev/docker/Dockerfile.centos7-static-build @@ -35,7 +35,7 @@ RUN set -ex; \ -e 's/mirror\.centos\.org/vault.centos.org/' \ /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo; \ yum install -y git patch wget sudo java-1.8.0-openjdk-devel ccache; \ - git clone --depth=1 https://github.com/apache/incubator-gluten /opt/gluten; \ + git clone --depth=1 -b wip_fix_centos7_docker_build https://github.com/zhouyuan/gluten /opt/gluten; \ echo "check_certificate = off" >> ~/.wgetrc; \ cd /opt/gluten && bash ./dev/vcpkg/setup-build-depends.sh; \ mkdir -p ${VCPKG_PATH}; \ diff --git a/dev/docker/Dockerfile.centos8-dynamic-build b/dev/docker/Dockerfile.centos8-dynamic-build index 04d5da8571a6..ac02c7b9003b 100644 --- a/dev/docker/Dockerfile.centos8-dynamic-build +++ b/dev/docker/Dockerfile.centos8-dynamic-build @@ -45,7 +45,7 @@ RUN set -ex; \ wget -nv https://archive.apache.org/dist/celeborn/celeborn-0.6.0/apache-celeborn-0.6.0-bin.tgz -P /opt/; \ wget -nv https://archive.apache.org/dist/incubator/uniffle/0.9.2/apache-uniffle-0.9.2-incubating-bin.tar.gz -P /opt/; \ wget -nv https://archive.apache.org/dist/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz -P /opt/; \ - git clone --depth=1 https://github.com/apache/incubator-gluten /opt/gluten; \ + git clone --depth=1 -b wip_fix_centos7_docker_build https://github.com/zhouyuan/gluten /opt/gluten; \ cd /opt/gluten/.github/workflows/util/; \ ./install_spark_resources.sh 3.2; \ ./install_spark_resources.sh 3.3; \ diff --git a/dev/docker/Dockerfile.centos9-dynamic-build b/dev/docker/Dockerfile.centos9-dynamic-build index e50268cbc0dc..85cce7773525 100644 --- a/dev/docker/Dockerfile.centos9-dynamic-build +++ b/dev/docker/Dockerfile.centos9-dynamic-build @@ -43,7 +43,7 @@ RUN set -ex; \ wget -nv https://archive.apache.org/dist/celeborn/celeborn-0.6.0/apache-celeborn-0.6.0-bin.tgz -P /opt/; \ wget -nv https://archive.apache.org/dist/incubator/uniffle/0.9.2/apache-uniffle-0.9.2-incubating-bin.tar.gz -P /opt/; \ wget -nv https://archive.apache.org/dist/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz -P /opt/; \ - git clone --depth=1 https://github.com/apache/incubator-gluten /opt/gluten; \ + git clone --depth=1 -b wip_fix_centos7_docker_build https://github.com/zhouyuan/gluten /opt/gluten; \ cd /opt/gluten/.github/workflows/util/; \ ./install_spark_resources.sh 3.2; \ ./install_spark_resources.sh 3.3; \ From b497a63ca4d96781893ae07979dea9bb1323103c Mon Sep 17 00:00:00 2001 From: Yuan Date: Tue, 2 Sep 2025 09:30:32 +0100 Subject: [PATCH 4/9] switch to use rh227 git Signed-off-by: Yuan --- dev/vcpkg/setup-build-depends.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/vcpkg/setup-build-depends.sh b/dev/vcpkg/setup-build-depends.sh index 95132c9bdd46..46487f53f39c 100755 --- a/dev/vcpkg/setup-build-depends.sh +++ b/dev/vcpkg/setup-build-depends.sh @@ -99,9 +99,9 @@ install_centos_7() { # Requires git >= 2.7.4 if [[ "$(git --version)" != "git version 2."* ]]; then yum -y remove git - yum install -y https://mirrors.ibiblio.org/IUS/ius-release-el7.rpm - sed -i -e "s|https://repo.ius.io/7|https://vault.ius.io/el7|g" /etc/yum.repos.d/ius*repo - yum install -y git236 + yum -y install rh-git227 + source /opt/rh/rh-git227/enable + ln -s /opt/rh/rh-git227/root/usr/bin/git /usr/local/bin/git fi # flex>=2.6.0 From 5966924c266da0f901129fd28d6c9c8df1ec983b Mon Sep 17 00:00:00 2001 From: Yuan Date: Tue, 2 Sep 2025 09:48:53 +0100 Subject: [PATCH 5/9] enable Signed-off-by: Yuan --- dev/docker/Dockerfile.centos7-static-build | 1 + dev/vcpkg/setup-build-depends.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/docker/Dockerfile.centos7-static-build b/dev/docker/Dockerfile.centos7-static-build index c2faf49cf9cd..63616b470e3d 100644 --- a/dev/docker/Dockerfile.centos7-static-build +++ b/dev/docker/Dockerfile.centos7-static-build @@ -42,6 +42,7 @@ RUN set -ex; \ echo "Build arrow, then install the native libs to system paths and jar package to .m2/ directory."; \ cd /opt/gluten; \ source /opt/rh/devtoolset-11/enable; \ + source /opt/rh/rh-git227/enable; \ bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON --enable_s3=ON --enable_gcs=ON \ --enable_hdfs=ON --enable_abfs=ON build_arrow; \ mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta -Pspark-3.5 -DskipTests; \ diff --git a/dev/vcpkg/setup-build-depends.sh b/dev/vcpkg/setup-build-depends.sh index 46487f53f39c..01b42ba9b891 100755 --- a/dev/vcpkg/setup-build-depends.sh +++ b/dev/vcpkg/setup-build-depends.sh @@ -101,7 +101,7 @@ install_centos_7() { yum -y remove git yum -y install rh-git227 source /opt/rh/rh-git227/enable - ln -s /opt/rh/rh-git227/root/usr/bin/git /usr/local/bin/git + echo "source /opt/rh/rh-git227/enable" >> ~/.bashrc fi # flex>=2.6.0 From 222fb312ff5a3234e38762b85af99f970f1868e7 Mon Sep 17 00:00:00 2001 From: Yuan Date: Tue, 2 Sep 2025 09:54:50 +0100 Subject: [PATCH 6/9] Revert "test" This reverts commit 48c2500d7fb953289faf15e4fe6b9dd38c7702d3. --- .github/workflows/docker_image.yml | 37 ++++++++++----------- dev/docker/Dockerfile.centos7-static-build | 2 +- dev/docker/Dockerfile.centos8-dynamic-build | 2 +- dev/docker/Dockerfile.centos9-dynamic-build | 2 +- 4 files changed, 20 insertions(+), 23 deletions(-) diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml index 74a9083d40dc..0a665fde0bd5 100644 --- a/.github/workflows/docker_image.yml +++ b/.github/workflows/docker_image.yml @@ -16,9 +16,6 @@ name: Build and Push Docker Image on: - pull_request: - paths: - - .github/workflows/docker_image.yml push: branches: - main @@ -48,11 +45,11 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - # - name: Login to Docker Hub - # uses: docker/login-action@v2 - # with: - # username: ${{ secrets.DOCKERHUB_USER }} - # password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push Docker image uses: docker/build-push-action@v6 @@ -194,7 +191,7 @@ jobs: retention-days: 1 build-centos-8: - # if: ${{ startsWith(github.repository, 'apache/') }} + if: ${{ startsWith(github.repository, 'apache/') }} runs-on: ${{ matrix.os }} strategy: matrix: @@ -226,11 +223,11 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - # - name: Login to Docker Hub - # uses: docker/login-action@v2 - # with: - # username: ${{ secrets.DOCKERHUB_USER }} - # password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push by digest id: build @@ -257,7 +254,7 @@ jobs: retention-days: 1 build-centos-9: - # if: ${{ startsWith(github.repository, 'apache/') }} + if: ${{ startsWith(github.repository, 'apache/') }} runs-on: ${{ matrix.os }} strategy: matrix: @@ -289,11 +286,11 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - # - name: Login to Docker Hub - # uses: docker/login-action@v2 - # with: - # username: ${{ secrets.DOCKERHUB_USER }} - # password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push by digest id: build diff --git a/dev/docker/Dockerfile.centos7-static-build b/dev/docker/Dockerfile.centos7-static-build index 63616b470e3d..e015dc517834 100644 --- a/dev/docker/Dockerfile.centos7-static-build +++ b/dev/docker/Dockerfile.centos7-static-build @@ -35,7 +35,7 @@ RUN set -ex; \ -e 's/mirror\.centos\.org/vault.centos.org/' \ /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo; \ yum install -y git patch wget sudo java-1.8.0-openjdk-devel ccache; \ - git clone --depth=1 -b wip_fix_centos7_docker_build https://github.com/zhouyuan/gluten /opt/gluten; \ + git clone --depth=1 https://github.com/apache/incubator-gluten /opt/gluten; \ echo "check_certificate = off" >> ~/.wgetrc; \ cd /opt/gluten && bash ./dev/vcpkg/setup-build-depends.sh; \ mkdir -p ${VCPKG_PATH}; \ diff --git a/dev/docker/Dockerfile.centos8-dynamic-build b/dev/docker/Dockerfile.centos8-dynamic-build index ac02c7b9003b..04d5da8571a6 100644 --- a/dev/docker/Dockerfile.centos8-dynamic-build +++ b/dev/docker/Dockerfile.centos8-dynamic-build @@ -45,7 +45,7 @@ RUN set -ex; \ wget -nv https://archive.apache.org/dist/celeborn/celeborn-0.6.0/apache-celeborn-0.6.0-bin.tgz -P /opt/; \ wget -nv https://archive.apache.org/dist/incubator/uniffle/0.9.2/apache-uniffle-0.9.2-incubating-bin.tar.gz -P /opt/; \ wget -nv https://archive.apache.org/dist/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz -P /opt/; \ - git clone --depth=1 -b wip_fix_centos7_docker_build https://github.com/zhouyuan/gluten /opt/gluten; \ + git clone --depth=1 https://github.com/apache/incubator-gluten /opt/gluten; \ cd /opt/gluten/.github/workflows/util/; \ ./install_spark_resources.sh 3.2; \ ./install_spark_resources.sh 3.3; \ diff --git a/dev/docker/Dockerfile.centos9-dynamic-build b/dev/docker/Dockerfile.centos9-dynamic-build index 85cce7773525..e50268cbc0dc 100644 --- a/dev/docker/Dockerfile.centos9-dynamic-build +++ b/dev/docker/Dockerfile.centos9-dynamic-build @@ -43,7 +43,7 @@ RUN set -ex; \ wget -nv https://archive.apache.org/dist/celeborn/celeborn-0.6.0/apache-celeborn-0.6.0-bin.tgz -P /opt/; \ wget -nv https://archive.apache.org/dist/incubator/uniffle/0.9.2/apache-uniffle-0.9.2-incubating-bin.tar.gz -P /opt/; \ wget -nv https://archive.apache.org/dist/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz -P /opt/; \ - git clone --depth=1 -b wip_fix_centos7_docker_build https://github.com/zhouyuan/gluten /opt/gluten; \ + git clone --depth=1 https://github.com/apache/incubator-gluten /opt/gluten; \ cd /opt/gluten/.github/workflows/util/; \ ./install_spark_resources.sh 3.2; \ ./install_spark_resources.sh 3.3; \ From 7db1a13898087ab3b1a10a439634abb58c209d8d Mon Sep 17 00:00:00 2001 From: Yuan Date: Tue, 2 Sep 2025 09:55:29 +0100 Subject: [PATCH 7/9] revert Signed-off-by: Yuan --- .github/workflows/docker_image.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml index 0a665fde0bd5..40aefa6c18b6 100644 --- a/.github/workflows/docker_image.yml +++ b/.github/workflows/docker_image.yml @@ -36,6 +36,7 @@ env: jobs: build-vcpkg-centos-7: + if: ${{ startsWith(github.repository, 'apache/') }} runs-on: ubuntu-latest steps: From 38cbccf3f09f9ced4480585768d2bb269d3b7c40 Mon Sep 17 00:00:00 2001 From: Yuan Date: Tue, 2 Sep 2025 11:05:16 +0100 Subject: [PATCH 8/9] fix duplicated get_velox and setup dependencies Signed-off-by: Yuan --- dev/builddeps-veloxbe.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh index 0493ad15800a..0eb1a0b2e3ef 100755 --- a/dev/builddeps-veloxbe.sh +++ b/dev/builddeps-veloxbe.sh @@ -208,8 +208,9 @@ fi concat_velox_param function build_arrow { - get_velox - setup_dependencies + if [ ! -d "$GLUTEN_DIR/ep/build-velox/build/velox_ep" ]; then + get_velox && setup_dependencies + fi cd $GLUTEN_DIR/dev source ./build_arrow.sh } From ef7eb63e0b3aeda90787907272273797b3a39a6b Mon Sep 17 00:00:00 2001 From: Yuan Date: Tue, 2 Sep 2025 11:06:27 +0100 Subject: [PATCH 9/9] add back comment Signed-off-by: Yuan --- dev/vcpkg/setup-build-depends.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/vcpkg/setup-build-depends.sh b/dev/vcpkg/setup-build-depends.sh index 01b42ba9b891..d2cec5b744bc 100755 --- a/dev/vcpkg/setup-build-depends.sh +++ b/dev/vcpkg/setup-build-depends.sh @@ -99,6 +99,7 @@ install_centos_7() { # Requires git >= 2.7.4 if [[ "$(git --version)" != "git version 2."* ]]; then yum -y remove git + # Requires 'centos-release-scl' package to be installed. yum -y install rh-git227 source /opt/rh/rh-git227/enable echo "source /opt/rh/rh-git227/enable" >> ~/.bashrc