Merge branch 'main' into open-sharing

Author: ZhiHanZ

.github/actions/artifact_upload/action.yml

@@ -34,7 +34,7 @@ runs:
 
     - name: Upload artifact to github
       if: steps.info.outputs.src == 'github'
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       with:
         name: ${{ inputs.profile }}-${{ inputs.sha }}-${{ inputs.target }}-${{ inputs.category }}
         path: ./target/${{ inputs.target }}/${{ inputs.profile }}/databend-*

.github/actions/build_linux/action.yml

@@ -14,7 +14,7 @@ runs:
       uses: ./.github/actions/setup_build_tool
       with:
         image: ${{ inputs.target }}
-        bypass_env_vars: RUSTFLAGS,RUST_LOG,RUSTC_WRAPPER,SCCACHE_BUCKET,SCCACHE_S3_KEY_PREFIX,SCCACHE_S3_USE_SSL,AWS_DEFAULT_REGION,AWS_REGION,AWS_ROLE_ARN,AWS_STS_REGIONAL_ENDPOINTS,AWS_WEB_IDENTITY_TOKEN_FILE
+        bypass_env_vars: RUSTFLAGS,RUST_LOG,RUSTC_WRAPPER,SCCACHE_BUCKET,SCCACHE_S3_KEY_PREFIX,SCCACHE_S3_USE_SSL,AWS_DEFAULT_REGION,AWS_REGION,AWS_ROLE_ARN,AWS_STS_REGIONAL_ENDPOINTS,AWS_WEB_IDENTITY_TOKEN_FILE,je_cv_pthread_getname_np
 
     - name: Build Debug
       if: inputs.profile == 'debug'

.github/actions/publish_binary/nfpm.yaml

@@ -0,0 +1,37 @@
+name: "${name}"
+arch: "amd64"
+platform: "linux"
+version: "${version}"
+section: "database"
+priority: "extra"
+maintainer: "Databend Authors <opensource@datafuselabs.com>"
+description: |
+  Databend is a powerful cloud data warehouse. Built for elasticity and efficiency.
+  Free and open. Also available in the cloud: https://app.databend.com
+vendor: "Datafuse Labs"
+homepage: "https://databend.rs"
+license: "Apache-2.0"
+contents:
+  # Binaries
+  - src: ${path}/bin/databend-query
+    dst: /usr/bin/databend-query
+  - src: ${path}/bin/databend-meta
+    dst: /usr/bin/databend-meta
+  - src: ${path}/bin/databend-metactl
+    dst: /usr/bin/databend-metactl
+  - src: ${path}/bin/databend-metabench
+    dst: /usr/bin/databend-metabench
+
+  # Configs
+  - src: ${path}/configs/databend-query.toml
+    dst: /etc/databend/databend-query.toml
+    type: config
+  - src: ${path}/configs/databend-meta.toml
+    dst: /etc/databend/databend-meta.toml
+    type: config
+
+  # Docs
+  - src: ${path}/readme.txt
+    dst: /usr/share/doc/databend/readme.txt
+    file_info:
+      mode: 0644

.github/workflows/databend-release.yml

@@ -128,7 +128,7 @@ jobs:
           version=${{ needs.create_release.outputs.version }}
           sha256sum databend-${version}-${target}.tar.gz >> sha256-${version}-${target}.txt
       - name: post sha256
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: sha256sums
           path: sha256-${{ needs.create_release.outputs.version }}-${{ steps.target.outputs.target }}.txt
@@ -202,7 +202,7 @@ jobs:
           version=${{ needs.create_release.outputs.version }}
           sha256sum databend-${version}-${target}.tar.gz >> sha256-${version}-${target}.txt
       - name: post sha256
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: sha256sums
           path: sha256-${{ needs.create_release.outputs.version }}-${{ steps.target.outputs.target }}.txt
@@ -215,6 +215,73 @@ jobs:
           target: ${{ steps.target.outputs.target }}
           repo_role_arn: ${{ secrets.REPO_ROLE_ARN }}
 
+  publish_hive:
+    name: hive assets
+    runs-on: [self-hosted, X64, Linux, development]
+    needs: [create_release]
+    strategy:
+      fail-fast: false
+      matrix:
+        arch:
+          - x86_64
+        platform:
+          - gnu
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Get the version
+        id: get_version
+        run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
+      - name: Get target
+        id: target
+        run: echo 'target=${{ matrix.arch }}-unknown-linux-${{ matrix.platform }}' >> $GITHUB_OUTPUT
+      - name: Setup Build Tool
+        uses: ./.github/actions/setup_build_tool
+        with:
+          image: ${{ steps.target.outputs.target }}
+      - name: Build Binary
+        run: |
+          cargo build --release --target=${{ steps.target.outputs.target }} --features hive
+      - name: Copyobj zlib for gnu binaries
+        if: matrix.platform == 'gnu'
+        run: |
+          target=${{ steps.target.outputs.target }}
+          build-tool /usr/bin/${{ matrix.arch }}-linux-gnu-objcopy --compress-debug-sections=zlib-gnu ./target/${target}/release/databend-query
+          build-tool /usr/bin/${{ matrix.arch }}-linux-gnu-objcopy --compress-debug-sections=zlib-gnu ./target/${target}/release/databend-meta
+          build-tool /usr/bin/${{ matrix.arch }}-linux-gnu-objcopy --compress-debug-sections=zlib-gnu ./target/${target}/release/databend-metactl
+      - name: Pack binaries
+        run: |
+          target=${{ steps.target.outputs.target }}
+          version=${{ needs.create_release.outputs.version }}
+          mkdir -p release/${target}/{bin,configs}
+          cp ./target/${target}/release/databend-* release/${target}/bin/
+          rm -f release/${target}/bin/*.d
+          cp ./scripts/ci/deploy/config/databend-query-node-1.toml release/${target}/configs/databend-query.toml
+          cp ./scripts/ci/deploy/config/databend-meta-node-1.toml release/${target}/configs/databend-meta.toml
+          cp ./.github/actions/publish_binary/databend-release-readme.txt release/${target}/readme.txt
+          cp -r ./.github/actions/publish_binary/databend-scripts-for-release release/${target}/scripts
+          tar -C ./release/${target} -czvf databend-hive-${version}-${target}.tar.gz bin configs scripts readme.txt
+      - name: generate sha256sums
+        run: |
+          target=${{ steps.target.outputs.target }}
+          version=${{ needs.create_release.outputs.version }}
+          sha256sum databend-hive-${version}-${target}.tar.gz >> sha256-hive-${version}-${target}.txt
+      - name: post sha256
+        uses: actions/upload-artifact@v3
+        with:
+          name: sha256sums
+          path: sha256-hive-${{ needs.create_release.outputs.version }}-${{ steps.target.outputs.target }}.txt
+          retention-days: 1
+      - name: Publish Binaries
+        uses: ./.github/actions/publish_binary
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          version: hive-${{ needs.create_release.outputs.version }}
+          target: ${{ steps.target.outputs.target }}
+          repo_role_arn: ${{ secrets.REPO_ROLE_ARN }}
+
   publish_sqllogic_testsuites:
     name: sqllogic testsuites
     runs-on: ubuntu-latest
@@ -346,8 +413,64 @@ jobs:
           file: ./docker/${{ matrix.distro }}/${{ matrix.service }}.Dockerfile
           build-args: VERSION=${{ needs.create_release.outputs.version }}
 
+  release_packages:
+    name: release packages
+    runs-on: ubuntu-latest
+    needs: [create_release, publish_linux]
+    strategy:
+      matrix:
+        arch:
+          - x86_64
+        platform:
+          - musl
+        packager:
+          - deb
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install nfpm@latest
+        run: curl -o /tmp/nfpm.tar.gz -sSL https://github.com/goreleaser/nfpm/releases/download/v2.22.2/nfpm_2.22.2_Linux_x86_64.tar.gz && cd /tmp && tar xf /tmp/nfpm.tar.gz && sudo mv nfpm /usr/local/bin && sudo chmod a+x /usr/local/bin/nfpm
+      - name: Get target
+        id: target
+        run: echo 'target=${{ matrix.arch }}-unknown-linux-${{ matrix.platform }}' >> $GITHUB_OUTPUT
+      - name: Download binaries for usage
+        id: download_binaries
+        run: |
+          target=${{ steps.target.outputs.target }}
+          version="${{ needs.create_release.outputs.version }}"
+          mkdir -p ./distro/
+          wget -P distro -qc https://repo.databend.rs/databend/${version}/databend-${version}-${target}.tar.gz
+          tar x -C distro -f ./distro/databend-${version}-${target}.tar.gz
+      - name: Build Packages
+        id: build_packages
+        run: |
+          export name="databend"
+          export version="${{ needs.create_release.outputs.version }}"
+          export path="distro"
+          nfpm pkg --packager ${{ matrix.packager }} -f <(envsubst '${name} ${version} ${path}' < .github/actions/publish_binary/nfpm.yaml)
+      - name: generate sha256sums
+        run: |
+          version="${{ needs.create_release.outputs.version }}"
+          sha256sum databend_*.${{ matrix.packager }} >> sha256-${version}-${{ matrix.packager }}.txt
+      - name: post sha256
+        uses: actions/upload-artifact@v3
+        with:
+          name: sha256sums
+          path: sha256-${{ needs.create_release.outputs.version }}-${{ matrix.packager }}.txt
+          retention-days: 1
+      - name: Update release to github
+        shell: bash
+        # Reference: https://cli.github.com/manual/gh_release_upload
+        run: |
+          target=${{ steps.target.outputs.target }}
+          version="${{ needs.create_release.outputs.version }}"
+          # name looks like: `databend_0.8.144~nightly_amd64.deb`
+          gh release upload ${version} databend_*.${{ matrix.packager }} --clobber
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
   upload-sha256sums-plugin:
-    needs: [create_release, publish_linux, publish_macos]
+    needs: [create_release, publish_linux, publish_macos, release_packages]
     runs-on: ubuntu-latest
     name: upload-sha256sums
     steps:

.github/workflows/dev-linux.yml

@@ -64,6 +64,15 @@ jobs:
           # fetch all tags,  metasrv and metaclient need tag as its version.
           fetch-depth: 0
       - uses: ./.github/actions/build_linux
+        env:
+          # To avoid the `undefined reference to pthread_getname_np` error during linking,
+          # here we override set config environment var `je_cv_pthread_getname_np` to `no`,
+          # as suggested by tikv-jemalloc-sys's maintainer @BusyJay:
+          # https://github.com/tikv/jemallocator/issues/30#issuecomment-1183786410
+          #
+          # Hopefully this trick can be removed when rust upgraded to musl 1.2.3
+          # https://github.com/rust-lang/rust/pull/102891
+          je_cv_pthread_getname_np: no
         with:
           target: ${{ matrix.arch }}-unknown-linux-musl
           profile: debug

.github/workflows/production.yml

@@ -64,6 +64,15 @@ jobs:
           target: ${{ matrix.target }}
           profile: release
 
+  build_hive:
+    runs-on: [self-hosted, X64, Linux, development]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/build_linux_hive
+        with:
+          target: x86_64-unknown-linux-gnu
+          profile: release
+
   test_unit:
     timeout-minutes: 30
     runs-on: [self-hosted, X64, Linux]