Skip to content

Commit 43f1374

Browse files
SkyFan2002SkyFan2002
committed
Merge remote-tracking branch 'upstream/main' into refactor_append
# Conflicts: # src/query/service/src/interpreters/interpreter_insert.rs # src/query/storages/fuse/src/operations/common/processors/sink_commit.rs
2 parents 84851c2 + 8cbda70 commit 43f1374

File tree

186 files changed

+5191
-5129
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

186 files changed

+5191
-5129
lines changed

‎.github/actions/build_linux/action.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,14 @@ runs:
9292
readelf -p .comment ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query || true
9393
ldd ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query || true
9494
95+
- name: Spilt Binary Symbols
96+
shell: bash
97+
run: |
98+
objcopy --only-keep-debug ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query.debug
99+
chmod 0644 ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query.debug
100+
strip --strip-debug --remove-section=.comment --remove-section=.note ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query
101+
pushd ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }} && objcopy --add-gnu-debuglink databend-query.debug databend-query && popd
102+
95103
# - name: Compress Binaries with UPX
96104
# if: env.BUILD_PROFILE == 'debug'
97105
# uses: crazy-max/ghaction-upx@v2

‎.github/actions/pack_binaries/action.yml

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
name: "Pack Binary"
2+
description: "Pack releases binaries"
3+
inputs:
4+
target:
5+
description: "Release target"
6+
required: true
7+
category:
8+
description: "Release default/hdfs/udf/testsuite"
9+
required: false
10+
default: default
11+
version:
12+
description: "Release version"
13+
required: true
14+
15+
runs:
16+
using: "composite"
17+
steps:
18+
- name: Download artifact
19+
uses: ./.github/actions/artifact_download
20+
with:
21+
sha: ${{ github.sha }}
22+
target: ${{ inputs.target }}
23+
category: ${{ inputs.category }}
24+
path: distro/bin
25+
artifacts: metactl,meta,query,query.debug
26+
- name: Pack Binaries
27+
id: pack_binaries
28+
shell: bash
29+
run: |
30+
target=${{ inputs.target }}
31+
version=${{ inputs.version }}
32+
case ${{ inputs.category }} in
33+
default)
34+
pkg_name="databend-${version}-${target}"
35+
;;
36+
*)
37+
pkg_name="databend-${{ inputs.category }}-${version}-${target}"
38+
;;
39+
esac
40+
mkdir -p distro/{bin,configs,systemd,scripts}
41+
cp ./scripts/distribution/systemd/databend-* distro/systemd/
42+
cp ./scripts/distribution/configs/databend-* distro/configs/
43+
cp ./scripts/distribution/release-readme.txt distro/readme.txt
44+
cp -r ./scripts/distribution/local-scripts/* distro/scripts/
45+
cp -r ./scripts/distribution/package-scripts/* distro/scripts/
46+
tar -C ./distro --exclude='*.debug' -czvf ${pkg_name}.tar.gz bin configs systemd scripts readme.txt
47+
sha256sum ${pkg_name}.tar.gz >> sha256-${pkg_name}.txt
48+
echo "pkg_name=$pkg_name" >> $GITHUB_OUTPUT
49+
- name: post sha256
50+
uses: actions/upload-artifact@v4
51+
with:
52+
name: sha256sums-${{ inputs.category }}-${{ inputs.target }}
53+
path: sha256-${{ steps.pack_binaries.outputs.pkg_name }}.txt
54+
retention-days: 1
55+
- name: post binaries
56+
uses: actions/upload-artifact@v4
57+
with:
58+
name: ${{ steps.pack_binaries.outputs.pkg_name }}.tar.gz
59+
path: ${{ steps.pack_binaries.outputs.pkg_name }}.tar.gz
60+
retention-days: 1
61+
- name: Pack DBG Binaries
62+
id: pack_dbg_binaries
63+
shell: bash
64+
run: |
65+
target=${{ inputs.target }}
66+
version=${{ inputs.version }}
67+
case ${{ inputs.category }} in
68+
default)
69+
pkg_name="databend-${version}-${target}-dbg"
70+
;;
71+
*)
72+
pkg_name="databend-${{ inputs.category }}-${version}-${target}-dbg"
73+
;;
74+
esac
75+
mkdir -p distro/{bin,configs,systemd,scripts}
76+
cp ./scripts/distribution/systemd/databend-* distro/systemd/
77+
cp ./scripts/distribution/configs/databend-* distro/configs/
78+
cp ./scripts/distribution/release-readme.txt distro/readme.txt
79+
cp -r ./scripts/distribution/local-scripts/* distro/scripts/
80+
cp -r ./scripts/distribution/package-scripts/* distro/scripts/
81+
tar -C ./distro -czvf ${pkg_name}.tar.gz bin configs systemd scripts readme.txt
82+
sha256sum ${pkg_name}.tar.gz >> sha256-${pkg_name}.txt
83+
echo "pkg_name=$pkg_name" >> $GITHUB_OUTPUT
84+
- name: post dbg sha256
85+
uses: actions/upload-artifact@v4
86+
with:
87+
name: sha256sums-${{ inputs.category }}-${{ inputs.target }}-gdb
88+
path: sha256-${{ steps.pack_dbg_binaries.outputs.pkg_name }}.txt
89+
retention-days: 1
90+
- name: post dbg binaries
91+
uses: actions/upload-artifact@v4
92+
with:
93+
name: ${{ steps.pack_dbg_binaries.outputs.pkg_name }}.tar.gz
94+
path: ${{ steps.pack_dbg_binaries.outputs.pkg_name }}.tar.gz
95+
retention-days: 1

‎.github/actions/pack_distribution/action.yml

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
name: "Pack Deb"
2+
description: "Pack releases deb"
3+
inputs:
4+
arch:
5+
description: "Release arch"
6+
required: true
7+
packager:
8+
description: "Release default/hdfs/udf/testsuite"
9+
required: false
10+
default: default
11+
version:
12+
description: "Release version"
13+
required: true
14+
15+
runs:
16+
using: "composite"
17+
steps:
18+
- name: Install nfpm@latest
19+
shell: bash
20+
run: |
21+
curl -sSLo nfpm.tar.gz https://github.com/goreleaser/nfpm/releases/download/v2.26.0/nfpm_2.26.0_Linux_x86_64.tar.gz
22+
tar xf nfpm.tar.gz
23+
sudo mv nfpm /usr/local/bin
24+
sudo chmod a+x /usr/local/bin/nfpm
25+
rm nfpm.tar.gz
26+
- name: Get target
27+
id: target
28+
shell: bash
29+
run: |
30+
echo 'target=${{ inputs.arch }}-unknown-linux-gnu' >> $GITHUB_OUTPUT
31+
- name: Download artifacts
32+
uses: ./.github/actions/artifact_download
33+
with:
34+
sha: ${{ github.sha }}
35+
target: ${{ steps.target.outputs.target }}
36+
category: default
37+
artifacts: metactl,meta,query,query.debug
38+
path: distro/bin
39+
- name: Build Packages
40+
shell: bash
41+
id: build_packages
42+
run: |
43+
export name="databend"
44+
export version="${{ inputs.version }}"
45+
export path="distro"
46+
case "${{ inputs.arch }}" in
47+
x86_64)
48+
export arch="amd64"
49+
;;
50+
aarch64)
51+
export arch="arm64"
52+
;;
53+
esac
54+
55+
deb_version=${version/-/.}
56+
pkg_name="databend_${deb_version/v/}_${{ inputs.arch }}.${{ inputs.packager }}"
57+
mkdir -p distro/{bin,configs,systemd,scripts}
58+
cp ./scripts/distribution/systemd/databend-* distro/systemd/
59+
cp ./scripts/distribution/configs/databend-* distro/configs/
60+
cp ./scripts/distribution/release-readme.txt distro/readme.txt
61+
cp -r ./scripts/distribution/local-scripts/* distro/scripts/
62+
cp -r ./scripts/distribution/package-scripts/* distro/scripts/
63+
nfpm pkg --packager ${{ inputs.packager }} -t "$pkg_name" -f <(envsubst '${name} ${version} ${path} ${arch}' < scripts/distribution/nfpm.yaml)
64+
echo "pkg_name=$pkg_name" >> $GITHUB_OUTPUT
65+
- name: Build dbg Packages
66+
shell: bash
67+
id: build_dbg_packages
68+
run: |
69+
export name="databend_dbg"
70+
export version="${{ inputs.version }}"
71+
export path="distro"
72+
case "${{ inputs.arch }}" in
73+
x86_64)
74+
export arch="amd64"
75+
;;
76+
aarch64)
77+
export arch="arm64"
78+
;;
79+
esac
80+
81+
deb_version=${version/-/.}
82+
pkg_name="databend_${deb_version/v/}_${{ inputs.arch }}_dbg.${{ inputs.packager }}"
83+
nfpm pkg --packager ${{ inputs.packager }} -t "$pkg_name" -f <(envsubst '${name} ${version} ${path} ${arch}' < scripts/distribution/nfpm-dbg.yaml)
84+
echo "pkg_name=$pkg_name" >> $GITHUB_OUTPUT
85+
- name: Update release to github
86+
shell: bash
87+
env:
88+
GH_TOKEN: ${{ github.token }}
89+
# Reference: https://cli.github.com/manual/gh_release_upload
90+
run: |
91+
version="${{ inputs.version }}"
92+
# name looks like: `databend_0.8.144~nightly_amd64.deb`
93+
gh release upload ${version} ${{ steps.build_packages.outputs.pkg_name }} --clobber
94+
gh release upload ${version} ${{ steps.build_dbg_packages.outputs.pkg_name }} --clobber

‎.github/actions/publish_binary/action.yml

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,13 +37,19 @@ runs:
3737
shell: bash
3838
# Reference: https://cli.github.com/manual/gh_release_upload
3939
run: |
40-
gh release upload ${{ inputs.version }} ${{ steps.name.outputs.name }}.* --clobber
40+
gh release upload ${{ inputs.version }} ${{ steps.name.outputs.name }}.tar.gz --clobber
41+
if [ -f ${{ steps.name.outputs.name }}-dbg.tar.gz ]; then
42+
gh release upload ${{ inputs.version }} ${{ steps.name.outputs.name }}-dbg.tar.gz --clobber
43+
fi
4144
4245
- name: Sync normal release to R2
4346
shell: bash
4447
if: inputs.category == 'default'
4548
run: |
4649
aws s3 cp ${{ steps.name.outputs.name }}.tar.gz s3://repo/databend/${{ inputs.version }}/${{ steps.name.outputs.name }}.tar.gz --no-progress
50+
if [ -f ${{ steps.name.outputs.name }}-dbg.tar.gz ]; then
51+
aws s3 cp ${{ steps.name.outputs.name }}-dbg.tar.gz s3://repo/databend/${{ inputs.version }}/${{ steps.name.outputs.name }}-dbg.tar.gz --no-progress
52+
fi
4753
gh api /repos/databendlabs/databend/tags > tags.json
4854
aws s3 cp ./tags.json s3://repo/databend/tags.json
4955
gh api /repos/databendlabs/databend/releases > releases.json

‎.github/actions/setup_bendsql/action.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ runs:
1010
if bendsql --version; then
1111
exit 0
1212
fi
13-
curl --retry 5 -Lo /tmp/bendsql.tar.gz https://github.com/databendlabs/bendsql/releases/download/v0.18.3/bendsql-x86_64-unknown-linux-gnu.tar.gz
13+
curl --retry 5 -Lo /tmp/bendsql.tar.gz https://github.com/databendlabs/bendsql/releases/download/v0.23.2/bendsql-x86_64-unknown-linux-gnu.tar.gz
1414
tar -xzf /tmp/bendsql.tar.gz -C /tmp
1515
mv /tmp/bendsql /usr/local/bin/bendsql
1616
bendsql --version
@@ -21,7 +21,7 @@ runs:
2121
if bendsql --version; then
2222
exit 0
2323
fi
24-
curl --retry 5 -Lo /tmp/bendsql.tar.gz https://github.com/databendlabs/bendsql/releases/download/v0.18.3/bendsql-x86_64-apple-darwin.tar.gz
24+
curl --retry 5 -Lo /tmp/bendsql.tar.gz https://github.com/databendlabs/bendsql/releases/download/v0.23.2/bendsql-x86_64-apple-darwin.tar.gz
2525
tar -xzf /tmp/bendsql.tar.gz -C /tmp
2626
mv /tmp/bendsql /usr/local/bin/bendsql
2727
bendsql --version

‎.github/workflows/release.yml

Lines changed: 8 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,6 @@ jobs:
139139
matrix:
140140
target:
141141
- x86_64-unknown-linux-musl
142-
- aarch64-unknown-linux-musl
143142
steps:
144143
- name: Checkout
145144
uses: actions/checkout@v4
@@ -196,47 +195,17 @@ jobs:
196195
target: aarch64-unknown-linux-gnu
197196
- category: default
198197
target: x86_64-unknown-linux-musl
199-
- category: default
200-
target: aarch64-unknown-linux-musl
201198
steps:
202199
- name: Checkout
203200
uses: actions/checkout@v4
204201
with:
205202
ref: ${{ needs.create_release.outputs.sha }}
206-
- name: Download artifact
207-
uses: ./.github/actions/artifact_download
203+
- name: Pack Binaries
204+
uses: ./.github/actions/pack_binaries
208205
with:
209-
sha: ${{ github.sha }}
206+
version: ${{ needs.create_release.outputs.version }}
210207
target: ${{ matrix.target }}
211208
category: ${{ matrix.category }}
212-
path: distro/bin
213-
artifacts: metactl,meta,query
214-
- name: Pack Binaries
215-
run: |
216-
target=${{ matrix.target }}
217-
version=${{ needs.create_release.outputs.version }}
218-
case ${{ matrix.category }} in
219-
default)
220-
pkg_name="databend-${version}-${target}"
221-
;;
222-
*)
223-
pkg_name="databend-${{ matrix.category }}-${version}-${target}"
224-
;;
225-
esac
226-
mkdir -p distro/{bin,configs,systemd,scripts}
227-
cp ./scripts/distribution/systemd/databend-* distro/systemd/
228-
cp ./scripts/distribution/configs/databend-* distro/configs/
229-
cp ./scripts/distribution/release-readme.txt distro/readme.txt
230-
cp -r ./scripts/distribution/local-scripts/* distro/scripts/
231-
cp -r ./scripts/distribution/package-scripts/* distro/scripts/
232-
tar -C ./distro -czvf ${pkg_name}.tar.gz bin configs systemd scripts readme.txt
233-
sha256sum ${pkg_name}.tar.gz >> sha256-${pkg_name}.txt
234-
- name: post sha256
235-
uses: actions/upload-artifact@v4
236-
with:
237-
name: sha256sums-${{ matrix.category }}-${{ matrix.target }}
238-
path: sha256-*.txt
239-
retention-days: 1
240209
- name: Publish Binaries
241210
uses: ./.github/actions/publish_binary
242211
env:
@@ -455,55 +424,12 @@ jobs:
455424
uses: actions/checkout@v4
456425
with:
457426
ref: ${{ needs.create_release.outputs.sha }}
458-
- name: Install nfpm@latest
459-
run: |
460-
curl -sSLo nfpm.tar.gz https://github.com/goreleaser/nfpm/releases/download/v2.26.0/nfpm_2.26.0_Linux_x86_64.tar.gz
461-
tar xf nfpm.tar.gz
462-
sudo mv nfpm /usr/local/bin
463-
sudo chmod a+x /usr/local/bin/nfpm
464-
rm nfpm.tar.gz
465-
- name: Get target
466-
id: target
467-
run: |
468-
echo 'target=${{ matrix.arch }}-unknown-linux-gnu' >> $GITHUB_OUTPUT
469-
- name: Download artifacts
470-
uses: ./.github/actions/artifact_download
427+
- name: Pack distribution
428+
uses: ./.github/actions/pack_distribution
471429
with:
472-
sha: ${{ github.sha }}
473-
target: ${{ steps.target.outputs.target }}
474-
category: default
475-
artifacts: metactl,meta,query
476-
path: distro/bin
477-
- name: Build Packages
478-
id: build_packages
479-
run: |
480-
export name="databend"
481-
export version="${{ needs.create_release.outputs.version }}"
482-
export path="distro"
483-
case "${{ matrix.arch }}" in
484-
x86_64)
485-
export arch="amd64"
486-
;;
487-
aarch64)
488-
export arch="arm64"
489-
;;
490-
esac
491-
mkdir -p distro/{bin,configs,systemd,scripts}
492-
cp ./scripts/distribution/systemd/databend-* distro/systemd/
493-
cp ./scripts/distribution/configs/databend-* distro/configs/
494-
cp ./scripts/distribution/release-readme.txt distro/readme.txt
495-
cp -r ./scripts/distribution/local-scripts/* distro/scripts/
496-
cp -r ./scripts/distribution/package-scripts/* distro/scripts/
497-
nfpm pkg --packager ${{ matrix.packager }} -f <(envsubst '${name} ${version} ${path} ${arch}' < scripts/distribution/nfpm.yaml)
498-
- name: Update release to github
499-
shell: bash
500-
env:
501-
GH_TOKEN: ${{ github.token }}
502-
# Reference: https://cli.github.com/manual/gh_release_upload
503-
run: |
504-
version="${{ needs.create_release.outputs.version }}"
505-
# name looks like: `databend_0.8.144~nightly_amd64.deb`
506-
gh release upload ${version} databend_*.${{ matrix.packager }} --clobber
430+
version: ${{ needs.create_release.outputs.version }}
431+
arch: ${{ matrix.arch }}
432+
packager: ${{ matrix.packager }}
507433

508434
# bindings_python:
509435
# if: inputs.stable

0 commit comments

Comments
 (0)