Skip to content

Commit e7385db

Browse files
XuanwoXuanwo
committed
Merge remote-tracking branch 'origin/main' into fix/parse-json-path
2 parents 6d25bce + 43bebc1 commit e7385db

File tree

165 files changed

+3992
-1096
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

165 files changed

+3992
-1096
lines changed

‎.github/actions/artifact_upload/action.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ runs:
3434
3535
- name: Upload artifact to github
3636
if: steps.info.outputs.src == 'github'
37-
uses: actions/upload-artifact@v2
37+
uses: actions/upload-artifact@v3
3838
with:
3939
name: ${{ inputs.profile }}-${{ inputs.sha }}-${{ inputs.target }}-${{ inputs.category }}
4040
path: ./target/${{ inputs.target }}/${{ inputs.profile }}/databend-*

‎.github/actions/publish_binary/nfpm.yaml

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
name: "${name}"
2+
arch: "amd64"
3+
platform: "linux"
4+
version: "${version}"
5+
section: "database"
6+
priority: "extra"
7+
maintainer: "Databend Authors <opensource@datafuselabs.com>"
8+
description: |
9+
Databend is a powerful cloud data warehouse. Built for elasticity and efficiency.
10+
Free and open. Also available in the cloud: https://app.databend.com
11+
vendor: "Datafuse Labs"
12+
homepage: "https://databend.rs"
13+
license: "Apache-2.0"
14+
contents:
15+
# Binaries
16+
- src: ${path}/bin/databend-query
17+
dst: /usr/bin/databend-query
18+
- src: ${path}/bin/databend-meta
19+
dst: /usr/bin/databend-meta
20+
- src: ${path}/bin/databend-metactl
21+
dst: /usr/bin/databend-metactl
22+
- src: ${path}/bin/databend-metabench
23+
dst: /usr/bin/databend-metabench
24+
25+
# Configs
26+
- src: ${path}/configs/databend-query.toml
27+
dst: /etc/databend/databend-query.toml
28+
type: config
29+
- src: ${path}/configs/databend-meta.toml
30+
dst: /etc/databend/databend-meta.toml
31+
type: config
32+
33+
# Docs
34+
- src: ${path}/readme.txt
35+
dst: /usr/share/doc/databend/readme.txt
36+
file_info:
37+
mode: 0644

‎.github/workflows/databend-release.yml

Lines changed: 126 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ jobs:
128128
version=${{ needs.create_release.outputs.version }}
129129
sha256sum databend-${version}-${target}.tar.gz >> sha256-${version}-${target}.txt
130130
- name: post sha256
131-
uses: actions/upload-artifact@v2
131+
uses: actions/upload-artifact@v3
132132
with:
133133
name: sha256sums
134134
path: sha256-${{ needs.create_release.outputs.version }}-${{ steps.target.outputs.target }}.txt
@@ -202,7 +202,7 @@ jobs:
202202
version=${{ needs.create_release.outputs.version }}
203203
sha256sum databend-${version}-${target}.tar.gz >> sha256-${version}-${target}.txt
204204
- name: post sha256
205-
uses: actions/upload-artifact@v2
205+
uses: actions/upload-artifact@v3
206206
with:
207207
name: sha256sums
208208
path: sha256-${{ needs.create_release.outputs.version }}-${{ steps.target.outputs.target }}.txt
@@ -215,6 +215,73 @@ jobs:
215215
target: ${{ steps.target.outputs.target }}
216216
repo_role_arn: ${{ secrets.REPO_ROLE_ARN }}
217217

218+
publish_hive:
219+
name: hive assets
220+
runs-on: [self-hosted, X64, Linux, development]
221+
needs: [create_release]
222+
strategy:
223+
fail-fast: false
224+
matrix:
225+
arch:
226+
- x86_64
227+
platform:
228+
- gnu
229+
steps:
230+
- name: Checkout
231+
uses: actions/checkout@v3
232+
with:
233+
fetch-depth: 0
234+
- name: Get the version
235+
id: get_version
236+
run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
237+
- name: Get target
238+
id: target
239+
run: echo 'target=${{ matrix.arch }}-unknown-linux-${{ matrix.platform }}' >> $GITHUB_OUTPUT
240+
- name: Setup Build Tool
241+
uses: ./.github/actions/setup_build_tool
242+
with:
243+
image: ${{ steps.target.outputs.target }}
244+
- name: Build Binary
245+
run: |
246+
cargo build --release --target=${{ steps.target.outputs.target }} --features hive
247+
- name: Copyobj zlib for gnu binaries
248+
if: matrix.platform == 'gnu'
249+
run: |
250+
target=${{ steps.target.outputs.target }}
251+
build-tool /usr/bin/${{ matrix.arch }}-linux-gnu-objcopy --compress-debug-sections=zlib-gnu ./target/${target}/release/databend-query
252+
build-tool /usr/bin/${{ matrix.arch }}-linux-gnu-objcopy --compress-debug-sections=zlib-gnu ./target/${target}/release/databend-meta
253+
build-tool /usr/bin/${{ matrix.arch }}-linux-gnu-objcopy --compress-debug-sections=zlib-gnu ./target/${target}/release/databend-metactl
254+
- name: Pack binaries
255+
run: |
256+
target=${{ steps.target.outputs.target }}
257+
version=${{ needs.create_release.outputs.version }}
258+
mkdir -p release/${target}/{bin,configs}
259+
cp ./target/${target}/release/databend-* release/${target}/bin/
260+
rm -f release/${target}/bin/*.d
261+
cp ./scripts/ci/deploy/config/databend-query-node-1.toml release/${target}/configs/databend-query.toml
262+
cp ./scripts/ci/deploy/config/databend-meta-node-1.toml release/${target}/configs/databend-meta.toml
263+
cp ./.github/actions/publish_binary/databend-release-readme.txt release/${target}/readme.txt
264+
cp -r ./.github/actions/publish_binary/databend-scripts-for-release release/${target}/scripts
265+
tar -C ./release/${target} -czvf databend-hive-${version}-${target}.tar.gz bin configs scripts readme.txt
266+
- name: generate sha256sums
267+
run: |
268+
target=${{ steps.target.outputs.target }}
269+
version=${{ needs.create_release.outputs.version }}
270+
sha256sum databend-hive-${version}-${target}.tar.gz >> sha256-hive-${version}-${target}.txt
271+
- name: post sha256
272+
uses: actions/upload-artifact@v3
273+
with:
274+
name: sha256sums
275+
path: sha256-hive-${{ needs.create_release.outputs.version }}-${{ steps.target.outputs.target }}.txt
276+
retention-days: 1
277+
- name: Publish Binaries
278+
uses: ./.github/actions/publish_binary
279+
with:
280+
github_token: ${{ secrets.GITHUB_TOKEN }}
281+
version: hive-${{ needs.create_release.outputs.version }}
282+
target: ${{ steps.target.outputs.target }}
283+
repo_role_arn: ${{ secrets.REPO_ROLE_ARN }}
284+
218285
publish_sqllogic_testsuites:
219286
name: sqllogic testsuites
220287
runs-on: ubuntu-latest
@@ -346,8 +413,64 @@ jobs:
346413
file: ./docker/${{ matrix.distro }}/${{ matrix.service }}.Dockerfile
347414
build-args: VERSION=${{ needs.create_release.outputs.version }}
348415

416+
release_packages:
417+
name: release packages
418+
runs-on: ubuntu-latest
419+
needs: [create_release, publish_linux]
420+
strategy:
421+
matrix:
422+
arch:
423+
- x86_64
424+
platform:
425+
- musl
426+
packager:
427+
- deb
428+
steps:
429+
- name: Checkout
430+
uses: actions/checkout@v3
431+
- name: Install nfpm@latest
432+
run: curl -o /tmp/nfpm.tar.gz -sSL https://github.com/goreleaser/nfpm/releases/download/v2.22.2/nfpm_2.22.2_Linux_x86_64.tar.gz && cd /tmp && tar xf /tmp/nfpm.tar.gz && sudo mv nfpm /usr/local/bin && sudo chmod a+x /usr/local/bin/nfpm
433+
- name: Get target
434+
id: target
435+
run: echo 'target=${{ matrix.arch }}-unknown-linux-${{ matrix.platform }}' >> $GITHUB_OUTPUT
436+
- name: Download binaries for usage
437+
id: download_binaries
438+
run: |
439+
target=${{ steps.target.outputs.target }}
440+
version="${{ needs.create_release.outputs.version }}"
441+
mkdir -p ./distro/
442+
wget -P distro -qc https://repo.databend.rs/databend/${version}/databend-${version}-${target}.tar.gz
443+
tar x -C distro -f ./distro/databend-${version}-${target}.tar.gz
444+
- name: Build Packages
445+
id: build_packages
446+
run: |
447+
export name="databend"
448+
export version="${{ needs.create_release.outputs.version }}"
449+
export path="distro"
450+
nfpm pkg --packager ${{ matrix.packager }} -f <(envsubst '${name} ${version} ${path}' < .github/actions/publish_binary/nfpm.yaml)
451+
- name: generate sha256sums
452+
run: |
453+
version="${{ needs.create_release.outputs.version }}"
454+
sha256sum databend_*.${{ matrix.packager }} >> sha256-${version}-${{ matrix.packager }}.txt
455+
- name: post sha256
456+
uses: actions/upload-artifact@v3
457+
with:
458+
name: sha256sums
459+
path: sha256-${{ needs.create_release.outputs.version }}-${{ matrix.packager }}.txt
460+
retention-days: 1
461+
- name: Update release to github
462+
shell: bash
463+
# Reference: https://cli.github.com/manual/gh_release_upload
464+
run: |
465+
target=${{ steps.target.outputs.target }}
466+
version="${{ needs.create_release.outputs.version }}"
467+
# name looks like: `databend_0.8.144~nightly_amd64.deb`
468+
gh release upload ${version} databend_*.${{ matrix.packager }} --clobber
469+
env:
470+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
471+
349472
upload-sha256sums-plugin:
350-
needs: [create_release, publish_linux, publish_macos]
473+
needs: [create_release, publish_linux, publish_macos, release_packages]
351474
runs-on: ubuntu-latest
352475
name: upload-sha256sums
353476
steps:

‎.github/workflows/production.yml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,15 @@ jobs:
6464
target: ${{ matrix.target }}
6565
profile: release
6666

67+
build_hive:
68+
runs-on: [self-hosted, X64, Linux, development]
69+
steps:
70+
- uses: actions/checkout@v3
71+
- uses: ./.github/actions/build_linux_hive
72+
with:
73+
target: x86_64-unknown-linux-gnu
74+
profile: release
75+
6776
test_unit:
6877
timeout-minutes: 30
6978
runs-on: [self-hosted, X64, Linux]

‎Cargo.lock

Lines changed: 33 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ members = [
5858
"src/query/storages/stage",
5959
"src/query/storages/system",
6060
"src/query/storages/view",
61+
"src/query/storages/parquet",
6162
"src/query/users",
6263
# databend-query
6364
"src/query/service",

‎docs/doc/10-deploy/07-query/10-query-config.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -271,7 +271,7 @@ clickhouse_handler_port = 9001
271271

272272
# Query HTTP Handler.
273273
http_handler_host = "0.0.0.0"
274-
http_handler_port = 8081
274+
http_handler_port = 8000
275275

276276
tenant_id = "tenant1"
277277
cluster_id = "cluster1"

‎docs/doc/11-integrations/00-api/03-streaming-load.md

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,14 @@ curl -H "<parameter>:<value>" [-H "<parameter>:<value>"...] -F "upload=@<file_l
2020

2121
The request usually includes many occurrences of the argument `-H` and each is followed by one of the following parameters to tell Databend how to handle the file you're loading data from. Please note that `insert_sql` is required, and other parameters are optional.
2222

23-
| Parameter | Values | Supported Formats | Examples |
24-
|-------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------|------------------------------------------------|
25-
| insert_sql | [INSERT_statement] + format [file_format] | All | -H "insert_sql: insert into ontime format CSV" |
26-
| format_skip_header | Tells Databend how many lines at the beginning of the file to skip for header.<br /> 0 (default): No lines to skip;<br /> 1: Skip the first line;<br /> N: Skip the first N lines. | CSV / TSV / NDJSON | -H "format_skip_header: 1" |
27-
| format_compression | Tells Databend the compression format of the file.<br /> NONE (default): Do NOT decompress the file;<br /> AUTO: Automatically decompress the file by suffix;<br /> You can also use one of these values to explicitly specify the compression format: GZIP \| BZ2 \| BROTLI \| ZSTD \| DEFALTE \| RAW_DEFLATE. | CSV / TSV / NDJSON | -H "format_compression:auto" |
28-
| format_field_delimiter | Tells Databend the characters used in the file to separate fields.<br /> Default for CSV files: `,`.<br /> Default for TSV files: `\t`. | CSV / TSV | -H "format_field_delimiter:," |
29-
| format_record_delimiter | Tells Databend the new line characters used in the file to separate records.<br /> Default: `\n`. | CSV / TSV | -H "format_recorder_delimiter:\n" |
30-
| format_quote | Tells Databend the quote characters for strings in CSV file.<br /> Default: ""(Double quotes). | CSV | |
23+
| Parameter | Values | Supported Formats | Examples |
24+
|-------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------|---------------------------------------------------------------------------------------------------------------------------------------|
25+
| insert_sql | [INSERT_statement] + format [file_format] | All | -H "insert_sql: insert into ontime format CSV" |
26+
| format_skip_header | Tells Databend how many lines at the beginning of the file to skip for header.<br /> 0 (default): No lines to skip;<br /> 1: Skip the first line;<br /> N: Skip the first N lines. | CSV / TSV / NDJSON | -H "format_skip_header: 1" |
27+
| format_compression | Tells Databend the compression format of the file.<br /> NONE (default): Do NOT decompress the file;<br /> AUTO: Automatically decompress the file by suffix;<br /> You can also use one of these values to explicitly specify the compression format: GZIP \ | BZ2 \| BROTLI \ | ZSTD \| DEFALTE \| RAW_DEFLATE. | CSV / TSV / NDJSON | -H "format_compression:auto" |
28+
| format_field_delimiter | Tells Databend the characters used in the file to separate fields.<br /> Default for CSV files: `,`.<br /> Default for TSV files: `\t`.<br /> Hive output files using [SOH control character (\x01)]( https://en.wikipedia.org/wiki/C0_and_C1_control_codes#SOH) as the field delimiter. | CSV / TSV | -H "format_field_delimiter:,". |
29+
| format_record_delimiter | Tells Databend the new line characters used in the file to separate records.<br /> Default: `\n`. | CSV / TSV | -H "format_recorder_delimiter:\n" |
30+
| format_quote | Tells Databend the quote characters for strings in CSV file.<br /> Default: ""(Double quotes). | CSV | |
3131

3232
## Alternatives to Streaming Load API
3333

0 commit comments

Comments
 (0)