Merge remote-tracking branch 'upstream/main' into refactor_append

# Conflicts:
#	src/query/service/src/interpreters/interpreter_copy_into_table.rs
#	src/query/service/src/interpreters/interpreter_replace.rs
#	src/query/service/src/pipelines/builders/builder_copy_into_table.rs
#	src/query/sql/src/executor/physical_plans/physical_copy_into_table.rs
#	src/query/sql/src/planner/binder/copy_into_table.rs
#	src/query/sql/src/planner/plans/copy_into_table.rs
#	src/query/sql/src/planner/plans/insert.rs

Author: SkyFan2002

.config/workspace-schema.json

@@ -0,0 +1,57 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "dependencies": {
+      "type": "object",
+      "additionalProperties": {
+        "type": "object",
+        "properties": {
+          "workspace": {
+            "type": "boolean",
+            "enum": [
+              true
+            ]
+          }
+        },
+        "required": [
+          "workspace"
+        ]
+      }
+    },
+    "dev-dependencies": {
+      "type": "object",
+      "additionalProperties": {
+        "type": "object",
+        "properties": {
+          "workspace": {
+            "type": "boolean",
+            "enum": [
+              true
+            ]
+          }
+        },
+        "required": [
+          "workspace"
+        ]
+      }
+    },
+    "build-dependencies": {
+      "type": "object",
+      "additionalProperties": {
+        "type": "object",
+        "properties": {
+          "workspace": {
+            "type": "boolean",
+            "enum": [
+              true
+            ]
+          }
+        },
+        "required": [
+          "workspace"
+        ]
+      }
+    }
+  }
+}

.github/actions/build_linux/action.yml

@@ -30,7 +30,7 @@ runs:
       uses: ./.github/actions/setup_build_tool
       with:
         target: ${{ inputs.target }}
-        bypass_env_vars: RUSTFLAGS,RUST_LOG
+        bypass_env_vars: RUSTFLAGS,RUST_LOG,DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY
 
     - name: Cross setup
       if: startsWith(inputs.target, 'aarch64-')
@@ -64,49 +64,25 @@ runs:
         target=${{ inputs.target }}
         echo "BUILD_ARCH=${target/-unknown-linux-*}" >> $GITHUB_ENV
 
-    # build all binaries for debug
-    - name: Build Debug
-      if: env.BUILD_PROFILE == 'debug' && inputs.artifacts == 'all'
-      shell: bash
-      run: |
-        artifacts="meta,metactl,metabench,query,sqllogictests"
-        for artifact in ${artifacts//,/ }; do
-          echo "==> building databend-$artifact ..."
-          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --bin databend-$artifact
-        done
-        cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }}
-        ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-*
-
-    - name: Build Debug for specific artifacts
-      if: env.BUILD_PROFILE == 'debug' && inputs.artifacts != 'all'
-      shell: bash
-      run: |
-        artifacts="${{ inputs.artifacts }}"
-        for artifact in ${artifacts//,/ }; do
-          echo "==> building databend-$artifact ..."
-          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --bin databend-$artifact
-        done
-        ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-$artifact
-
     - name: Build Release
-      if: env.BUILD_PROFILE == 'release' && inputs.artifacts == 'all'
+      if: inputs.artifacts == 'all'
       shell: bash
       run: |
         artifacts="meta,metactl,metabench,query,sqllogictests"
         for artifact in ${artifacts//,/ }; do
           echo "==> building databend-$artifact ..."
-          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --release --bin databend-$artifact
+          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --profile ${{ env.BUILD_PROFILE }} --bin databend-$artifact
         done
         ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-$artifact
 
     - name: Build Release for specific artifacts
-      if: env.BUILD_PROFILE == 'release' && inputs.artifacts != 'all'
+      if: inputs.artifacts != 'all'
       shell: bash
       run: |
         artifacts="${{ inputs.artifacts }}"
         for artifact in ${artifacts//,/ }; do
           echo "==> building databend-$artifact ..."
-          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --release --bin databend-$artifact
+          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --profile ${{ env.BUILD_PROFILE }} --bin databend-$artifact
         done
         ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-$artifact
 

.github/actions/build_macos/action.yml

@@ -32,7 +32,6 @@ runs:
         echo "JEMALLOC_SYS_WITH_LG_PAGE=14" >> $GITHUB_ENV
         echo "JEMALLOC_SYS_WITH_MALLOC_CONF=oversize_threshold:0,dirty_decay_ms:5000,muzzy_decay_ms:5000" >> $GITHUB_ENV
 
-
     - name: Setup build env
       shell: bash
       run: |
@@ -41,33 +40,15 @@ runs:
         echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
         echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
 
-    - name: Build Debug for all artifacts
-      if: env.BUILD_PROFILE == 'debug' && inputs.artifacts == 'all'
-      shell: bash
-      run: cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }}
-
-    - name: Build Debug for specific artifacts
-      if: env.BUILD_PROFILE == 'debug' && inputs.artifacts != 'all'
-      shell: bash
-      run: |
-        artifacts="${{ inputs.artifacts }}"
-        echo "==> building libs ..."
-        cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --lib
-        for artifact in ${artifacts//,/ }; do
-          echo "==> building databend-$artifact ..."
-          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --bin databend-$artifact
-        done
-
-    - name: Build Release
-      if: env.BUILD_PROFILE == 'release'
+    - name: Build
       shell: bash
       run: |
         artifacts="${{ inputs.artifacts }}"
         echo "==> building libs ..."
-        cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --release --lib
+        cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --profile ${{ env.BUILD_PROFILE }} --lib
         for artifact in ${artifacts//,/ }; do
           echo "==> building databend-$artifact ..."
-          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --release --bin databend-$artifact
+          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --profile ${{ env.BUILD_PROFILE }} --bin databend-$artifact
         done
 
     - name: Upload artifact

.github/actions/check/action.yml

@@ -38,7 +38,9 @@ runs:
 
     - name: Check toml format
       shell: bash
-      run: taplo fmt --check
+      run: |
+        taplo check
+        taplo fmt --check
 
     - name: Audit dependencies
       shell: bash

.github/actions/setup_license/action.yml

@@ -5,6 +5,10 @@ inputs:
     description: "Self-hosted runner provider, aws or gcp"
     required: true
     default: "aws"
+  type:
+    description: "License type, enterprise or trial"
+    required: true
+    default: "trial"
 
 runs:
   using: "composite"
@@ -13,15 +17,15 @@ runs:
       if: inputs.runner_provider == 'aws'
       shell: bash
       run: |
-        aws s3 cp s3://databend-ci/misc/license.key license.key
-        aws s3 cp s3://databend-ci/misc/license.json license.json
+        aws s3 cp s3://databend-ci/misc/license-${{ inputs.type }}.key license.key
+        aws s3 cp s3://databend-ci/misc/license-${{ inputs.type }}.json license.json
 
     - name: Get License from GCS
       if: inputs.runner_provider == 'gcp'
       shell: bash
       run: |
-        gcloud storage cp gs://databend-ci/misc/license.key license.key
-        gcloud storage cp gs://databend-ci/misc/license.json license.json
+        gcloud storage cp gs://databend-ci/misc/license-${{ inputs.type }}.key license.key
+        gcloud storage cp gs://databend-ci/misc/license-${{ inputs.type }}.json license.json
 
     - name: Output License
       shell: bash

.github/scripts/bump_version.js

@@ -32,9 +32,10 @@ module.exports = async ({ github, context, core }) => {
       core.setFailed("Stable release must be triggered with a nightly tag");
     }
   } else {
+    core.setOutput("sha", context.sha);
     if (TAG) {
       core.setOutput("tag", TAG);
-      core.info(`Release create manually with tag ${TAG}`);
+      core.info(`Release create manually with tag ${TAG} (${context.sha})`);
     } else {
       let releases = await github.rest.repos.listReleases({
         owner: context.repo.owner,
@@ -52,7 +53,6 @@ module.exports = async ({ github, context, core }) => {
       let patch = (parseInt(result[3]) + 1).toString();
       let next_tag = `v${major}.${minor}.${patch}-nightly`;
       core.setOutput("tag", next_tag);
-      core.setOutput("sha", context.sha);
       core.info(`Nightly release ${next_tag} from ${tag} (${context.sha})`);
     }
   }

.github/workflows/bindings.python.yml

@@ -22,7 +22,7 @@ concurrency:
 
 jobs:
   linux:
-    runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 8c16g, aws]
+    runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 4c16g, aws]
     strategy:
       matrix:
         include:

.github/workflows/cloud.yml

@@ -45,7 +45,7 @@ jobs:
 
   build:
     needs: info
-    runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, aws]
+    runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 8c32g, aws]
     strategy:
       matrix:
         include:
@@ -59,6 +59,8 @@ jobs:
       - name: Build Release
         uses: ./.github/actions/build_linux
         timeout-minutes: 60
+        env:
+          DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }}
         with:
           sha: ${{ needs.info.outputs.sha }}
           target: ${{ matrix.arch }}-unknown-linux-gnu
@@ -69,7 +71,7 @@ jobs:
   docker:
     needs: [info, build]
     timeout-minutes: 10
-    runs-on: [self-hosted, X64, Linux, 4c8g, aws]
+    runs-on: [self-hosted, X64, Linux, 2c8g, aws]
     outputs:
       tag: ${{ steps.prepare.outputs.tag }}
     steps:

.github/workflows/dev.yml

@@ -23,7 +23,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Check Source File Changes
-        uses: tj-actions/changed-files@v39
+        uses: tj-actions/changed-files@v45
         id: src
         with:
           files_ignore: |
@@ -50,16 +50,17 @@ jobs:
     uses: ./.github/workflows/reuse.linux.yml
     secrets: inherit
     with:
-      build_profile: debug
+      build_profile: ci
       runner_provider: aws
+      license_type: trial
 
   linux_hive:
     needs: changes
     if: needs.changes.outputs.any_src_changed == 'true'
     uses: ./.github/workflows/reuse.linux.hive.yml
     secrets: inherit
     with:
-      build_profile: debug
+      build_profile: ci
       runner_provider: aws
 
   ready:

.github/workflows/production.yml renamed to .github/workflows/main.yml

@@ -1,10 +1,9 @@
-name: Production
+name: Main
 
 on:
   push:
     branches:
       - main
-  merge_group:
 
 jobs:
   changes:
@@ -16,7 +15,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Check Source File Changes
-        uses: tj-actions/changed-files@v39
+        uses: tj-actions/changed-files@v45
         id: src
         with:
           files_ignore: |
@@ -45,6 +44,7 @@ jobs:
     with:
       build_profile: release
       runner_provider: aws
+      license_type: enterprise
 
   # macos:
   #   needs: linux