Merge branch 'fix/limit_with_cluster' of github.com:zhang2014/datafuse into fix/limit_with_cluster

Author: zhang2014

.github/actions/test_sqllogic_cluster_linux/action.yml

@@ -23,12 +23,13 @@ runs:
         sha: ${{ github.sha }}
         target: ${{ inputs.target }}
 
-    - name: Test setup
-      shell: bash
-      run: |
-        bash ./scripts/setup/dev_setup.sh -yd
-
     - name: Run sqllogic Tests with Cluster mode
       shell: bash
       run: |
-        bash ./scripts/ci/ci-run-sqllogic-tests-cluster.sh ${{ inputs.dirs }}
+        docker run --rm --tty --net=host \
+          --user $(id -u):$(id -g) \
+          --volume "${PWD}:/workspace" \
+          --workdir "/workspace" \
+          datafuselabs/build-tool:sqllogic \
+          bash ./scripts/ci/ci-run-sqllogic-tests-cluster.sh \
+          ${{ inputs.dirs }}

.github/actions/test_sqllogic_standalone_linux/action.yml

@@ -23,18 +23,13 @@ runs:
         sha: ${{ github.sha }}
         target: ${{ inputs.target }}
 
-    - name: Test setup
-      shell: bash
-      run: |
-        bash ./scripts/setup/dev_setup.sh -yd
-
     - name: Run sqllogic Tests with Standalone mode with embedded meta-store
       shell: bash
       run: |
-        bash ./scripts/ci/ci-run-sqllogic-tests.sh ${{ inputs.dirs }}
-
-    # - name: Upload failure
-    #   if: failure()
-    #   uses: ./.github/actions/artifact_failure
-    #   with:
-    #     name: test-sqllogic-standalone-linux
+        docker run --rm --tty --net=host \
+          --user $(id -u):$(id -g) \
+          --volume "${PWD}:/workspace" \
+          --workdir "/workspace" \
+          datafuselabs/build-tool:sqllogic \
+          bash ./scripts/ci/ci-run-sqllogic-tests.sh \
+          ${{ inputs.dirs }}

.github/workflows/build-sqllogic.yml

@@ -0,0 +1,38 @@
+name: Build Sqllogic Test Image
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "tests/logictest/**"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: ./.github/actions/setup_docker
+        id: login
+        with:
+          repo: build-tool
+          ecr_role_arn: ${{ secrets.ECR_ROLE_ARN }}
+          dockerhub_user: ${{ secrets.DOCKERHUB_USERNAME }}
+          dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v3
+        with:
+          push: true
+          tags: |
+            ${{ steps.login.outputs.dockerhub_repo }}:sqllogic
+            ${{ steps.login.outputs.ecr_repo }}:sqllogic
+          platforms: linux/amd64
+          context: ./tests/logictest/
+          file: ./tests/logictest/Dockerfile

.github/workflows/databend-release.yml

@@ -191,6 +191,24 @@ jobs:
           target: ${{ steps.target.outputs.target }}
           repo_role_arn: ${{ secrets.REPO_ROLE_ARN }}
 
+  publish_sqllogic_testsuites:
+    name: sqllogic testsuites
+    runs-on: ubuntu-latest
+    needs: [create_release]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Get the version
+        id: get_version
+        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
+      - name: Upload to github release
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          version=${{ needs.create_release.outputs.version }}
+          tar -C ./tests/logictest -czvf testsuites-${version}.tar.gz suites
+          gh release upload ${version} testsuites-${version}.tar.gz --clobber
+
   release_docker_combined:
     name: docker image combined
     runs-on: ubuntu-latest
@@ -305,34 +323,3 @@ jobs:
           context: .
           file: ./docker/${{ matrix.distro }}/${{ matrix.service }}.Dockerfile
           build-args: VERSION=${{ needs.create_release.outputs.version }}
-
-  release_logic_test_docker_image:
-    name: release logic test docker image
-    runs-on: ubuntu-latest
-    needs: [create_release]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - uses: ./.github/actions/setup_docker
-        id: login
-        with:
-          repo: sqllogictest
-          ecr_role_arn: ${{ secrets.ECR_ROLE_ARN }}
-          dockerhub_user: ${{ secrets.DOCKERHUB_USERNAME }}
-          dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Build and push
-        id: docker_build
-        uses: docker/build-push-action@v3
-        with:
-          push: true
-          tags: |
-            ${{ steps.login.outputs.dockerhub_repo }}:latest
-            ${{ steps.login.outputs.dockerhub_repo }}:${{ needs.create_release.outputs.version }}
-            ${{ steps.login.outputs.ecr_repo }}:latest
-            ${{ steps.login.outputs.ecr_repo }}:${{ needs.create_release.outputs.version }}
-          platforms: linux/amd64
-          context: ./tests/logictest/
-          file: ./tests/logictest/Dockerfile
-          build-args: VERSION=${{ needs.create_release.outputs.version }}

Cargo.lock

@@ -0,0 +1,56 @@
+---
+title: Idempotent Copy
+description: Avoid duplicating when copy stage files into a table
+---
+
+- Tracking Issue: https://github.com/datafuselabs/databend/issues/6338
+
+## Summary
+
+When streaming copy stage files into a table, there is a chance that some files have already been copied, So it needs some ways to avoid duplicate copying files, make it an `idempotent` operation.
+
+## Save copy into table stage files meta information in meta service
+
+Whenever copy stage files into a table, save the stage file meta information into the meta service:
+
+- key: combined with `(tenant, database, table, file name)`.
+- value: value MUST includes all the meta of a stage file, such as `content-length`,`etag`,`last modified`.
+
+
+
+![](/img/rfc/20220909-idempotent-copy/stage-file-meta.png)
+
+
+
+The expiration time of the stage file meta information is 64 days by default.
+
+## Avoiding duplicates when copy stage files into a table
+
+Using the stage file meta information, whenever copy stage files into a table, follow these steps:
+
+* First, get all the table file meta information of the copy stage files that want to copy into the table(if any).
+* Second, get all the stage file meta information.
+* Third, compare the table file meta information with stage file meta information:
+  * If they matched, this file is just ignored without copying.
+  * Else, copy the stage file and up-insert into the table stage file meta.
+
+
+
+![](/img/rfc/20220909-idempotent-copy/example.png)
+
+
+
+Take the image above as an example:
+
+* Client make a request to copy thress files (file1, file2, file3) into table.
+
+* Get the table stage file meta of (file1, file2, file3).
+
+* In the meta service, only found (file1,file3) stage file information.
+
+* Compare the table stage file information with stage file information, and found that file1 has not been changed, so file1 will be ignored in this copy operation, and (file2,file3) will be copied.
+
+* After copying new files, (file2, file3) stage file information will be saved into table file information.
+
+
+

docs/doc/60-contributing/03-rfcs/20220909-idempotent-copy.md

@@ -33,8 +33,8 @@ common-grpc = { path = "../common/grpc" }
 common-macros = { path = "../common/macros" }
 common-meta-api = { path = "../meta/api" }
 common-meta-app = { path = "../meta/app" }
+common-meta-client = { path = "../meta/client" }
 common-meta-embedded = { path = "../meta/embedded" }
-common-meta-grpc = { path = "../meta/grpc" }
 common-meta-raft-store = { path = "../meta/raft-store" }
 common-meta-sled-store = { path = "../meta/sled-store" }
 common-meta-store = { path = "../meta/store" }