Merge branch 'main' into refactor/interpreter_2

Author: zhang2014

.github/actions/fuse_compat/action.yml

@@ -12,9 +12,6 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Setup Build Tool
-      uses: ./.github/actions/setup_build_tool
-
     - name: Download artifact
       uses: ./.github/actions/artifact_download
       with:
@@ -26,8 +23,20 @@ runs:
     - name: Test compatibility
       shell: bash
       run: |
-        build-tool bash ./tests/fuse-compat/test-fuse-compat.sh 0.7.150
-        build-tool bash ./tests/fuse-compat/test-fuse-compat.sh 0.7.151
+        docker run --rm --tty --net=host \
+          --user $(id -u):$(id -g) \
+          --env BUILD_PROFILE \
+          --volume "${PWD}:/workspace" \
+          --workdir "/workspace" \
+          datafuselabs/build-tool:sqllogic \
+          bash ./tests/fuse-compat/test-fuse-compat.sh 0.7.150
+        docker run --rm --tty --net=host \
+          --user $(id -u):$(id -g) \
+          --env BUILD_PROFILE \
+          --volume "${PWD}:/workspace" \
+          --workdir "/workspace" \
+          datafuselabs/build-tool:sqllogic \
+          bash ./tests/fuse-compat/test-fuse-compat.sh 0.7.151
 
     - name: Upload failure
       if: failure()

.github/actions/test_sqllogic_standalone_linux/action.yml

@@ -23,7 +23,7 @@ runs:
         sha: ${{ github.sha }}
         target: ${{ inputs.target }}
 
-    - name: Run sqllogic Tests with Standalone mode with embedded meta-store
+    - name: Run sqllogic Tests with Standalone mode
       shell: bash
       run: |
         docker run --rm --tty --net=host \

.github/workflows/databend-release.yml

@@ -306,11 +306,13 @@ jobs:
         id: get_image_tags
         shell: bash
         run: |
-          dockerhub_base=${{ steps.login.outputs.dockerhub_repo }}:${{ needs.create_release.outputs.version }}
-          ecr_base=${{ steps.login.outputs.ecr_repo }}:${{ needs.create_release.outputs.version }}
-          _tags="${dockerhub_base}-${{ matrix.distro }},${ecr_base}-${{ matrix.distro }}"
+          _tags="${{ steps.login.outputs.dockerhub_repo }}:${{ needs.create_release.outputs.version }}-${{ matrix.distro }}"
+          _tags="${_tags},${{ steps.login.outputs.ecr_repo }}:${{ needs.create_release.outputs.version }}-${{ matrix.distro }}"
           if [[ "${{ matrix.distro }}" == "debian" ]]; then
-            _tags="${_tags},${dockerhub_base},${ecr_base}"
+            _tags="${_tags},${{ steps.login.outputs.dockerhub_repo }}:${{ needs.create_release.outputs.version }}"
+            _tags="${_tags},${{ steps.login.outputs.ecr_repo }}:${{ needs.create_release.outputs.version }}"
+            _tags="${_tags},${{ steps.login.outputs.dockerhub_repo }}:latest"
+            _tags="${_tags},${{ steps.login.outputs.ecr_repo }}:latest"
           fi
           echo ::set-output name=IMAGE_TAGS::${_tags}
 

.github/workflows/dev-linux.yml

@@ -129,7 +129,7 @@ jobs:
       - uses: ./.github/actions/test_stateless_cluster_linux
 
   test_sqllogic_standalone_linux:
-    timeout-minutes: 15
+    timeout-minutes: 30
     name: test_sqllogic_${{ matrix.dirs }}_standalone_linux
     runs-on: [self-hosted, X64, Linux, development]
     needs: build_gnu

Cargo.lock

@@ -200,7 +200,8 @@ Databend is licensed under [Apache 2.0](LICENSE).
 
 ## Acknowledgement
 
-- Databend is inspired by [ClickHouse](https://github.com/clickhouse/clickhouse) and [Snowflake](https://docs.snowflake.com/en/user-guide/intro-key-concepts.html#snowflake-architecture), its computing model is based on [apache-arrow](https://arrow.apache.org/).
+- Databend is inspired by [ClickHouse](https://github.com/clickhouse/clickhouse) and [Snowflake](https://docs.snowflake.com/en/user-guide/intro-key-concepts.html#snowflake-architecture).
+- Databend's computing model is based on [Arrow2](https://github.com/jorgecarleitao/arrow2), Arrow2 is a faster and safer implementation of the Apache Arrow Columnar Format.
 - The [documentation website](https://databend.rs) is hosted by [Vercel](https://vercel.com/?utm_source=databend&utm_campaign=oss).
 - Thanks to [Mergify](https://mergify.com/) for sponsoring advanced features like Batch Merge.
 - Thanks to [QingCloud](https://qingcloud.com) for sponsoring CI resources.

README.md

@@ -0,0 +1,29 @@
+---
+title: FUSE_BLOCK
+---
+
+Returns the block information of a snapshot of table.
+
+## Syntax
+
+```sql
+FUSE_BLOCK('<database_name>', '<table_name>'[, '<snapshot_id>'])
+```
+
+## Examples
+
+```sql
+CREATE TABLE mytable(c int);
+INSERT INTO mytable values(1);
+INSERT INTO mytable values(2); 
+
+SELECT * FROM FUSE_BLOCK('default', 'mytable');
+
+---
++----------------------------------+----------------------------+----------------------------------------------------+------------+----------------------------------------------------+-------------------+
+| snapshot_id                      | timestamp                  | block_location                                     | block_size | bloom_filter_location                              | bloom_filter_size |
++----------------------------------+----------------------------+----------------------------------------------------+------------+----------------------------------------------------+-------------------+
+| 51e84b56458f44269b05a059b364a659 | 2022-09-15 07:14:14.137268 | 1/7/_b/39a6dbbfd9b44ad5a8ec8ab264c93cf5_v0.parquet |          4 | 1/7/_i/39a6dbbfd9b44ad5a8ec8ab264c93cf5_v1.parquet |               221 |
+| 51e84b56458f44269b05a059b364a659 | 2022-09-15 07:14:14.137268 | 1/7/_b/d0ee9688c4d24d6da86acd8b0d6f4fad_v0.parquet |          4 | 1/7/_i/d0ee9688c4d24d6da86acd8b0d6f4fad_v1.parquet |               219 |
++----------------------------------+----------------------------+----------------------------------------------------+------------+----------------------------------------------------+-------------------+
+```

docs/doc/30-reference/20-functions/111-system-functions/fuse_block.md

@@ -224,6 +224,16 @@ COPY INTO mytable
   FROM 's3://mybucket/data.csv'
   FILE_FORMAT = (type = 'CSV' field_delimiter = ','  record_delimiter = '\n' skip_header = 1) size_limit=10;
 ```
+  
+`Parquet` file example:
+```sql
+COPY INTO mytable
+  FROM 's3://mybucket/data.parquet'
+  CONNECTION = (
+      ACCESS_KEY_ID = '<your-access-key-ID>'
+      SECRET_ACCESS_KEY = '<your-secret-access-key>')
+  FILE_FORMAT = (type = 'PARQUET');
+```
 
 **Azure Blob storage**
 
@@ -236,4 +246,4 @@ COPY INTO mytable
         ACCOUNT_NAME = '<account_name>'
         ACCOUNT_KEY = '<account_key>'
     )
-```
+```

docs/doc/30-reference/30-sql/10-dml/dml-copy-into-table.md

@@ -12,41 +12,51 @@ This is Databend Roadmap 2022 :rocket:, sync from the [#3706](https://github.com
 
 # Main tasks
 
-### 1. Query
+ Roadmap 2021: https://github.com/datafuselabs/databend/issues/746
+
+# Main tasks
+
+### 1. Query 
 
 
 | Task                                         | Status    | Release Target | Comments        | 
 | ----------------------------------------------- | --------- | -------------- | --------------- | 
-| [Query Cluster Track #747](https://github.com/datafuselabs/databend/issues/747) | PROGRESS  |   |  |
-| [RBAC Privileges #2793](https://github.com/datafuselabs/databend/issues/2793) | PROGRESS  |   |   |
-| [ New Planner Framework #1217](https://github.com/datafuselabs/databend/issues/1218)| PROGRESS  | | [RFC](https://databend.rs/doc/contributing/rfcs/new-sql-planner-framework)|
-| [ Database Sharing #3430](https://github.com/datafuselabs/databend/issues/3430)| PROGRESS  |     | |
-| [ STAGE Command #2976](https://github.com/datafuselabs/databend/issues/2976)| PROGRESS  |     | |
-| [ COPY Command #4104](https://github.com/datafuselabs/databend/issues/4104)| PROGRESS  |     | |
+| [Query Cluster Track #747](https://github.com/datafuselabs/databend/issues/747) | DONE  |   |  |
+| [RBAC Privileges #2793](https://github.com/datafuselabs/databend/issues/2793) | DONE  |   |   |
+| [ New Planner Framework #1217](https://github.com/datafuselabs/databend/issues/1218)| DONE  | | [RFC](https://databend.rs/doc/contributing/rfcs/new-sql-planner-framework)|
+| [ Database Sharing #3430](https://github.com/datafuselabs/databend/issues/3430)| DONE  |     | |
+| [ STAGE Command #2976](https://github.com/datafuselabs/databend/issues/2976)| DONE  |     | |
+| [ COPY Command #4104](https://github.com/datafuselabs/databend/issues/4104)| DONE  |     | |
 | [Index Design #3711](https://github.com/datafuselabs/databend/issues/3711) | PROGRESS  |   |   |
-| [Push-Based + Pull-Based processor](https://github.com/datafuselabs/databend/issues/3379)| PROGRESS  |   |  |
-| [Semi-structured Data Types #3916](https://github.com/datafuselabs/databend/issues/3916) | PROGRESS  |   |   |
+| [Push-Based + Pull-Based processor](https://github.com/datafuselabs/databend/issues/3379)| DONE  |   |  |
+| [Semi-structured Data Types #3916](https://github.com/datafuselabs/databend/issues/3916) | DONE  |   |   |
+| [Table Cluster Key #4268](https://github.com/datafuselabs/databend/issues/4268) | DONE  |   |   |
+| Transactions | DONE  |   |  |
 | [Support Fulltext Index #3915](https://github.com/datafuselabs/databend/issues/3915) | PLANNING  |   |   |
-| [Table Cluster Key #4268](https://github.com/datafuselabs/databend/issues/4268) | PLANNING  |   |   |
-| Tansactions | PLANNING  |   |  |
-| Window Functions | PLANNING  |   |  |
-| Lambda Functions | PLANNING  |   |  |
-| Array Functions | PLANNING  |   |  |
+| [Hive External Data Source #4826](https://github.com/datafuselabs/databend/issues/4826) | DONE  |   |   |
+| [Window Functions](https://github.com/datafuselabs/databend/issues/4653) | DONE  |   |  |
+| Lambda Functions | DONE  |   |  |
+| Array Functions | DONE  |   |  |
 | Compile Aggregate Functions(JIT) | PLANNING  |   | |
-| Common Table Expressions | PLANNING  |   | [MySQL CTE](https://dev.mysql.com/doc/refman/8.0/en/with.html#common-table-expressions) |
-| External  Cache | PLANNING  |   | |
+| [Common Table Expressions #6246](https://github.com/datafuselabs/databend/issues/6246) | DONE  |   | [MySQL CTE](https://dev.mysql.com/doc/refman/8.0/en/with.html#common-table-expressions) |
+| [External  Cache](https://github.com/datafuselabs/databend/issues/6786) #6786 | PROGRESS  |   | |
 | External  Table | PLANNING  |   | [Snowflake ET](https://docs.snowflake.com/en/sql-reference/sql/create-external-table.html)|
-| Update&Delete | PLANNING  |   | |
+| Delete | DONE  |   | |
+| Update | PROGRESS  |   | |
 | Streaming Ingestion  | PLANNING  |   | |
-| Streaming Analytics  | PLANNING  |   | |
+| [Resource Quota](https://github.com/datafuselabs/databend/issues/6935)  | PROGRESS  |   | |
+| [LakeHouse](https://github.com/datafuselabs/databend/issues/7592)  | PROGRESS  |   | v0.9|
 
 
 ### 2. Testing
 
 | Task                                         | Status    | Release Target | Comments        | 
 | ----------------------------------------------- | --------- | -------------- | --------------- | 
-| [ Continuous Benchmarking #3084](https://github.com/datafuselabs/databend/issues/3084) | PROGRESS  |    | |
+| [ Continuous Benchmarking #3084](https://github.com/datafuselabs/databend/issues/3084) | DONE  |    | https://perf.databend.rs |
+
 
 # Releases
-- [x] #2525
-- [x] #2257 
+- [x] [Release proposal: Nightly v0.8 #4591](https://github.com/datafuselabs/databend/issues/4591) 
+- [x] [Release proposal: Nightly v0.7 #3428](https://github.com/datafuselabs/databend/issues/3428) 
+- [x] [Release proposal: Nightly v0.6 #2525](https://github.com/datafuselabs/databend/issues/2525)
+- [x] [Release proposal: Nightly v0.5 #2257](https://github.com/datafuselabs/databend/issues/2257) 

docs/doc/60-contributing/02-roadmap.md

@@ -76,6 +76,8 @@ To support creating and managing shares, Databend provides the following set of
 - Describe Share
 - Show Grants
 - Create Database, Table FROM Share
+- Select data from the Share DB tables
+- Show shared database tables
 
 ### Create Share
 
@@ -210,6 +212,33 @@ Syntax:
 CREATE DATABASE <name> FROM SHARE <provider_tenant>.<share_name>
 ```
 
+
+
+### Select data from the Share DB tables
+
+After tenants have created a database from a shared database, tenants can select data from the shared table like a normal table, before that the providers MUST have permitted access permission to the shared database and table.
+
+Syntax:
+
+```sql
+Select * from <share_db_name>.<table_name>
+```
+
+
+
+### Show shared database tables
+
+After tenants have created a database from a shared database,  tenants can show tables from the shared database, it only outputs the tables which have been permitted to access.
+
+ Syntax:
+
+```sql
+use <share_db_name>;
+show tables;
+```
+
+
+
 ## Example of Using Share with SQL
 
 To create a share using SQL: