Merge branch 'main' into ISSUE-7630

Author: mergify[bot]

.github/actions/fuse_compat/action.yml

@@ -12,9 +12,6 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Setup Build Tool
-      uses: ./.github/actions/setup_build_tool
-
     - name: Download artifact
       uses: ./.github/actions/artifact_download
       with:
@@ -26,8 +23,20 @@ runs:
     - name: Test compatibility
       shell: bash
       run: |
-        build-tool bash ./tests/fuse-compat/test-fuse-compat.sh 0.7.150
-        build-tool bash ./tests/fuse-compat/test-fuse-compat.sh 0.7.151
+        docker run --rm --tty --net=host \
+          --user $(id -u):$(id -g) \
+          --env BUILD_PROFILE \
+          --volume "${PWD}:/workspace" \
+          --workdir "/workspace" \
+          datafuselabs/build-tool:sqllogic \
+          bash ./tests/fuse-compat/test-fuse-compat.sh 0.7.150
+        docker run --rm --tty --net=host \
+          --user $(id -u):$(id -g) \
+          --env BUILD_PROFILE \
+          --volume "${PWD}:/workspace" \
+          --workdir "/workspace" \
+          datafuselabs/build-tool:sqllogic \
+          bash ./tests/fuse-compat/test-fuse-compat.sh 0.7.151
 
     - name: Upload failure
       if: failure()

Cargo.lock

@@ -34,7 +34,7 @@ members = [
     "src/query/pipeline/sinks",
     "src/query/pipeline/sources",
     "src/query/pipeline/transforms",
-    "src/query/planners",
+    "src/query/legacy-planners",
     "src/query/settings",
     "src/query/storages/fuse",
     "src/query/storages/fuse-meta",

Cargo.toml

@@ -24,8 +24,8 @@ SUBSTRING(str FROM pos FOR len)
 | Arguments   | Description |
 | ----------- | ----------- |
 | str | The main string from where the character to be extracted |
-| pos | The one-indexed position expression to start at. If negative, counts from the end |
-| len | The number expression of characters to extract |
+| pos | The position (starting from 1) the substring to start at. If negative, counts from the end |
+| len | The maximun length of the substring to extract |
 
 ## Return Type
 

docs/doc/30-reference/20-functions/40-string-functions/substring.md

@@ -1,7 +1,3 @@
 {
-  "label": "Cluster Key",
-  "link": {
-    "type": "generated-index",
-    "slug": "/reference/sql/ddl/clusterkey"
-  }
+  "label": "Cluster Key"
 }

docs/doc/30-reference/30-sql/00-ddl/70-clusterkey/_category_.json

@@ -0,0 +1,15 @@
+---
+title: What is a Cluster Key?
+sidebar_position: 1
+slug: ./
+---
+
+The cluster key is a data object for tables in Databend. It explicitly tells Databend how to divide and group rows of a table into the storage partitions rather than using the data ingestion order. 
+
+A table's cluster key is usually one or more columns or expressions. If you define a cluster key for a table, Databend reorganizes your data based on the cluster key and stores similar rows into the same or adjacent storage partitions.
+
+The benefit of defining a cluster key is optimizing the query performance.   The cluster key acts as a link between the metadata in the Databend's Meta Service Layer and the storage partitions. After the cluster key is defined for a table, the table's metadata implements a key-value-like list that shows the correspondences between the column or expression values and their storage partitions. When a query comes, Databend can quickly locate the storage partition by the metadata and fetch the results. To make this work, the cluster key you set must match the way how you filter the data in queries. For example, if you're most likely to query a table that holds all the employees' profile information by their first names, set the cluster key to the first name column.
+
+In Databend, you [SET CLUSTER KEY](dml-set-cluster-key.md) when you create a table, and you can [ALTER CLUSTER KEY](https://databend.rs/doc/reference/sql/ddl/clusterkey/dml-alter-cluster-key) if necessary. A fully-clustered table might become chaotic if it continues to have ingestion or Data Manipulation Language operations (such as INSERT, UPDATE, DELETE), you will need to [RECLUSTER TABLE](./dml-recluster-table.md) to fix the chaos.
+
+It's important to note that, most of the time you do not need to set the cluster key. Clustering or re-clustering a table consumes time and your credits if you're in Databend Cloud. Databend recommends setting cluster keys for large tables with slow query issues.

docs/doc/30-reference/30-sql/00-ddl/70-clusterkey/index.md

@@ -12,41 +12,51 @@ This is Databend Roadmap 2022 :rocket:, sync from the [#3706](https://github.com
 
 # Main tasks
 
-### 1. Query
+ Roadmap 2021: https://github.com/datafuselabs/databend/issues/746
+
+# Main tasks
+
+### 1. Query 
 
 
 | Task                                         | Status    | Release Target | Comments        | 
 | ----------------------------------------------- | --------- | -------------- | --------------- | 
-| [Query Cluster Track #747](https://github.com/datafuselabs/databend/issues/747) | PROGRESS  |   |  |
-| [RBAC Privileges #2793](https://github.com/datafuselabs/databend/issues/2793) | PROGRESS  |   |   |
-| [ New Planner Framework #1217](https://github.com/datafuselabs/databend/issues/1218)| PROGRESS  | | [RFC](https://databend.rs/doc/contributing/rfcs/new-sql-planner-framework)|
-| [ Database Sharing #3430](https://github.com/datafuselabs/databend/issues/3430)| PROGRESS  |     | |
-| [ STAGE Command #2976](https://github.com/datafuselabs/databend/issues/2976)| PROGRESS  |     | |
-| [ COPY Command #4104](https://github.com/datafuselabs/databend/issues/4104)| PROGRESS  |     | |
+| [Query Cluster Track #747](https://github.com/datafuselabs/databend/issues/747) | DONE  |   |  |
+| [RBAC Privileges #2793](https://github.com/datafuselabs/databend/issues/2793) | DONE  |   |   |
+| [ New Planner Framework #1217](https://github.com/datafuselabs/databend/issues/1218)| DONE  | | [RFC](https://databend.rs/doc/contributing/rfcs/new-sql-planner-framework)|
+| [ Database Sharing #3430](https://github.com/datafuselabs/databend/issues/3430)| DONE  |     | |
+| [ STAGE Command #2976](https://github.com/datafuselabs/databend/issues/2976)| DONE  |     | |
+| [ COPY Command #4104](https://github.com/datafuselabs/databend/issues/4104)| DONE  |     | |
 | [Index Design #3711](https://github.com/datafuselabs/databend/issues/3711) | PROGRESS  |   |   |
-| [Push-Based + Pull-Based processor](https://github.com/datafuselabs/databend/issues/3379)| PROGRESS  |   |  |
-| [Semi-structured Data Types #3916](https://github.com/datafuselabs/databend/issues/3916) | PROGRESS  |   |   |
+| [Push-Based + Pull-Based processor](https://github.com/datafuselabs/databend/issues/3379)| DONE  |   |  |
+| [Semi-structured Data Types #3916](https://github.com/datafuselabs/databend/issues/3916) | DONE  |   |   |
+| [Table Cluster Key #4268](https://github.com/datafuselabs/databend/issues/4268) | DONE  |   |   |
+| Transactions | DONE  |   |  |
 | [Support Fulltext Index #3915](https://github.com/datafuselabs/databend/issues/3915) | PLANNING  |   |   |
-| [Table Cluster Key #4268](https://github.com/datafuselabs/databend/issues/4268) | PLANNING  |   |   |
-| Tansactions | PLANNING  |   |  |
-| Window Functions | PLANNING  |   |  |
-| Lambda Functions | PLANNING  |   |  |
-| Array Functions | PLANNING  |   |  |
+| [Hive External Data Source #4826](https://github.com/datafuselabs/databend/issues/4826) | DONE  |   |   |
+| [Window Functions](https://github.com/datafuselabs/databend/issues/4653) | DONE  |   |  |
+| Lambda Functions | DONE  |   |  |
+| Array Functions | DONE  |   |  |
 | Compile Aggregate Functions(JIT) | PLANNING  |   | |
-| Common Table Expressions | PLANNING  |   | [MySQL CTE](https://dev.mysql.com/doc/refman/8.0/en/with.html#common-table-expressions) |
-| External  Cache | PLANNING  |   | |
+| [Common Table Expressions #6246](https://github.com/datafuselabs/databend/issues/6246) | DONE  |   | [MySQL CTE](https://dev.mysql.com/doc/refman/8.0/en/with.html#common-table-expressions) |
+| [External  Cache](https://github.com/datafuselabs/databend/issues/6786) #6786 | PROGRESS  |   | |
 | External  Table | PLANNING  |   | [Snowflake ET](https://docs.snowflake.com/en/sql-reference/sql/create-external-table.html)|
-| Update&Delete | PLANNING  |   | |
+| Delete | DONE  |   | |
+| Update | PROGRESS  |   | |
 | Streaming Ingestion  | PLANNING  |   | |
-| Streaming Analytics  | PLANNING  |   | |
+| [Resource Quota](https://github.com/datafuselabs/databend/issues/6935)  | PROGRESS  |   | |
+| [LakeHouse](https://github.com/datafuselabs/databend/issues/7592)  | PROGRESS  |   | v0.9|
 
 
 ### 2. Testing
 
 | Task                                         | Status    | Release Target | Comments        | 
 | ----------------------------------------------- | --------- | -------------- | --------------- | 
-| [ Continuous Benchmarking #3084](https://github.com/datafuselabs/databend/issues/3084) | PROGRESS  |    | |
+| [ Continuous Benchmarking #3084](https://github.com/datafuselabs/databend/issues/3084) | DONE  |    | https://perf.databend.rs |
+
 
 # Releases
-- [x] #2525
-- [x] #2257 
+- [x] [Release proposal: Nightly v0.8 #4591](https://github.com/datafuselabs/databend/issues/4591) 
+- [x] [Release proposal: Nightly v0.7 #3428](https://github.com/datafuselabs/databend/issues/3428) 
+- [x] [Release proposal: Nightly v0.6 #2525](https://github.com/datafuselabs/databend/issues/2525)
+- [x] [Release proposal: Nightly v0.5 #2257](https://github.com/datafuselabs/databend/issues/2257) 

docs/doc/60-contributing/02-roadmap.md

@@ -76,6 +76,8 @@ To support creating and managing shares, Databend provides the following set of
 - Describe Share
 - Show Grants
 - Create Database, Table FROM Share
+- Select data from the Share DB tables
+- Show shared database tables
 
 ### Create Share
 
@@ -210,6 +212,33 @@ Syntax:
 CREATE DATABASE <name> FROM SHARE <provider_tenant>.<share_name>
 ```
 
+
+
+### Select data from the Share DB tables
+
+After tenants have created a database from a shared database, tenants can select data from the shared table like a normal table, before that the providers MUST have permitted access permission to the shared database and table.
+
+Syntax:
+
+```sql
+Select * from <share_db_name>.<table_name>
+```
+
+
+
+### Show shared database tables
+
+After tenants have created a database from a shared database,  tenants can show tables from the shared database, it only outputs the tables which have been permitted to access.
+
+ Syntax:
+
+```sql
+use <share_db_name>;
+show tables;
+```
+
+
+
 ## Example of Using Share with SQL
 
 To create a share using SQL:

docs/doc/60-contributing/03-rfcs/20220809-share.md

@@ -105,7 +105,13 @@ pub fn init_logging(name: &str, cfg: &Config) -> Vec<WorkerGuard> {
             .install_batch(opentelemetry::runtime::Tokio)
             .expect("install");
 
-        jaeger_layer = Some(tracing_opentelemetry::layer().with_tracer(tracer));
+        // Load filter from `RUST_LOG`. Default to `ERROR`.
+        let env_filter = EnvFilter::from_default_env();
+        jaeger_layer = Some(
+            tracing_opentelemetry::layer()
+                .with_tracer(tracer)
+                .with_filter(env_filter),
+        );
     }
     let subscriber = subscriber.with(jaeger_layer);
 

src/common/tracing/src/logging.rs

@@ -30,6 +30,9 @@ pub enum KVApiKeyError {
     #[error("Expect {expect} segments, but: '{got}'")]
     WrongNumberOfSegments { expect: usize, got: String },
 
+    #[error("Expect at least {expect} segments, but {actual} segments found")]
+    AtleastSegments { expect: usize, actual: usize },
+
     #[error("Invalid id string: '{s}': {reason}")]
     InvalidId { s: String, reason: String },
 }