Merge branch 'main' into ISSUE-7609/drop_pin

Author: mergify[bot]

Cargo.lock

@@ -34,7 +34,7 @@ members = [
     "src/query/pipeline/sinks",
     "src/query/pipeline/sources",
     "src/query/pipeline/transforms",
-    "src/query/planners",
+    "src/query/legacy-planners",
     "src/query/settings",
     "src/query/storages/fuse",
     "src/query/storages/fuse-meta",

Cargo.toml

@@ -24,8 +24,8 @@ SUBSTRING(str FROM pos FOR len)
 | Arguments   | Description |
 | ----------- | ----------- |
 | str | The main string from where the character to be extracted |
-| pos | The one-indexed position expression to start at. If negative, counts from the end |
-| len | The number expression of characters to extract |
+| pos | The position (starting from 1) the substring to start at. If negative, counts from the end |
+| len | The maximun length of the substring to extract |
 
 ## Return Type
 

docs/doc/30-reference/20-functions/40-string-functions/substring.md

@@ -1,7 +1,3 @@
 {
-  "label": "Cluster Key",
-  "link": {
-    "type": "generated-index",
-    "slug": "/reference/sql/ddl/clusterkey"
-  }
+  "label": "Cluster Key"
 }

docs/doc/30-reference/30-sql/00-ddl/70-clusterkey/_category_.json

@@ -0,0 +1,15 @@
+---
+title: What is a Cluster Key?
+sidebar_position: 1
+slug: ./
+---
+
+The cluster key is a data object for tables in Databend. It explicitly tells Databend how to divide and group rows of a table into the storage partitions rather than using the data ingestion order. 
+
+A table's cluster key is usually one or more columns or expressions. If you define a cluster key for a table, Databend reorganizes your data based on the cluster key and stores similar rows into the same or adjacent storage partitions.
+
+The benefit of defining a cluster key is optimizing the query performance.   The cluster key acts as a link between the metadata in the Databend's Meta Service Layer and the storage partitions. After the cluster key is defined for a table, the table's metadata implements a key-value-like list that shows the correspondences between the column or expression values and their storage partitions. When a query comes, Databend can quickly locate the storage partition by the metadata and fetch the results. To make this work, the cluster key you set must match the way how you filter the data in queries. For example, if you're most likely to query a table that holds all the employees' profile information by their first names, set the cluster key to the first name column.
+
+In Databend, you [SET CLUSTER KEY](dml-set-cluster-key.md) when you create a table, and you can [ALTER CLUSTER KEY](https://databend.rs/doc/reference/sql/ddl/clusterkey/dml-alter-cluster-key) if necessary. A fully-clustered table might become chaotic if it continues to have ingestion or Data Manipulation Language operations (such as INSERT, UPDATE, DELETE), you will need to [RECLUSTER TABLE](./dml-recluster-table.md) to fix the chaos.
+
+It's important to note that, most of the time you do not need to set the cluster key. Clustering or re-clustering a table consumes time and your credits if you're in Databend Cloud. Databend recommends setting cluster keys for large tables with slow query issues.

docs/doc/30-reference/30-sql/00-ddl/70-clusterkey/index.md

@@ -30,6 +30,9 @@ pub enum KVApiKeyError {
     #[error("Expect {expect} segments, but: '{got}'")]
     WrongNumberOfSegments { expect: usize, got: String },
 
+    #[error("Expect at least {expect} segments, but {actual} segments found")]
+    AtleastSegments { expect: usize, actual: usize },
+
     #[error("Invalid id string: '{s}': {reason}")]
     InvalidId { s: String, reason: String },
 }

src/meta/api/src/kv_api_key.rs

@@ -483,6 +483,7 @@ impl KVApiTestSuite {
                 key: txn_key.clone(),
                 value: b"new_v1".to_vec(),
                 prev_value: true,
+                expire_at: None,
             })),
         }];
 
@@ -640,6 +641,7 @@ impl KVApiTestSuite {
                     key: txn_key.clone(),
                     value: b"new_v1".to_vec(),
                     prev_value: true,
+                    expire_at: None,
                 })),
             }];
 
@@ -691,6 +693,7 @@ impl KVApiTestSuite {
                     key: txn_key1.clone(),
                     value: b"new_v1".to_vec(),
                     prev_value: true,
+                    expire_at: None,
                 })),
             }];
 
@@ -745,6 +748,7 @@ impl KVApiTestSuite {
                         key: txn_key1.clone(),
                         value: val1_new.to_vec(),
                         prev_value: true,
+                        expire_at: None,
                     })),
                 },
                 // change k2
@@ -753,6 +757,7 @@ impl KVApiTestSuite {
                         key: txn_key2.clone(),
                         value: b"new_v2".to_vec(),
                         prev_value: true,
+                        expire_at: None,
                     })),
                 },
                 // get k1
@@ -852,6 +857,7 @@ impl KVApiTestSuite {
                         key: txn_key1.clone(),
                         value: val1_new.to_vec(),
                         prev_value: true,
+                        expire_at: None,
                     })),
                 },
                 // get k1

src/meta/api/src/kv_api_test_suite.rs

@@ -226,6 +226,18 @@ pub fn txn_op_put(key: &impl KVApiKey, value: Vec<u8>) -> TxnOp {
             key: key.to_key(),
             value,
             prev_value: true,
+            expire_at: None,
+        })),
+    }
+}
+
+pub fn txn_op_put_with_expire(key: &impl KVApiKey, value: Vec<u8>, expire_at: u64) -> TxnOp {
+    TxnOp {
+        request: Some(Request::Put(TxnPutRequest {
+            key: key.to_key(),
+            value,
+            prev_value: true,
+            expire_at: Some(expire_at),
         })),
     }
 }

src/meta/api/src/kv_api_utils.rs

@@ -64,6 +64,7 @@ pub use kv_api_utils::table_has_to_exist;
 pub use kv_api_utils::txn_cond_seq;
 pub use kv_api_utils::txn_op_del;
 pub use kv_api_utils::txn_op_put;
+pub use kv_api_utils::txn_op_put_with_expire;
 pub use kv_api_utils::TXN_MAX_RETRY_TIMES;
 pub use schema_api::SchemaApi;
 pub(crate) use schema_api_impl::get_db_or_err;

src/meta/api/src/lib.rs

@@ -26,6 +26,8 @@ use common_meta_app::schema::DropDatabaseReq;
 use common_meta_app::schema::DropTableReply;
 use common_meta_app::schema::DropTableReq;
 use common_meta_app::schema::GetDatabaseReq;
+use common_meta_app::schema::GetTableCopiedFileReply;
+use common_meta_app::schema::GetTableCopiedFileReq;
 use common_meta_app::schema::GetTableReq;
 use common_meta_app::schema::ListDatabaseReq;
 use common_meta_app::schema::ListTableReq;
@@ -36,12 +38,16 @@ use common_meta_app::schema::RenameTableReq;
 use common_meta_app::schema::TableIdent;
 use common_meta_app::schema::TableInfo;
 use common_meta_app::schema::TableMeta;
+use common_meta_app::schema::TruncateTableReply;
+use common_meta_app::schema::TruncateTableReq;
 use common_meta_app::schema::UndropDatabaseReply;
 use common_meta_app::schema::UndropDatabaseReq;
 use common_meta_app::schema::UndropTableReply;
 use common_meta_app::schema::UndropTableReq;
 use common_meta_app::schema::UpdateTableMetaReply;
 use common_meta_app::schema::UpdateTableMetaReq;
+use common_meta_app::schema::UpsertTableCopiedFileReply;
+use common_meta_app::schema::UpsertTableCopiedFileReq;
 use common_meta_app::schema::UpsertTableOptionReply;
 use common_meta_app::schema::UpsertTableOptionReq;
 use common_meta_types::GCDroppedDataReply;
@@ -104,6 +110,18 @@ pub trait SchemaApi: Send + Sync {
         table_id: MetaId,
     ) -> Result<(TableIdent, Arc<TableMeta>), MetaError>;
 
+    async fn get_table_copied_file_info(
+        &self,
+        req: GetTableCopiedFileReq,
+    ) -> Result<GetTableCopiedFileReply, MetaError>;
+
+    async fn upsert_table_copied_file_info(
+        &self,
+        req: UpsertTableCopiedFileReq,
+    ) -> Result<UpsertTableCopiedFileReply, MetaError>;
+
+    async fn truncate_table(&self, req: TruncateTableReq) -> Result<TruncateTableReply, MetaError>;
+
     async fn upsert_table_option(
         &self,
         req: UpsertTableOptionReq,