Merge branch 'main' of https://github.com/datafuselabs/databend into fixxxxx

Author: andylokandy

Cargo.lock

@@ -0,0 +1,56 @@
+---
+title: Idempotent Copy
+description: Avoid duplicating when copy stage files into a table
+---
+
+- Tracking Issue: https://github.com/datafuselabs/databend/issues/6338
+
+## Summary
+
+When streaming copy stage files into a table, there is a chance that some files have already been copied, So it needs some ways to avoid duplicate copying files, make it an `idempotent` operation.
+
+## Save copy into table stage files meta information in meta service
+
+Whenever copy stage files into a table, save the stage file meta information into the meta service:
+
+- key: combined with `(tenant, database, table, file name)`.
+- value: value MUST includes all the meta of a stage file, such as `content-length`,`etag`,`last modified`.
+
+
+
+![](/img/rfc/20220909-idempotent-copy/stage-file-meta.png)
+
+
+
+The expiration time of the stage file meta information is 64 days by default.
+
+## Avoiding duplicates when copy stage files into a table
+
+Using the stage file meta information, whenever copy stage files into a table, follow these steps:
+
+* First, get all the table file meta information of the copy stage files that want to copy into the table(if any).
+* Second, get all the stage file meta information.
+* Third, compare the table file meta information with stage file meta information:
+  * If they matched, this file is just ignored without copying.
+  * Else, copy the stage file and up-insert into the table stage file meta.
+
+
+
+![](/img/rfc/20220909-idempotent-copy/example.png)
+
+
+
+Take the image above as an example:
+
+* Client make a request to copy thress files (file1, file2, file3) into table.
+
+* Get the table stage file meta of (file1, file2, file3).
+
+* In the meta service, only found (file1,file3) stage file information.
+
+* Compare the table stage file information with stage file information, and found that file1 has not been changed, so file1 will be ignored in this copy operation, and (file2,file3) will be copied.
+
+* After copying new files, (file2, file3) stage file information will be saved into table file information.
+
+
+

docs/doc/60-contributing/03-rfcs/20220909-idempotent-copy.md

@@ -56,14 +56,9 @@ pub enum MetaGrpcWriteReq {
 pub enum MetaGrpcReadReq {
     GetKV(GetKVReq),
     MGetKV(MGetKVReq),
-    // #[deprecated(since = "0.7.57-nightly", note = "deprecated since 2022-05-23")]
-    PrefixListKV(PrefixListReq),
     ListKV(ListKVReq), // since 2022-05-23
 }
 
-#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]
-pub struct PrefixListReq(pub String);
-
 /// Try convert tonic::Request<RaftRequest> to DoActionAction.
 impl TryInto<MetaGrpcWriteReq> for Request<RaftRequest> {
     type Error = tonic::Status;

docs/public/img/rfc/20220909-idempotent-copy/example.png

@@ -74,11 +74,6 @@ impl ActionHandler {
                 incr_meta_metrics_meta_request_result(r.is_ok());
                 RaftReply::from(r)
             }
-            MetaGrpcReadReq::PrefixListKV(a) => {
-                let r = self.meta_node.prefix_list_kv(&a.0).await;
-                incr_meta_metrics_meta_request_result(r.is_ok());
-                RaftReply::from(r)
-            }
         }
     }
 

docs/public/img/rfc/20220909-idempotent-copy/stage-file-meta.png

@@ -47,7 +47,7 @@ pub static METASRV_SEMVER: Lazy<Version> = Lazy::new(|| {
 pub static MIN_METACLI_SEMVER: Version = Version {
     major: 0,
     minor: 7,
-    patch: 57,
+    patch: 59,
     pre: Prerelease::EMPTY,
     build: BuildMetadata::EMPTY,
 };

src/meta/client/src/grpc_action.rs

@@ -21,5 +21,6 @@ common-expression = { path = "../expression" }
 # TODO(andylokandy): Use the version from crates.io once
 # https://github.com/reem/rust-ordered-float/pull/110 is released.
 ordered-float = { git = "https://github.com/andylokandy/rust-ordered-float.git", branch = "as", features = ["serde"] }
+itertools = "0.10"
 serde = { version = "1.0.137", features = ["derive"] }
 serde_json = "1.0.81"

src/meta/service/src/executor/action_handler.rs

@@ -12,10 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use codegen::writes::codegen_arithmetic_type;
-use codegen::writes::codegen_arithmetic_type_v2;
-
 fn main() {
-    codegen_arithmetic_type();
-    codegen_arithmetic_type_v2();
+    codegen::writes::codegen_arithmetic_type();
+    codegen::writes::codegen_arithmetic_type_v2();
+    codegen::writes::codegen_register();
 }

src/meta/service/src/version.rs

@@ -12,4 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#![feature(exit_status_error)]
+
 pub mod writes;