Merge pull request #7526 from drmingdrmer/3-ver

ci(proto): add compat test for CopyOptions::purge

Author: drmingdrmer

Cargo.lock

@@ -28,3 +28,4 @@ serde_json = "1.0.81"
 [dev-dependencies]
 anyhow = "1.0.58"
 maplit = "1.0.2"
+pretty_assertions = "1.2.1"

src/meta/proto-conv/Cargo.toml

@@ -12,6 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// For use of const fn: `Option::<T>::unwrap` at compile time.
+#![feature(const_option)]
+
 //! Provides conversion from and to protobuf defined meta data, which is used for transport.
 //!
 //! Thus protobuf messages has the maximized compatibility.

src/meta/proto-conv/src/lib.rs

@@ -14,7 +14,33 @@
 
 use crate::Incompatible;
 
-pub const VER: u64 = 5;
+/// Describes metadata changes.
+///
+/// This is a list of every `VER` and the corresponding change it introduces.
+///
+/// ## For developers
+///
+/// Every time fields are added/removed into/from data types in this crate:
+/// - Add a new line to this list to describe what changed.
+/// - Add a test case to ensure protobuf message serialized by this this version can be loaded,
+///   similar to: test_user_stage_fs_v6() in tests/it/user_stage.rs;
+///
+/// `VER` is the current metadata version and is automatically set to the last version.
+/// `MIN_COMPATIBLE_VER` is the oldest compatible version.
+const META_CHANGE_LOG: &[(u64, &str)] = &[
+    //
+    (1, "----------: Initial"),
+    (2, "2022-07-13: Add: share.proto"),
+    (3, "2022-07-29: Add: user.proto/UserOption::default_role"),
+    (4, "2022-08-22: Add: config.proto/GcsStorageConfig"),
+    (
+        5,
+        "2022-08-25: Add: ShareMeta::share_from_db_ids; DatabaseMeta::from_share",
+    ),
+    (6, "2022-09-08: Add: users.proto/CopyOptions::purge"),
+];
+
+pub const VER: u64 = META_CHANGE_LOG.last().unwrap().0;
 pub const MIN_COMPATIBLE_VER: u64 = 1;
 
 pub fn check_ver(msg_ver: u64, msg_min_compatible: u64) -> Result<(), Incompatible> {

src/meta/proto-conv/src/util.rs

@@ -0,0 +1,72 @@
+// Copyright 2022 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt::Debug;
+use std::fmt::Display;
+
+use common_proto_conv::FromToProto;
+use common_proto_conv::VER;
+use pretty_assertions::assert_eq;
+
+/// Tests converting rust types from/to protobuf defined types.
+/// It also print out encoded protobuf message as data for backward compatibility test.
+pub(crate) fn test_pb_from_to<MT>(name: impl Display, m: MT) -> anyhow::Result<()>
+where
+    MT: FromToProto + PartialEq + Debug,
+    MT::PB: common_protos::prost::Message,
+{
+    let p = m.to_pb()?;
+
+    let mut buf = vec![];
+    common_protos::prost::Message::encode(&p, &mut buf)?;
+    // The encoded data should be saved for compatability test.
+    println!("// Encoded data of version {} of {}:", VER, name);
+    println!("// It is generated with common::test_pb_from_to.");
+    println!("let {}_v{} = vec!{:?};", name, VER, buf);
+
+    let got = MT::from_pb(p)?;
+    assert_eq!(m, got, "convert from/to protobuf: {}", name);
+    Ok(())
+}
+
+/// Tests loading old version data.
+pub(crate) fn test_load_old<MT>(name: impl Display, buf: &[u8], want: MT) -> anyhow::Result<()>
+where
+    MT: FromToProto + PartialEq + Debug,
+    MT::PB: common_protos::prost::Message + Default,
+{
+    let p: MT::PB = common_protos::prost::Message::decode(buf).map_err(print_err)?;
+    let got = MT::from_pb(p).map_err(print_err)?;
+
+    assert_eq!(want, got, "loading {} with version {} program", name, VER);
+    Ok(())
+}
+
+pub(crate) fn print_err<T: Debug>(e: T) -> T {
+    eprintln!("Error: {:?}", e);
+    e
+}
+
+macro_rules! func_name {
+    () => {{
+        fn f() {}
+        fn type_name_of<T>(_: T) -> &'static str {
+            std::any::type_name::<T>()
+        }
+        let name = type_name_of(f);
+        let n = &name[..name.len() - 3];
+        let nn = n.replace("::{{closure}}", "");
+        nn
+    }};
+}

src/meta/proto-conv/tests/it/common.rs

@@ -11,5 +11,9 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
+#[macro_use]
+pub(crate) mod common;
 mod proto_conv;
 mod user_proto_conv;
+mod user_stage;

src/meta/proto-conv/tests/it/main.rs

@@ -14,7 +14,6 @@
 
 use std::collections::BTreeMap;
 use std::collections::BTreeSet;
-use std::fmt::Debug;
 use std::sync::Arc;
 
 use common_datavalues as dv;
@@ -24,9 +23,12 @@ use common_meta_app::schema as mt;
 use common_meta_app::share;
 use common_proto_conv::FromToProto;
 use common_proto_conv::Incompatible;
+use common_proto_conv::VER;
 use common_protos::pb;
 use maplit::btreemap;
 
+use crate::common::print_err;
+
 fn s(ss: impl ToString) -> String {
     ss.to_string()
 }
@@ -233,13 +235,17 @@ fn test_pb_from_to() -> anyhow::Result<()> {
 fn test_incompatible() -> anyhow::Result<()> {
     let db_meta = new_db_meta();
     let mut p = db_meta.to_pb()?;
-    p.ver = 6;
-    p.min_compatible = 6;
+    p.ver = VER + 1;
+    p.min_compatible = VER + 1;
 
     let res = mt::DatabaseMeta::from_pb(p);
     assert_eq!(
         Incompatible {
-            reason: s("executable ver=5 is smaller than the message min compatible ver: 6")
+            reason: format!(
+                "executable ver={} is smaller than the message min compatible ver: {}",
+                VER,
+                VER + 1
+            )
         },
         res.unwrap_err()
     );
@@ -487,8 +493,3 @@ fn test_load_old() -> anyhow::Result<()> {
 
     Ok(())
 }
-
-fn print_err<T: Debug>(e: T) -> T {
-    eprintln!("Error: {:?}", e);
-    e
-}