Update kvproto (#323)

* update proto

Signed-off-by: andylokandy <andylokandy@hotmail.com>

* fix text

Signed-off-by: andylokandy <andylokandy@hotmail.com>

Author: andylokandy

src/request/mod.rs

@@ -182,17 +182,7 @@ mod test {
             |_: &dyn Any| {
                 Ok(Box::new(kvrpcpb::CommitResponse {
                     region_error: None,
-                    error: Some(kvrpcpb::KeyError {
-                        locked: None,
-                        retryable: String::new(),
-                        abort: String::new(),
-                        conflict: None,
-                        already_exist: None,
-                        deadlock: None,
-                        commit_ts_expired: None,
-                        txn_not_found: None,
-                        commit_ts_too_large: None,
-                    }),
+                    error: Some(kvrpcpb::KeyError::default()),
                     commit_version: 0,
                 }) as Box<dyn Any>)
             },

tikv-client-proto/proto/backup.proto renamed to tikv-client-proto/proto/brpb.proto

@@ -3,7 +3,7 @@ package backup;
 
 import "kvrpcpb.proto";
 import "errorpb.proto";
-
+import "encryptionpb.proto";
 import "gogoproto/gogo.proto";
 import "rustproto.proto";
 
@@ -19,11 +19,19 @@ message BackupMeta {
     // ID and version of backuped cluster.
     uint64 cluster_id = 1;
     string cluster_version = 2;
+    // Save the version of BR running backup jobs.
+    string br_version = 11;
+    // The backupmeta scheme version.
+    int32 version = 12;
 
     // path field is no longer used.
     reserved 3; reserved "path";
     // A set of files that compose a backup.
+    // Note: `files` is deprecated, as it bloats backupmeta. It is kept for
+    //       compatibility, so new BR can restore older backups.
     repeated File files = 4;
+    // An index to files contains data files.
+    MetaFile file_index = 13;
 
     // A pair of timestamp specifies a time range of a backup.
     // For full backup, the start_version equals to the end_version,
@@ -33,19 +41,34 @@ message BackupMeta {
     uint64 start_version = 5;
     uint64 end_version = 6;
 
-    // Additional metadata describes database and table info.
+    // Table metadata describes database and table info.
+    // Note: `schemas` is deprecated, as it bloats backupmeta. It is kept for
+    //       compatibility, so new BR can restore older backups.
     repeated Schema schemas = 7;
+    // An index to files contains Schemas.
+    MetaFile schema_index = 14;
 
-    // If in raw kv mode, `start_versions`, `end_versions` and `schemas` will be ignored, and the
-    // backup data's range is represented by raw_ranges.
+    // If in raw kv mode, `start_versions`, `end_versions` and `schemas` will be
+    // ignored, and the backup data's range is represented by raw_ranges.
     bool is_raw_kv = 8;
+    // Note: `raw_ranges` is deprecated, as it bloats backupmeta. It is kept for
+    //       compatibility, so new BR can restore older backups.
     repeated RawRange raw_ranges = 9;
+    // An index to files contains RawRanges.
+    MetaFile raw_range_index = 15;
 
-    // In incremental backup, DDLs which are completed in (lastBackupTS, backupTS] will be stored here.
+    // In incremental backup, DDLs which are completed in
+    // (lastBackupTS, backupTS] will be stored here.
+    // Note: `raw_ranges` is deprecated, as it bloats backupmeta. It is kept for
+    //       compatibility, so new BR can restore older backups.
     bytes ddls = 10;
+    // An index to files contains DDLs.
+    MetaFile ddl_indexes = 16;
+    // the backup result into `backupmeta` file
+    string backup_result = 17;
 
-    // Save the version of BR running backup jobs.
-    string br_version = 11;
+    // API version implies the encode of the key and value.
+    kvrpcpb.APIVersion api_version = 18;
 }
 
 message File {
@@ -64,6 +87,23 @@ message File {
     string cf = 10;
 
     uint64 size = 11;
+    // cipher_iv is used for AES cipher
+    bytes  cipher_iv = 12;
+}
+
+// MetaFile describes a multi-level index of data used in backup.
+message MetaFile {
+    // A set of files that contains a MetaFile.
+    // It is used as a multi-level index.
+    repeated File meta_files = 1;
+    // A set of files that contains user data.
+    repeated File data_files = 2;
+    // A set of files that contains Schemas.
+    repeated Schema schemas = 3;
+    // A set of files that contains RawRanges.
+    repeated RawRange raw_ranges = 4;
+    // A set of files that contains DDLs.
+    repeated bytes ddls = 5;
 }
 
 message Schema {
@@ -109,6 +149,11 @@ enum CompressionType {
     ZSTD = 3;
 }
 
+message CipherInfo {
+    encryptionpb.EncryptionMethod cipher_type = 1;
+    bytes cipher_key = 2;
+}
+
 message BackupRequest {
     uint64 cluster_id = 1;
 
@@ -135,6 +180,8 @@ message BackupRequest {
     CompressionType compression_type = 12;
     // sst compression level, some algorithms support negative compression levels
     int32 compression_level = 13;
+    // The cipher_info is Used to encrypt sst
+    CipherInfo cipher_info = 14;
 }
 
 message StorageBackend {
@@ -144,6 +191,8 @@ message StorageBackend {
         S3 s3 = 3;
         GCS gcs = 4;
         CloudDynamic cloud_dynamic = 5;
+        HDFS hdfs = 6;
+        AzureBlobStorage azure_blob_storage = 7;
     }
 }
 
@@ -188,6 +237,29 @@ message GCS {
     string credentials_blob = 6;
 }
 
+// AzureBlobStorage storage backend saves files into azure blob storage.
+message AzureBlobStorage {
+    string endpoint = 1;
+    // Alias: container
+    string bucket = 2;
+    // Notice: prefix starts without `/`, otherwise the first directory's name is empty.
+    string prefix = 3;
+    // Alias: access_tier.
+    // See https://docs.microsoft.com/en-us/azure/storage/blobs/access-tiers-overview
+    string storage_class = 4;
+
+    // if empty, try to read account_name from the node's environment variable $AZURE_STORAGE_ACCOUNT.
+    string account_name = 5;
+    // Use shared key to access the azure blob
+    // If the node's environment variables($AZURE_CLIENT_ID, $AZURE_TENANT_ID, $AZURE_CLIENT_SECRET) exist,
+    // prefer to use token to access the azure blob.
+    //
+    //   See https://docs.microsoft.com/en-us/azure/storage/common/identity-library-acquire-token?toc=/azure/storage/blobs/toc.json
+    // 
+    // Otherwise, if empty, try to read shared key from the node's environment variable $AZURE_STORAGE_KEY.
+    string shared_key = 6;
+}
+
 message Bucket {
     string endpoint = 1;
     string region = 3;
@@ -199,10 +271,16 @@ message Bucket {
 // CloudDynamic allows testing new cloud providers and new fields without changing protobuf definitions
 message CloudDynamic {
     Bucket bucket = 1;
-    string provider_name = 2; // s3 and gcs are supported
+    string provider_name = 2; // s3, gcs and azureBlobStorage are supported
     map<string, string> attrs = 3;
 }
 
+// HDFS storage backend saves file into HDFS compatible storages
+message HDFS {
+    //  a URL: hdfs:///some/path or hdfs://host:port/some/path
+    string remote = 1;
+}
+
 
 message BackupResponse {
     Error error = 1;

tikv-client-proto/proto/cdcpb.proto

@@ -29,12 +29,22 @@ message Compatibility {
     string required_version = 1;
 }
 
+// ClusterIDMismatch is an error variable that
+// tells people that the cluster ID of the request does not match the TiKV cluster ID.
+message ClusterIDMismatch {
+    // The current tikv cluster ID.
+    uint64 current = 1;
+    // The cluster ID of the TiCDC request.
+    uint64 request = 2;
+}
+
 message Error {
     errorpb.NotLeader not_leader = 1;
     errorpb.RegionNotFound region_not_found = 2;
     errorpb.EpochNotMatch epoch_not_match = 3;
     DuplicateRequest duplicate_request = 4;
     Compatibility compatibility = 5;
+    ClusterIDMismatch cluster_id_mismatch = 6;
 }
 
 message TxnInfo {

tikv-client-proto/proto/coprocessor.proto

@@ -36,6 +36,8 @@ message Request {
     // Any schema-ful storage to validate schema correctness if necessary.
     int64 schema_ver = 8;
     bool is_trace_enabled = 9;
+    // paging_size is 0 when it's disabled, otherwise, it should be a positive number.
+    uint64 paging_size = 10;
 }
 
 message Response {

tikv-client-proto/proto/debugpb.proto

@@ -70,6 +70,9 @@ service Debug {
 
     // Get cluster ID
     rpc GetClusterInfo(GetClusterInfoRequest) returns (GetClusterInfoResponse) {}
+
+    // Get all region IDs in the store
+    rpc GetAllRegionsInStore(GetAllRegionsInStoreRequest) returns (GetAllRegionsInStoreResponse) {}
 }
 
 enum DB {
@@ -240,6 +243,7 @@ message GetStoreInfoRequest {
 
 message GetStoreInfoResponse {
     uint64 store_id = 1;
+    kvrpcpb.APIVersion api_version = 2;
 }
 
 message GetClusterInfoRequest {
@@ -248,3 +252,10 @@ message GetClusterInfoRequest {
 message GetClusterInfoResponse {
     uint64 cluster_id = 1;
 }
+
+message GetAllRegionsInStoreRequest {
+}
+
+message GetAllRegionsInStoreResponse {
+    repeated uint64 regions = 1;
+}

tikv-client-proto/proto/disk_usage.proto

@@ -0,0 +1,14 @@
+syntax = "proto3";
+package disk_usage;
+
+import "rustproto.proto";
+
+option (rustproto.lite_runtime_all) = true;
+
+option java_package = "org.tikv.kvproto";
+
+enum DiskUsage {
+    Normal = 0;
+    AlmostFull = 1;
+    AlreadyFull = 2;
+}

tikv-client-proto/proto/errorpb.proto

@@ -21,6 +21,13 @@ message NotLeader {
     metapb.Peer leader = 2;
 }
 
+message DiskFull {
+    // The requested store ID
+    repeated uint64 store_id = 1;
+    // The detailed info
+    string reason = 2;
+}
+
 // StoreNotMatch is the error variant that tells the request is sent to wrong store. 
 // (i.e. inconsistency of the store ID that request shows and the real store ID of this server.)
 message StoreNotMatch {
@@ -37,6 +44,13 @@ message RegionNotFound {
     uint64 region_id = 1;
 }
 
+// RegionNotInitialized is the error variant that tells there isn't any initialized peer
+// matchesthe request region ID.
+message RegionNotInitialized  {
+    // The request region ID
+    uint64 region_id = 1;
+}
+
 // KeyNotInRegion is the error variant that tells the key the request requires isn't present in
 // this region. 
 message KeyNotInRegion {
@@ -132,4 +146,6 @@ message Error {
     ReadIndexNotReady read_index_not_ready = 11;
     ProposalInMergingMode proposal_in_merging_mode = 12;
     DataIsNotReady data_is_not_ready = 13;
+    RegionNotInitialized region_not_initialized = 14;
+    DiskFull disk_full = 15;
 }