Introduce database import and export protocol messages (#224)

farost · web-flow · commit f6528beec33d · 2025-06-09T17:15:19.000+01:00
## Release notes: usage and product changes Add `migration` protocol messages for usage in database import and database export operations: * a unidirectional `database_export` stream from a TypeDB server to export a specific database, similar to TypeDB 2.x; * a bidirectional `databases_import` stream between a client and a server to import an exported 2.x/3.x TypeDB database into a TypeDB 3.x server from a client. The format of migration items used for these operations is an extended version of TypeDB 2.x's migration items, so it is backward compatible with 2.x database files. **Important:** it's not intended to import 3.x databases into 2.x servers. ## Implementation Add `Migration { Item }` message. The format is an extended version of the [2.x protocol](https://github.com/typedb/typedb/blob/2.29.1/migrator/database/data.proto), so it contains "outdated" fields for its compatibility with old databases. Add `Migration { Export }` message. This operation consists of a single client `Req { database }` and multiple streamed server responses: 1. An initial response with the schema. 2. An unlimited number of migration items (multiple messages with multiple items in one message for potential optimizations) 3. A `Done` message to signal that the server is ready to close the stream without errors. It can be substituted by a silent stream closure, but I preferred explicitness here. Add `Migration { Import }` message. This operation consists of a stream of client requests: 1. An initial request with the name of the database and its schema string. 2. An unlimited number of migration items. 3. A `Done` message to signal that the client is finished without errors, and the server can perform the final validation. This `Done` message is required and cannot be removed because the client has to check whether there were finalization errors or not. and a stream of server responses (actually, there is either a single `Done` or a single error, but the stream is needed to return errors at any stage of the communication).
diff --git a/grpc/java/BUILD b/grpc/java/BUILD
@@ -13,6 +13,8 @@ java_grpc_library(
         "//proto:server-proto",
         "//proto:user-proto",
         "//proto:database-proto",
+        "//proto:migration-proto",
+        "//proto:error-proto",
         "//proto:answer-proto",
         "//proto:concept-proto",
         "//proto:connection-proto",
diff --git a/grpc/nodejs/BUILD b/grpc/nodejs/BUILD
@@ -29,6 +29,8 @@ ts_grpc_compile(
         "//proto:server-proto",
         "//proto:user-proto",
         "//proto:database-proto",
+        "//proto:migration-proto",
+        "//proto:error-proto",
         "//proto:answer-proto",
         "//proto:concept-proto",
         "//proto:connection-proto",
diff --git a/grpc/rust/BUILD b/grpc/rust/BUILD
@@ -16,6 +16,8 @@ rust_tonic_compile(
         "//proto:server-proto",
         "//proto:user-proto",
         "//proto:database-proto",
+        "//proto:migration-proto",
+        "//proto:error-proto",
         "//proto:answer-proto",
         "//proto:concept-proto",
         "//proto:connection-proto",
diff --git a/grpc/rust/build.rs b/grpc/rust/build.rs
@@ -9,6 +9,8 @@ fn main() -> std::io::Result<()> {
         "../../proto/concept.proto",
         "../../proto/connection.proto",
         "../../proto/database.proto",
+        "../../proto/error.proto",
+        "../../proto/migration.proto",
         "../../proto/options.proto",
         "../../proto/query.proto",
         "../../proto/server.proto",
diff --git a/proto/BUILD b/proto/BUILD
@@ -27,6 +27,15 @@ proto_library(
 proto_library(
     name = "database-proto",
     srcs = [":database.proto"],
+    deps = [
+        ":migration-proto",
+    ],
+)
+
+proto_library(
+    name = "migration-proto",
+    srcs = [":migration.proto"],
+    deps = [":concept-proto"],
 )
 
 proto_library(
@@ -70,17 +79,24 @@ proto_library(
     srcs = ["query.proto"],
     deps = [
         ":answer-proto",
-        ":options-proto",
         ":concept-proto",
+        ":error-proto",
+        ":options-proto",
     ],
 )
 
+proto_library(
+    name = "error-proto",
+    srcs = ["error.proto"],
+)
+
 proto_library(
     name = "transaction-proto",
     srcs = ["transaction.proto"],
     deps = [
         ":answer-proto",
         ":concept-proto",
+        ":error-proto",
         ":options-proto",
         ":query-proto",
     ]
diff --git a/proto/answer.proto b/proto/answer.proto
@@ -10,7 +10,6 @@ package typedb.protocol;
 
 message ConceptRow {
     repeated RowEntry row = 1;
-//    Explainables explainables = 2;
 }
 
 message RowEntry {
diff --git a/proto/concept.proto b/proto/concept.proto
@@ -29,22 +29,16 @@ message Thing {
 
 message Entity {
     bytes iid = 1;
-    // TODO: not sure we should include types by default, since now they can get quite large with annotations included in every single response!
-    //       perhaps it's a query option?
     optional EntityType entity_type = 2;
 }
 
 message Relation {
     bytes iid = 1;
-    // TODO: not sure we should include types by default, since now they can get quite large with annotations included in every single response!
-    //       perhaps it's a query option?
     optional RelationType relation_type = 2;
 }
 
 message Attribute {
     bytes iid = 1;
-    // TODO: not sure we should include types by default, since now they can get quite large with annotations included in every single response!
-    //       perhaps it's a query option?
     optional AttributeType attribute_type = 2;
     Value value = 3;
 }
@@ -57,7 +51,6 @@ message Value {
         Decimal decimal = 4;
         string string = 5;
         Date date = 6;
-        // time since epoch in milliseconds
         Datetime datetime = 7;
         Datetime_TZ datetime_tz = 8;
         Duration duration = 9;
@@ -152,14 +145,3 @@ message ValueType {
         string name = 1;
     };
 }
-
-message Annotation {
-    oneof annotation {
-        Key key = 1;
-        Unique unique = 2;
-        // TODO
-    }
-
-    message Key {}
-    message Unique {}
-}
diff --git a/proto/database.proto b/proto/database.proto
@@ -6,12 +6,15 @@ syntax = "proto3";
 
 package typedb.protocol;
 
+import "proto/migration.proto";
+
 message DatabaseManager {
 
     message Get {
         message Req {
             string name = 1;
         }
+
         message Res {
             DatabaseReplicas database = 1;
         }
@@ -28,6 +31,7 @@ message DatabaseManager {
         message Req {
             string name = 1;
         }
+
         message Res {
             bool contains = 1;
         }
@@ -37,10 +41,21 @@ message DatabaseManager {
         message Req {
             string name = 1;
         }
+
         message Res {
             DatabaseReplicas database = 1;
         }
     }
+
+    message Import {
+        message Client {
+            Migration.Import.Client client = 1;
+        }
+
+        message Server {
+            Migration.Import.Server server = 1;
+        }
+    }
 }
 
 message DatabaseReplicas {
@@ -62,6 +77,7 @@ message Database {
         message Req {
             string name = 1;
         }
+
         message Res {
             string schema = 1;
         }
@@ -71,19 +87,21 @@ message Database {
         message Req {
             string name = 1;
         }
+
         message Res {
             string schema = 1;
         }
     }
 
-//    message RuleSchema {
-//        message Req {
-//            string name = 1;
-//        }
-//        message Res {
-//            string schema = 1;
-//        }
-//    }
+    message Export {
+        message Req {
+            Migration.Export.Req req = 1;
+        }
+
+        message Server {
+            Migration.Export.Server server = 1;
+        }
+    }
 
     message Delete {
         message Req {
diff --git a/proto/error.proto b/proto/error.proto
@@ -0,0 +1,15 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+syntax = "proto3";
+
+package typedb.protocol;
+
+// This is an emulation of the google ErrorDetails message. Generally, ErrorDetails are submitted via the GRPC error
+// mechanism, but a manual error sending is required in streams
+message Error {
+    string error_code = 1;
+    string domain = 2;
+    repeated string stack_trace = 3;
+}
diff --git a/proto/migration.proto b/proto/migration.proto
@@ -0,0 +1,144 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+syntax = "proto3";
+
+package typedb.protocol;
+
+import "proto/concept.proto";
+
+message Migration {
+    message Export {
+        message Req {
+            string name = 1;
+        }
+
+        message Server {
+            oneof server {
+                InitialRes initial_res = 1;
+                ResPart res_part = 2;
+                Done done = 3;
+            }
+        }
+
+        message InitialRes {
+            string schema = 1;
+        }
+
+        message ResPart {
+            repeated Item items = 1;
+        }
+
+        message Done {}
+    }
+
+    message Import {
+        message Client {
+            oneof client {
+                InitialReq initial_req = 1;
+                ReqPart req_part = 2;
+                Done done = 3;
+            }
+
+            message InitialReq {
+                string name = 1;
+                string schema = 2;
+            }
+
+            message ReqPart {
+                repeated Item items = 1;
+            }
+
+            message Done { }
+        }
+
+        message Server {
+            Done done = 1;
+
+            message Done { }
+        }
+    }
+
+    //               _  _____ _____ _____ _   _ _____ ___ ___  _   _ _
+    //              / \|_   _|_   _| ____| \ | |_   _|_ _/ _ \| \ | | |
+    //             / _ \ | |   | | |  _| |  \| | | |  | | | | |  \| | |
+    //            / ___ \| |   | | | |___| |\  | | |  | | |_| | |\  |_|
+    //           /_/   \_\_|   |_| |_____|_| \_| |_| |___\___/|_| \_(_)
+    //
+    // ATTENTION: the messages below are used to import multiple versions of TypeDB.
+    // DO NOT reorder or delete existing and reserved indices. Be careful while extending this.
+    //
+
+    message Item {
+        oneof item {
+            Attribute attribute = 1;
+            Entity entity = 2;
+            Relation relation = 3;
+
+            Header header = 15;
+            Checksums checksums = 16;
+        }
+
+        message Entity {
+            string id = 1;
+            string label = 2;
+            repeated OwnedAttribute attributes = 3;
+        }
+
+        message Attribute {
+            string id = 1;
+            string label = 2;
+            repeated OwnedAttribute attributes = 3; // empty in 3.x, used for backwards compatibility
+            MigrationValue value = 4;
+        }
+
+        message Relation {
+            string id = 1;
+            string label = 2;
+            repeated OwnedAttribute attributes = 3;
+            repeated Role roles = 4;
+            message Role {
+                string label = 1;
+                repeated Player players = 2;
+                message Player {
+                    string id = 1;
+                }
+            }
+        }
+
+        message OwnedAttribute {
+            string id = 1;
+        }
+
+        message Header {
+            string typedb_version = 1;
+            string original_database = 2;
+        }
+
+        message Checksums {
+            int64 entity_count = 1;
+            int64 attribute_count = 2;
+            int64 relation_count = 3;
+            int64 role_count = 4;
+            int64 ownership_count = 5;
+            reserved 6; // was deleted and cannot be used until a breaking change occurs
+        }
+    }
+
+    message MigrationValue {
+        oneof value {
+            string string = 1;
+            bool boolean = 2;
+            int64 integer = 3;
+            double double = 4;
+            int64 datetime_millis = 5; // compatibility with 2.x, milliseconds since epoch
+            Value.Decimal decimal = 6;
+            Value.Date date = 8;
+            Value.Datetime datetime = 9;
+            Value.Datetime_TZ datetime_tz = 10;
+            Value.Duration duration = 11;
+            Value.Struct struct = 12;
+        }
+    }
+}
diff --git a/proto/query.proto b/proto/query.proto
@@ -6,6 +6,7 @@ syntax = "proto3";
 
 import "proto/answer.proto";
 import "proto/options.proto";
+import "proto/error.proto";
 
 package typedb.protocol;
 
@@ -69,11 +70,3 @@ message Query {
         }
     }
 }
-
-// This is an emulation of the google ErrorDetails message. Sometimes we submit ErrorDetails via the GRPC error mechanism
-// and sometimes (in streams) we have to manually send errors
-message Error {
-    string error_code = 1;
-    string domain = 2;
-    repeated string stack_trace = 3;
-}
diff --git a/proto/transaction.proto b/proto/transaction.proto
diff --git a/proto/typedb-service.proto b/proto/typedb-service.proto
diff --git a/proto/version.proto b/proto/version.proto

Original file line number	Diff line number	Diff line change
`@@ -10,7 +10,6 @@ package typedb.protocol;`
`10`	`10`
`11`	`11`	`message ConceptRow {`
`12`	`12`	`repeated RowEntry row = 1;`
`13`		`-// Explainables explainables = 2;`
`14`	`13`	`}`
`15`	`14`
`16`	`15`	`message RowEntry {`