Skip to content

Commit 0bbcc69

Browse files
SammyVimesjepett0
andauthored
Fix UUID handling in backup/restore in YDB CLI (#17198)
Fix YQL, BulkUpsert and ImportData backup and restore modes so that they can handle UUID fields Co-authored-by: jepett0 <111313089+jepett0@users.noreply.github.com>
1 parent 7a9927a commit 0bbcc69

File tree

6 files changed

+112
-4
lines changed

6 files changed

+112
-4
lines changed

ydb/library/backup/backup.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,7 @@ void PrintPrimitive(IOutputStream& out, const TValueParser& parser) {
152152
CASE_PRINT_PRIMITIVE_TYPE(out, Datetime64);
153153
CASE_PRINT_PRIMITIVE_TYPE(out, Timestamp64);
154154
CASE_PRINT_PRIMITIVE_TYPE(out, Interval64);
155+
CASE_PRINT_PRIMITIVE_TYPE(out, Uuid);
155156
CASE_PRINT_PRIMITIVE_STRING_TYPE(out, TzDate);
156157
CASE_PRINT_PRIMITIVE_STRING_TYPE(out, TzDatetime);
157158
CASE_PRINT_PRIMITIVE_STRING_TYPE(out, TzTimestamp);

ydb/library/backup/query_builder.cpp

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
#include "backup.h"
44

55
#include <yql/essentials/types/dynumber/dynumber.h>
6+
#include <yql/essentials/types/uuid/uuid.h>
67
#include <ydb/public/api/protos/ydb_value.pb.h>
78
#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/proto/accessor.h>
89

@@ -136,11 +137,11 @@ void TQueryBuilder::AddPrimitiveMember(EPrimitiveType type, TStringBuf buf) {
136137

137138
case EPrimitiveType::Datetime64:
138139
Value.Datetime64(TryParse<i64>(buf));
139-
break;
140+
break;
140141

141142
case EPrimitiveType::Timestamp64:
142143
Value.Timestamp64(TryParse<i64>(buf));
143-
break;
144+
break;
144145

145146
case EPrimitiveType::Interval64:
146147
Value.Interval64(TryParse<i64>(buf));
@@ -184,7 +185,8 @@ void TQueryBuilder::AddPrimitiveMember(EPrimitiveType type, TStringBuf buf) {
184185
break;
185186

186187
case EPrimitiveType::Uuid:
187-
Y_ENSURE(false, TStringBuilder() << "Unexpected Primitive kind while parsing line: " << type);
188+
Y_ENSURE(NKikimr::NUuid::IsValidUuid(buf));
189+
Value.Uuid(TUuidValue(std::string(buf.begin(), buf.end())));
188190
break;
189191

190192
}

ydb/public/lib/ydb_cli/dump/restore_import_data.cpp

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,20 @@
2222
#include <util/system/mutex.h>
2323
#include <util/thread/pool.h>
2424

25+
namespace NYdb {
26+
27+
bool operator<(const TUuidValue& lhs, const TUuidValue& rhs) {
28+
// Lexicographical comparison of UUIDs for TValue comparison.
29+
// It works just like TCell::CompareCellsAsByteString.
30+
// We need it since RPC Import Data expects keys to be sorted.
31+
const char* pa = lhs.Buf_.Bytes;
32+
const char* pb = rhs.Buf_.Bytes;
33+
int cmp = memcmp(pa, pb, 16);
34+
return cmp < 0;
35+
}
36+
37+
}
38+
2539
namespace NYdb::NDump {
2640

2741
using namespace NImport;
@@ -44,6 +58,7 @@ class TValue {
4458
Null,
4559
String,
4660
Pod,
61+
Uuid
4762
};
4863

4964
inline EType GetType() const {
@@ -54,6 +69,8 @@ class TValue {
5469
return EType::Null;
5570
case 2:
5671
return EType::String;
72+
case 9:
73+
return EType::Uuid;
5774
default:
5875
return EType::Pod;
5976
}
@@ -110,7 +127,8 @@ class TValue {
110127
i32,
111128
ui32,
112129
i64,
113-
ui64
130+
ui64,
131+
TUuidValue
114132
> Value;
115133

116134
}; // TValue
@@ -149,6 +167,8 @@ class TValueConverter {
149167
return TValue(Parser.GetString());
150168
case EPrimitiveType::Utf8:
151169
return TValue(Parser.GetUtf8());
170+
case EPrimitiveType::Uuid:
171+
return TValue(Parser.GetUuid());
152172
default:
153173
Y_ENSURE(false, "Unexpected primitive type: " << type);
154174
}
@@ -290,6 +310,10 @@ class TYdbDumpValueParser {
290310
return CheckedUnescape();
291311
}
292312

313+
TUuidValue GetUuid() const {
314+
return TUuidValue(std::string(Value));
315+
}
316+
293317
bool IsNull() const {
294318
return Value == "null";
295319
}

ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/value/value.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -539,3 +539,6 @@ class TValueBuilder : public TValueBuilderBase<TValueBuilder> {
539539
};
540540

541541
} // namespace NYdb
542+
543+
template<>
544+
void Out<NYdb::TUuidValue>(IOutputStream& o, const NYdb::TUuidValue& value);

ydb/public/sdk/cpp/src/client/value/value.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3374,3 +3374,8 @@ TValue TValueBuilder::Build() {
33743374
}
33753375

33763376
} // namespace NYdb
3377+
3378+
template<>
3379+
void Out<NYdb::TUuidValue>(IOutputStream& o, const NYdb::TUuidValue& value) {
3380+
o << value.ToString();
3381+
}

ydb/services/ydb/backup_ut/ydb_backup_ut.cpp

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1372,6 +1372,79 @@ Y_UNIT_TEST_SUITE(BackupRestore) {
13721372
[](const TYdbErrorException& e) { return e.GetStatus().GetStatus() == EStatus::SCHEME_ERROR; });
13731373
}
13741374

1375+
Y_UNIT_TEST(BackupUuid) {
1376+
TKikimrWithGrpcAndRootSchema server;
1377+
auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%u", server.GetPort())));
1378+
TTableClient tableClient(driver);
1379+
auto session = tableClient.GetSession().ExtractValueSync().GetSession();
1380+
TTempDir tempDir;
1381+
const auto& pathToBackup = tempDir.Path();
1382+
1383+
constexpr const char* dbPath = "/Root";
1384+
constexpr const char* table = "/Root/table";
1385+
1386+
ExecuteDataDefinitionQuery(session, Sprintf(R"(
1387+
CREATE TABLE `%s` (
1388+
Key Uuid,
1389+
Value Utf8,
1390+
PRIMARY KEY (Key)
1391+
);
1392+
)",
1393+
table
1394+
));
1395+
1396+
std::vector<std::string> uuids = {
1397+
"5b99a330-04ef-4f1a-9b64-ba6d5f44eafe",
1398+
"706cca52-b00a-4cbd-a21e-6538de188271",
1399+
"81b1e345-f2ae-4c9e-8d1a-75447be314f2",
1400+
"be2765f2-9f4c-4a22-8d2c-a1b77d84f4fb",
1401+
"d3f9e0a2-5871-4afe-a23a-8db160b449cd",
1402+
"d3f9e0a2-0000-0000-0000-8db160b449cd"
1403+
};
1404+
1405+
ExecuteDataModificationQuery(session, Sprintf(R"(
1406+
UPSERT INTO `%s` (Key, Value)
1407+
VALUES
1408+
(Uuid("%s"), "one"),
1409+
(Uuid("%s"), "two"),
1410+
(Uuid("%s"), "three"),
1411+
(Uuid("%s"), "four"),
1412+
(Uuid("%s"), "five"),
1413+
(Uuid("%s"), "six");
1414+
)", table, uuids[0].c_str(), uuids[1].c_str(), uuids[2].c_str(), uuids[3].c_str(), uuids[4].c_str(), uuids[5].c_str()
1415+
));
1416+
1417+
const auto originalContent = GetTableContent(session, table);
1418+
1419+
NDump::TClient backupClient(driver);
1420+
{
1421+
const auto result = backupClient.Dump(dbPath, pathToBackup, NDump::TDumpSettings().Database(dbPath));
1422+
UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
1423+
}
1424+
1425+
// Check that backup file contains all uuids as strings, making sure we stringify UUIDs correctly in backups
1426+
TString backupFileContent = TFileInput(pathToBackup.GetPath() + "/table/data_00.csv").ReadAll();
1427+
for (const auto& uuid : uuids) {
1428+
UNIT_ASSERT_C(backupFileContent.find(uuid) != TString::npos, "UUID not found in backup file");
1429+
}
1430+
1431+
for (auto backupMode : {NDump::TRestoreSettings::EMode::BulkUpsert, NDump::TRestoreSettings::EMode::ImportData, NDump::TRestoreSettings::EMode::Yql}) {
1432+
auto opts = NDump::TRestoreSettings().Mode(backupMode);
1433+
1434+
ExecuteDataDefinitionQuery(session, Sprintf(R"(
1435+
DROP TABLE `%s`;
1436+
)", table
1437+
));
1438+
1439+
auto result = backupClient.Restore(pathToBackup, dbPath, opts);
1440+
UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
1441+
1442+
const auto newContent = GetTableContent(session, table);
1443+
1444+
CompareResults(newContent, originalContent);
1445+
}
1446+
}
1447+
13751448
// TO DO: test index impl table split boundaries restoration from a backup
13761449

13771450
Y_UNIT_TEST(RestoreViewQueryText) {

0 commit comments

Comments
 (0)