Skip to content

Commit 46dd6bb

Browse files
jepett0CyberROFL
jepett0
and
CyberROFL
authored
test all primitive column types in backup tests (#17183)
Co-authored-by: Ilnaz Nizametdinov <i.nizametdinov@gmail.com>
1 parent 6d6cbd4 commit 46dd6bb

File tree

1 file changed

+241
-18
lines changed

1 file changed

+241
-18
lines changed

‎ydb/services/ydb/backup_ut/ydb_backup_ut.cpp

Lines changed: 241 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -123,33 +123,29 @@ struct TTenantsTestSettings : TKikimrTestSettings {
123123
namespace {
124124

125125
#define Y_UNIT_TEST_ALL_PROTO_ENUM_VALUES(N, ENUM_TYPE) \
126-
template <ENUM_TYPE Value> \
127126
struct TTestCase##N : public TCurrentTestCase { \
128-
TString ParametrizedTestName = #N "-" + ENUM_TYPE##_Name(Value); \
127+
ENUM_TYPE Value; \
128+
TString ParametrizedTestName; \
129129
\
130-
TTestCase##N() : TCurrentTestCase() { \
130+
TTestCase##N(ENUM_TYPE value) : TCurrentTestCase(), Value(value), ParametrizedTestName(#N "-" + ENUM_TYPE##_Name(Value)) { \
131131
Name_ = ParametrizedTestName.c_str(); \
132132
} \
133133
\
134-
static THolder<NUnitTest::TBaseTestCase> Create() { return ::MakeHolder<TTestCase##N<Value>>(); } \
134+
static THolder<NUnitTest::TBaseTestCase> Create(ENUM_TYPE value) { return ::MakeHolder<TTestCase##N>(value); } \
135135
void Execute_(NUnitTest::TTestContext&) override; \
136136
}; \
137137
struct TTestRegistration##N { \
138-
template <int I, int End> \
139-
static constexpr void AddTestsForEnumRange() { \
140-
if constexpr (I < End) { \
141-
TCurrentTest::AddTest(TTestCase##N<static_cast<ENUM_TYPE>(I)>::Create); \
142-
AddTestsForEnumRange<I + 1, End>(); \
143-
} \
144-
} \
145-
\
146138
TTestRegistration##N() { \
147-
AddTestsForEnumRange<0, ENUM_TYPE##_ARRAYSIZE>(); \
139+
const auto* enumDescriptor = google::protobuf::GetEnumDescriptor<ENUM_TYPE>(); \
140+
for (int i = 0; i < enumDescriptor->value_count(); ++i) { \
141+
const auto* valueDescriptor = enumDescriptor->value(i); \
142+
const auto value = static_cast<ENUM_TYPE>(valueDescriptor->number()); \
143+
TCurrentTest::AddTest([value] { return TTestCase##N::Create(value); }); \
144+
} \
148145
} \
149146
}; \
150147
static TTestRegistration##N testRegistration##N; \
151-
template <ENUM_TYPE Value> \
152-
void TTestCase##N<Value>::Execute_(NUnitTest::TTestContext& ut_context Y_DECLARE_UNUSED)
148+
void TTestCase##N::Execute_(NUnitTest::TTestContext& ut_context Y_DECLARE_UNUSED)
153149

154150
#define DEBUG_HINT (TStringBuilder() << "at line " << __LINE__)
155151

@@ -812,7 +808,7 @@ void TestViewDependentOnAnotherViewIsRestored(
812808
CompareResults(GetTableContent(session, dependentView), originalContent);
813809
}
814810

815-
std::pair<std::vector<TString>, std::vector<TString>>
811+
std::pair<std::vector<TString>, std::vector<TString>>
816812
GetChangefeedAndTopicDescriptions(const char* table, TSession& session, NTopic::TTopicClient& topicClient) {
817813
auto describeChangefeeds = DescribeChangefeeds(session, table);
818814
const auto vectorSize = describeChangefeeds.size();
@@ -836,7 +832,7 @@ GetChangefeedAndTopicDescriptions(const char* table, TSession& session, NTopic::
836832
);
837833
return protoStr;
838834
});
839-
835+
840836
return {changefeedsStr, topicsStr};
841837
}
842838

@@ -1209,6 +1205,200 @@ void TestExternalTableSettingsArePreserved(
12091205
);
12101206
}
12111207

1208+
// transform the type to the string usable in CREATE TABLE YQL statement
1209+
std::string_view GetYqlType(Ydb::Type::PrimitiveTypeId type) {
1210+
switch (type) {
1211+
case Ydb::Type_PrimitiveTypeId_BOOL: return "Bool";
1212+
case Ydb::Type_PrimitiveTypeId_INT8: return "Int8";
1213+
case Ydb::Type_PrimitiveTypeId_UINT8: return "Uint8";
1214+
case Ydb::Type_PrimitiveTypeId_INT16: return "Int16";
1215+
case Ydb::Type_PrimitiveTypeId_UINT16: return "Uint16";
1216+
case Ydb::Type_PrimitiveTypeId_INT32: return "Int32";
1217+
case Ydb::Type_PrimitiveTypeId_UINT32: return "Uint32";
1218+
case Ydb::Type_PrimitiveTypeId_INT64: return "Int64";
1219+
case Ydb::Type_PrimitiveTypeId_UINT64: return "Uint64";
1220+
case Ydb::Type_PrimitiveTypeId_FLOAT: return "Float";
1221+
case Ydb::Type_PrimitiveTypeId_DOUBLE: return "Double";
1222+
case Ydb::Type_PrimitiveTypeId_DATE: return "Date";
1223+
case Ydb::Type_PrimitiveTypeId_DATETIME: return "Datetime";
1224+
case Ydb::Type_PrimitiveTypeId_TIMESTAMP: return "Timestamp";
1225+
case Ydb::Type_PrimitiveTypeId_INTERVAL: return "Interval";
1226+
case Ydb::Type_PrimitiveTypeId_TZ_DATE: return "TzDate";
1227+
case Ydb::Type_PrimitiveTypeId_TZ_DATETIME: return "TzDatetime";
1228+
case Ydb::Type_PrimitiveTypeId_TZ_TIMESTAMP: return "TzTimestamp";
1229+
case Ydb::Type_PrimitiveTypeId_DATE32: return "Date32";
1230+
case Ydb::Type_PrimitiveTypeId_DATETIME64: return "Datetime64";
1231+
case Ydb::Type_PrimitiveTypeId_TIMESTAMP64: return "Timestamp64";
1232+
case Ydb::Type_PrimitiveTypeId_INTERVAL64: return "Interval64";
1233+
case Ydb::Type_PrimitiveTypeId_STRING: return "String";
1234+
case Ydb::Type_PrimitiveTypeId_UTF8: return "Utf8";
1235+
case Ydb::Type_PrimitiveTypeId_YSON: return "Yson";
1236+
case Ydb::Type_PrimitiveTypeId_JSON: return "Json";
1237+
case Ydb::Type_PrimitiveTypeId_UUID: return "Uuid";
1238+
case Ydb::Type_PrimitiveTypeId_JSON_DOCUMENT: return "JsonDocument";
1239+
case Ydb::Type_PrimitiveTypeId_DYNUMBER: return "DyNumber";
1240+
case Ydb::Type_PrimitiveTypeId_PRIMITIVE_TYPE_ID_UNSPECIFIED:
1241+
case Ydb::Type_PrimitiveTypeId_Type_PrimitiveTypeId_INT_MIN_SENTINEL_DO_NOT_USE_:
1242+
case Ydb::Type_PrimitiveTypeId_Type_PrimitiveTypeId_INT_MAX_SENTINEL_DO_NOT_USE_:
1243+
UNIT_FAIL("Unimplemented");
1244+
return "";
1245+
}
1246+
}
1247+
1248+
// sample values to insert into a table
1249+
std::string_view GetSampleValue(Ydb::Type::PrimitiveTypeId type) {
1250+
// date types need type casts
1251+
#define TYPE_CAST(Type, Initializer) "CAST(" #Initializer " AS " #Type ")"
1252+
#define TYPE_CONSTRUCTOR(Type, Initializer) #Type "(" #Initializer ")"
1253+
switch (type) {
1254+
case Ydb::Type_PrimitiveTypeId_BOOL: return "false";
1255+
case Ydb::Type_PrimitiveTypeId_INT8: return "0";
1256+
case Ydb::Type_PrimitiveTypeId_UINT8: return "0";
1257+
case Ydb::Type_PrimitiveTypeId_INT16: return "0";
1258+
case Ydb::Type_PrimitiveTypeId_UINT16: return "0";
1259+
case Ydb::Type_PrimitiveTypeId_INT32: return "0";
1260+
case Ydb::Type_PrimitiveTypeId_UINT32: return "0";
1261+
case Ydb::Type_PrimitiveTypeId_INT64: return "0";
1262+
case Ydb::Type_PrimitiveTypeId_UINT64: return "0";
1263+
case Ydb::Type_PrimitiveTypeId_FLOAT: return "0.0f";
1264+
case Ydb::Type_PrimitiveTypeId_DOUBLE: return "0.0";
1265+
case Ydb::Type_PrimitiveTypeId_DATE: return TYPE_CAST(Date, "2020-01-01");
1266+
case Ydb::Type_PrimitiveTypeId_DATETIME: return TYPE_CAST(Datetime, "2020-01-01");
1267+
case Ydb::Type_PrimitiveTypeId_TIMESTAMP: return TYPE_CAST(Timestamp, "2020-01-01");
1268+
case Ydb::Type_PrimitiveTypeId_INTERVAL: return TYPE_CAST(Interval, "P1H");
1269+
case Ydb::Type_PrimitiveTypeId_TZ_DATE: return TYPE_CAST(TzDate, "2020-01-01");
1270+
case Ydb::Type_PrimitiveTypeId_TZ_DATETIME: return TYPE_CAST(TzDatetime, "2020-01-01");
1271+
case Ydb::Type_PrimitiveTypeId_TZ_TIMESTAMP: return TYPE_CAST(TzTimestamp, "2020-01-01");
1272+
case Ydb::Type_PrimitiveTypeId_DATE32: return TYPE_CAST(Date32, "2020-01-01");
1273+
case Ydb::Type_PrimitiveTypeId_DATETIME64: return TYPE_CAST(Datetime64, "2020-01-01");
1274+
case Ydb::Type_PrimitiveTypeId_TIMESTAMP64: return TYPE_CAST(Timestamp64, "2020-01-01");
1275+
case Ydb::Type_PrimitiveTypeId_INTERVAL64: return TYPE_CAST(Interval64, "P1H");
1276+
case Ydb::Type_PrimitiveTypeId_STRING: return "\"foo\"";
1277+
case Ydb::Type_PrimitiveTypeId_UTF8: return "\"foo\"u";
1278+
case Ydb::Type_PrimitiveTypeId_YSON: return TYPE_CONSTRUCTOR(Yson, "{ foo = bar }");
1279+
case Ydb::Type_PrimitiveTypeId_JSON: return TYPE_CONSTRUCTOR(Json, "{ \"foo\": \"bar\" }");
1280+
case Ydb::Type_PrimitiveTypeId_UUID: return "RandomUuid(1)";
1281+
case Ydb::Type_PrimitiveTypeId_JSON_DOCUMENT: return TYPE_CONSTRUCTOR(JsonDocument, "{ \"foo\": \"bar\" }");
1282+
case Ydb::Type_PrimitiveTypeId_DYNUMBER: return TYPE_CONSTRUCTOR(DyNumber, "1");
1283+
case Ydb::Type_PrimitiveTypeId_PRIMITIVE_TYPE_ID_UNSPECIFIED:
1284+
case Ydb::Type_PrimitiveTypeId_Type_PrimitiveTypeId_INT_MIN_SENTINEL_DO_NOT_USE_:
1285+
case Ydb::Type_PrimitiveTypeId_Type_PrimitiveTypeId_INT_MAX_SENTINEL_DO_NOT_USE_:
1286+
UNIT_FAIL("Unimplemented");
1287+
return "";
1288+
}
1289+
#undef TYPE_CAST
1290+
#undef TYPE_CONSTRUCTOR
1291+
}
1292+
1293+
bool CanBePrimaryKey(Ydb::Type::PrimitiveTypeId type) {
1294+
switch (type) {
1295+
case Ydb::Type_PrimitiveTypeId_BOOL:
1296+
case Ydb::Type_PrimitiveTypeId_INT8:
1297+
case Ydb::Type_PrimitiveTypeId_UINT8:
1298+
case Ydb::Type_PrimitiveTypeId_INT16:
1299+
case Ydb::Type_PrimitiveTypeId_UINT16:
1300+
case Ydb::Type_PrimitiveTypeId_INT32:
1301+
case Ydb::Type_PrimitiveTypeId_UINT32:
1302+
case Ydb::Type_PrimitiveTypeId_INT64:
1303+
case Ydb::Type_PrimitiveTypeId_UINT64:
1304+
case Ydb::Type_PrimitiveTypeId_DATE:
1305+
case Ydb::Type_PrimitiveTypeId_DATETIME:
1306+
case Ydb::Type_PrimitiveTypeId_TIMESTAMP:
1307+
case Ydb::Type_PrimitiveTypeId_INTERVAL:
1308+
case Ydb::Type_PrimitiveTypeId_TZ_DATE:
1309+
case Ydb::Type_PrimitiveTypeId_TZ_DATETIME:
1310+
case Ydb::Type_PrimitiveTypeId_TZ_TIMESTAMP:
1311+
case Ydb::Type_PrimitiveTypeId_DATE32:
1312+
case Ydb::Type_PrimitiveTypeId_DATETIME64:
1313+
case Ydb::Type_PrimitiveTypeId_TIMESTAMP64:
1314+
case Ydb::Type_PrimitiveTypeId_INTERVAL64:
1315+
case Ydb::Type_PrimitiveTypeId_STRING:
1316+
case Ydb::Type_PrimitiveTypeId_UTF8:
1317+
case Ydb::Type_PrimitiveTypeId_UUID:
1318+
case Ydb::Type_PrimitiveTypeId_DYNUMBER:
1319+
return true;
1320+
case Ydb::Type_PrimitiveTypeId_FLOAT:
1321+
case Ydb::Type_PrimitiveTypeId_DOUBLE:
1322+
case Ydb::Type_PrimitiveTypeId_YSON:
1323+
case Ydb::Type_PrimitiveTypeId_JSON:
1324+
case Ydb::Type_PrimitiveTypeId_JSON_DOCUMENT:
1325+
return false;
1326+
case Ydb::Type_PrimitiveTypeId_PRIMITIVE_TYPE_ID_UNSPECIFIED:
1327+
case Ydb::Type_PrimitiveTypeId_Type_PrimitiveTypeId_INT_MIN_SENTINEL_DO_NOT_USE_:
1328+
case Ydb::Type_PrimitiveTypeId_Type_PrimitiveTypeId_INT_MAX_SENTINEL_DO_NOT_USE_:
1329+
UNIT_FAIL("Unimplemented");
1330+
return false;
1331+
}
1332+
}
1333+
1334+
bool DontTestThisType(Ydb::Type::PrimitiveTypeId type) {
1335+
switch (type) {
1336+
case Ydb::Type_PrimitiveTypeId_TZ_DATE:
1337+
case Ydb::Type_PrimitiveTypeId_TZ_DATETIME:
1338+
case Ydb::Type_PrimitiveTypeId_TZ_TIMESTAMP:
1339+
// CREATE TABLE with a column of this type is not supported by storage
1340+
return true;
1341+
case Ydb::Type_PrimitiveTypeId_PRIMITIVE_TYPE_ID_UNSPECIFIED:
1342+
case Ydb::Type_PrimitiveTypeId_Type_PrimitiveTypeId_INT_MIN_SENTINEL_DO_NOT_USE_:
1343+
case Ydb::Type_PrimitiveTypeId_Type_PrimitiveTypeId_INT_MAX_SENTINEL_DO_NOT_USE_:
1344+
// helper types
1345+
return true;
1346+
default:
1347+
return false;
1348+
}
1349+
}
1350+
1351+
auto GetTableName(std::string_view yqlType, std::string_view database = "/Root/") {
1352+
return std::format("{}{}Table", database, yqlType);
1353+
}
1354+
1355+
void TestPrimitiveType(
1356+
Ydb::Type::PrimitiveTypeId type, NQuery::TSession& session, TBackupFunction&& backup, TRestoreFunction&& restore
1357+
) {
1358+
const auto yqlType = GetYqlType(type);
1359+
const auto tableName = GetTableName(yqlType);
1360+
const auto sampleValue = GetSampleValue(type);
1361+
1362+
std::string_view key = sampleValue;
1363+
std::string_view value = "1";
1364+
if (CanBePrimaryKey(type)) {
1365+
ExecuteQuery(session, std::format(R"(
1366+
CREATE TABLE `{}` (Key {}, Value Int32, PRIMARY KEY (Key));
1367+
)", tableName, yqlType
1368+
), true);
1369+
} else {
1370+
{
1371+
// test if the type cannot in fact be a primary key to future-proof the test suite
1372+
const auto result = session.ExecuteQuery(std::format(R"(
1373+
CREATE TABLE `{}` (Key {}, Value Int32, PRIMARY KEY (Key));
1374+
)", tableName, yqlType
1375+
), NQuery::TTxControl::NoTx()).ExtractValueSync();
1376+
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SCHEME_ERROR, result.GetIssues().ToString());
1377+
}
1378+
ExecuteQuery(session, std::format(R"(
1379+
CREATE TABLE `{}` (Key Int32, Value {}, PRIMARY KEY (Key));
1380+
)", tableName, yqlType
1381+
), true);
1382+
std::swap(key, value);
1383+
}
1384+
ExecuteQuery(session, std::format(R"(
1385+
UPSERT INTO `{}` (Key, Value) VALUES ({}, {});
1386+
)", tableName, key, value
1387+
));
1388+
const auto originalTableContent = GetTableContent(session, tableName.c_str());
1389+
1390+
backup();
1391+
1392+
ExecuteQuery(session, std::format(R"(
1393+
DROP TABLE `{}`;
1394+
)", tableName
1395+
), true);
1396+
1397+
restore();
1398+
1399+
CompareResults(GetTableContent(session, tableName.c_str()), originalTableContent);
1400+
}
1401+
12121402
}
12131403

12141404
Y_UNIT_TEST_SUITE(BackupRestore) {
@@ -1404,7 +1594,7 @@ Y_UNIT_TEST_SUITE(BackupRestore) {
14041594

14051595
ExecuteDataModificationQuery(session, Sprintf(R"(
14061596
UPSERT INTO `%s` (Key, Value)
1407-
VALUES
1597+
VALUES
14081598
(Uuid("%s"), "one"),
14091599
(Uuid("%s"), "two"),
14101600
(Uuid("%s"), "three"),
@@ -1932,6 +2122,25 @@ Y_UNIT_TEST_SUITE(BackupRestore) {
19322122
Y_UNIT_TEST(PrefixedVectorIndex) {
19332123
TestTableWithIndexBackupRestore(NKikimrSchemeOp::EIndexTypeGlobalVectorKmeansTree, true);
19342124
}
2125+
2126+
Y_UNIT_TEST_ALL_PROTO_ENUM_VALUES(TestAllPrimitiveTypes, Ydb::Type::PrimitiveTypeId) {
2127+
if (DontTestThisType(Value)) {
2128+
return;
2129+
}
2130+
TKikimrWithGrpcAndRootSchema server;
2131+
auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%u", server.GetPort())));
2132+
NQuery::TQueryClient queryClient(driver);
2133+
auto session = queryClient.GetSession().ExtractValueSync().GetSession();
2134+
TTempDir tempDir;
2135+
const auto& pathToBackup = tempDir.Path();
2136+
2137+
TestPrimitiveType(
2138+
Value,
2139+
session,
2140+
CreateBackupLambda(driver, pathToBackup),
2141+
CreateRestoreLambda(driver, pathToBackup)
2142+
);
2143+
}
19352144
}
19362145

19372146
Y_UNIT_TEST_SUITE(BackupRestoreS3) {
@@ -2452,4 +2661,18 @@ Y_UNIT_TEST_SUITE(BackupRestoreS3) {
24522661
Y_UNIT_TEST(PrefixedVectorIndex) {
24532662
TestTableWithIndexBackupRestore(NKikimrSchemeOp::EIndexTypeGlobalVectorKmeansTree, true);
24542663
}
2664+
2665+
Y_UNIT_TEST_ALL_PROTO_ENUM_VALUES(TestAllPrimitiveTypes, Ydb::Type::PrimitiveTypeId) {
2666+
if (DontTestThisType(Value)) {
2667+
return;
2668+
}
2669+
TS3TestEnv testEnv;
2670+
2671+
TestPrimitiveType(
2672+
Value,
2673+
testEnv.GetQuerySession(),
2674+
CreateBackupLambda(testEnv.GetDriver(), testEnv.GetS3Port()),
2675+
CreateRestoreLambda(testEnv.GetDriver(), testEnv.GetS3Port(), { GetTableName(GetYqlType(Value), "") } )
2676+
);
2677+
}
24552678
}

0 commit comments

Comments
 (0)