Skip to content

Commit a48e948

Browse files
pixccpixcc
authored
Fix enable checksums persistance (#14757)
1 parent eb1eed1 commit a48e948

File tree

5 files changed

+107
-13
lines changed

5 files changed

+107
-13
lines changed

‎ydb/core/tx/schemeshard/schemeshard__init.cpp

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -803,7 +803,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
803803
return true;
804804
}
805805

806-
typedef std::tuple<TPathId, TString, TString, TString, TString, bool, TString, ui32> TBackupSettingsRec;
806+
typedef std::tuple<TPathId, TString, TString, TString, TString, bool, TString, ui32, bool> TBackupSettingsRec;
807807
typedef TDeque<TBackupSettingsRec> TBackupSettingsRows;
808808

809809
template <typename SchemaTable, typename TRowSet>
@@ -815,7 +815,8 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
815815
rowSet.template GetValueOrDefault<typename SchemaTable::ScanSettings>(""),
816816
rowSet.template GetValueOrDefault<typename SchemaTable::NeedToBill>(true),
817817
rowSet.template GetValueOrDefault<typename SchemaTable::TableDescription>(""),
818-
rowSet.template GetValueOrDefault<typename SchemaTable::NumberOfRetries>(0)
818+
rowSet.template GetValueOrDefault<typename SchemaTable::NumberOfRetries>(0),
819+
rowSet.template GetValueOrDefault<typename SchemaTable::EnableChecksums>(false)
819820
);
820821
}
821822

@@ -3791,6 +3792,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
37913792
bool needToBill = std::get<5>(rec);
37923793
TString tableDesc = std::get<6>(rec);
37933794
ui32 nRetries = std::get<7>(rec);
3795+
bool enableChecksums = std::get<8>(rec);
37943796

37953797
Y_ABORT_UNLESS(tableName.size() > 0);
37963798

@@ -3800,6 +3802,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
38003802
tableInfo->BackupSettings.SetTableName(tableName);
38013803
tableInfo->BackupSettings.SetNeedToBill(needToBill);
38023804
tableInfo->BackupSettings.SetNumberOfRetries(nRetries);
3805+
tableInfo->BackupSettings.SetEnableChecksums(enableChecksums);
38033806

38043807
if (ytSerializedSettings) {
38053808
auto settings = tableInfo->BackupSettings.MutableYTSettings();

‎ydb/core/tx/schemeshard/schemeshard_impl.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3360,15 +3360,17 @@ void TSchemeShard::PersistBackupSettings(
33603360
NIceDb::TUpdate<Schema::BackupSettings::ScanSettings>(settings.GetScanSettings().SerializeAsString()), \
33613361
NIceDb::TUpdate<Schema::BackupSettings::NeedToBill>(settings.GetNeedToBill()), \
33623362
NIceDb::TUpdate<Schema::BackupSettings::TableDescription>(settings.GetTable().SerializeAsString()), \
3363-
NIceDb::TUpdate<Schema::BackupSettings::NumberOfRetries>(settings.GetNumberOfRetries())); \
3363+
NIceDb::TUpdate<Schema::BackupSettings::NumberOfRetries>(settings.GetNumberOfRetries()), \
3364+
NIceDb::TUpdate<Schema::BackupSettings::EnableChecksums>(settings.GetEnableChecksums())); \
33643365
} else { \
33653366
db.Table<Schema::MigratedBackupSettings>().Key(pathId.OwnerId, pathId.LocalPathId).Update( \
33663367
NIceDb::TUpdate<Schema::MigratedBackupSettings::TableName>(settings.GetTableName()), \
33673368
NIceDb::TUpdate<Schema::MigratedBackupSettings::Kind>(settings.Get##Kind().SerializeAsString()), \
33683369
NIceDb::TUpdate<Schema::MigratedBackupSettings::ScanSettings>(settings.GetScanSettings().SerializeAsString()), \
33693370
NIceDb::TUpdate<Schema::MigratedBackupSettings::NeedToBill>(settings.GetNeedToBill()), \
33703371
NIceDb::TUpdate<Schema::MigratedBackupSettings::TableDescription>(settings.GetTable().SerializeAsString()), \
3371-
NIceDb::TUpdate<Schema::MigratedBackupSettings::NumberOfRetries>(settings.GetNumberOfRetries())); \
3372+
NIceDb::TUpdate<Schema::MigratedBackupSettings::NumberOfRetries>(settings.GetNumberOfRetries()), \
3373+
NIceDb::TUpdate<Schema::MigratedBackupSettings::EnableChecksums>(settings.GetEnableChecksums())); \
33723374
} \
33733375
}
33743376

‎ydb/core/tx/schemeshard/schemeshard_schema.h

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -583,6 +583,7 @@ struct Schema : NIceDb::Schema {
583583
struct NumberOfRetries : Column<8, NScheme::NTypeIds::Uint32> {};
584584
struct ScanSettings : Column<9, NScheme::NTypeIds::String> {};
585585
struct NeedToBill : Column<10, NScheme::NTypeIds::Bool> {};
586+
struct EnableChecksums : Column<11, NScheme::NTypeIds::Bool> {};
586587
// deprecated
587588
struct CreateDestinationFlag : Column<4, NScheme::NTypeIds::Bool> {};
588589
struct EraseOldDataFlag : Column<5, NScheme::NTypeIds::Bool> {};
@@ -598,7 +599,8 @@ struct Schema : NIceDb::Schema {
598599
TableDescription,
599600
NumberOfRetries,
600601
ScanSettings,
601-
NeedToBill
602+
NeedToBill,
603+
EnableChecksums
602604
>;
603605
};
604606

@@ -613,6 +615,7 @@ struct Schema : NIceDb::Schema {
613615
struct NumberOfRetries : Column<9, NScheme::NTypeIds::Uint32> {};
614616
struct ScanSettings : Column<10, NScheme::NTypeIds::String> {};
615617
struct NeedToBill : Column<11, NScheme::NTypeIds::Bool> {};
618+
struct EnableChecksums : Column<12, NScheme::NTypeIds::Bool> {};
616619
// deprecated
617620
struct CreateDestinationFlag : Column<5, NScheme::NTypeIds::Bool> {};
618621
struct EraseOldDataFlag : Column<6, NScheme::NTypeIds::Bool> {};
@@ -629,7 +632,8 @@ struct Schema : NIceDb::Schema {
629632
TableDescription,
630633
NumberOfRetries,
631634
ScanSettings,
632-
NeedToBill
635+
NeedToBill,
636+
EnableChecksums
633637
>;
634638
};
635639

‎ydb/core/tx/schemeshard/ut_export/ut_export.cpp

Lines changed: 78 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,13 @@
1+
#include <ydb/core/metering/metering.h>
12
#include <ydb/core/protos/schemeshard/operations.pb.h>
23
#include <ydb/core/tablet_flat/shared_cache_events.h>
3-
#include <ydb/core/tx/schemeshard/ut_helpers/helpers.h>
4-
#include <ydb/core/tx/schemeshard/ut_helpers/auditlog_helpers.h>
5-
#include <ydb/core/tx/schemeshard/schemeshard_billing_helpers.h>
4+
#include <ydb/core/testlib/actors/block_events.h>
65
#include <ydb/core/tx/datashard/datashard.h>
7-
#include <ydb/core/wrappers/ut_helpers/s3_mock.h>
6+
#include <ydb/core/tx/schemeshard/schemeshard_billing_helpers.h>
7+
#include <ydb/core/tx/schemeshard/ut_helpers/auditlog_helpers.h>
8+
#include <ydb/core/tx/schemeshard/ut_helpers/helpers.h>
89
#include <ydb/core/wrappers/s3_wrapper.h>
9-
#include <ydb/core/metering/metering.h>
10+
#include <ydb/core/wrappers/ut_helpers/s3_mock.h>
1011
#include <ydb/public/api/protos/ydb_export.pb.h>
1112

1213
#include <util/string/builder.h>
@@ -2386,6 +2387,78 @@ partitioning_settings {
23862387
)", port));
23872388
env.TestWaitNotification(runtime, txId);
23882389

2390+
UNIT_ASSERT_VALUES_EQUAL(s3Mock.GetData().size(), 8);
2391+
const auto* dataChecksum = s3Mock.GetData().FindPtr("/data_00.csv.sha256");
2392+
UNIT_ASSERT(dataChecksum);
2393+
UNIT_ASSERT_VALUES_EQUAL(*dataChecksum, "19dcd641390a61063ee45f3e6e06b8f0d3acfc33f934b9bf1ba204668a98f21d data_00.csv");
2394+
2395+
const auto* metadataChecksum = s3Mock.GetData().FindPtr("/metadata.json.sha256");
2396+
UNIT_ASSERT(metadataChecksum);
2397+
UNIT_ASSERT_VALUES_EQUAL(*metadataChecksum, "b72575244ae0cce8dffd45f3537d1e412bfe39de4268f4f85f529cb529870903 metadata.json");
2398+
2399+
const auto* schemeChecksum = s3Mock.GetData().FindPtr("/scheme.pb.sha256");
2400+
UNIT_ASSERT(schemeChecksum);
2401+
UNIT_ASSERT_VALUES_EQUAL(*schemeChecksum, "cb1fb80965ae92e6369acda2b3b5921fd5518c97d6437f467ce00492907f9eb6 scheme.pb");
2402+
2403+
const auto* permissionsChecksum = s3Mock.GetData().FindPtr("/permissions.pb.sha256");
2404+
UNIT_ASSERT(permissionsChecksum);
2405+
UNIT_ASSERT_VALUES_EQUAL(*permissionsChecksum, "b41fd8921ff3a7314d9c702dc0e71aace6af8443e0102add0432895c5e50a326 permissions.pb");
2406+
}
2407+
2408+
Y_UNIT_TEST(EnableChecksumsPersistance) {
2409+
TTestBasicRuntime runtime;
2410+
TTestEnv env(runtime);
2411+
ui64 txId = 100;
2412+
2413+
// Create test table
2414+
TestCreateTable(runtime, ++txId, "/MyRoot", R"(
2415+
Name: "Table"
2416+
Columns { Name: "key" Type: "Utf8" }
2417+
Columns { Name: "value" Type: "Utf8" }
2418+
KeyColumnNames: ["key"]
2419+
)");
2420+
env.TestWaitNotification(runtime, txId);
2421+
2422+
// Add some test data
2423+
UploadRow(runtime, "/MyRoot/Table", 0, {1}, {2}, {TCell::Make(1u)}, {TCell::Make(1u)});
2424+
2425+
TPortManager portManager;
2426+
const ui16 port = portManager.GetPort();
2427+
2428+
TS3Mock s3Mock({}, TS3Mock::TSettings(port));
2429+
UNIT_ASSERT(s3Mock.Start());
2430+
2431+
// Block sending backup task to datashards
2432+
TBlockEvents<TEvDataShard::TEvProposeTransaction> block(runtime, [](auto& ev) {
2433+
NKikimrTxDataShard::TFlatSchemeTransaction schemeTx;
2434+
UNIT_ASSERT(schemeTx.ParseFromString(ev.Get()->Get()->GetTxBody()));
2435+
return schemeTx.HasBackup();
2436+
});
2437+
2438+
// Start export and expect it to be blocked
2439+
TestExport(runtime, ++txId, "/MyRoot", Sprintf(R"(
2440+
ExportToS3Settings {
2441+
endpoint: "localhost:%d"
2442+
scheme: HTTP
2443+
items {
2444+
source_path: "/MyRoot/Table"
2445+
destination_prefix: ""
2446+
}
2447+
}
2448+
)", port));
2449+
2450+
runtime.WaitFor("backup task is sent to datashards", [&]{ return block.size() >= 1; });
2451+
2452+
// Stop blocking new events
2453+
block.Stop();
2454+
2455+
// Reboot SchemeShard to resend backup task
2456+
RebootTablet(runtime, TTestTxConfig::SchemeShard, runtime.AllocateEdgeActor());
2457+
2458+
// Wait for export to complete
2459+
env.TestWaitNotification(runtime, txId);
2460+
2461+
// Verify checksums are created
23892462
UNIT_ASSERT_VALUES_EQUAL(s3Mock.GetData().size(), 8);
23902463

23912464
const auto* dataChecksum = s3Mock.GetData().FindPtr("/data_00.csv.sha256");

‎ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1169,6 +1169,11 @@
11691169
"ColumnId": 10,
11701170
"ColumnName": "NeedToBill",
11711171
"ColumnType": "Bool"
1172+
},
1173+
{
1174+
"ColumnId": 11,
1175+
"ColumnName": "EnableChecksums",
1176+
"ColumnType": "Bool"
11721177
}
11731178
],
11741179
"ColumnsDropped": [],
@@ -1184,7 +1189,8 @@
11841189
7,
11851190
8,
11861191
9,
1187-
10
1192+
10,
1193+
11
11881194
],
11891195
"RoomID": 0,
11901196
"Codec": 0,
@@ -4549,6 +4555,11 @@
45494555
"ColumnId": 11,
45504556
"ColumnName": "NeedToBill",
45514557
"ColumnType": "Bool"
4558+
},
4559+
{
4560+
"ColumnId": 12,
4561+
"ColumnName": "EnableChecksums",
4562+
"ColumnType": "Bool"
45524563
}
45534564
],
45544565
"ColumnsDropped": [],
@@ -4565,7 +4576,8 @@
45654576
8,
45664577
9,
45674578
10,
4568-
11
4579+
11,
4580+
12
45694581
],
45704582
"RoomID": 0,
45714583
"Codec": 0,

0 commit comments

Comments
 (0)