Skip to content

Commit 23d3161

Browse files
EnjectionEnjection
authored
Add backup collection schema path item (#10252)
1 parent 374d43d commit 23d3161

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+1549
-61
lines changed

‎ydb/core/driver_lib/cli_base/cli_cmds_db.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,9 @@ class TClientCommandSchemaDescribe : public TClientCommand {
275275
case NKikimrSchemeOp::EPathTypePersQueueGroup:
276276
type = "<pq group>";
277277
break;
278+
case NKikimrSchemeOp::EPathTypeBackupCollection:
279+
type = "<backup collection>";
280+
break;
278281
default:
279282
type = "<unknown>";
280283
break;
@@ -487,6 +490,9 @@ class TClientCommandSchemaLs : public TClientCommand {
487490
case NKikimrSchemeOp::EPathTypeReplication:
488491
type = "<replication>";
489492
break;
493+
case NKikimrSchemeOp::EPathTypeBackupCollection:
494+
type = "<backup collection>";
495+
break;
490496
default:
491497
type = "<unknown>";
492498
break;

‎ydb/core/protos/counters_schemeshard.proto

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,10 @@ enum ESimpleCounters {
222222
COUNTER_IN_FLIGHT_OPS_TxAlterResourcePool = 176 [(CounterOpts) = {Name: "InFlightOps/AlterResourcePool"}];
223223

224224
COUNTER_IN_FLIGHT_OPS_TxRestoreIncrementalBackupAtTable = 177 [(CounterOpts) = {Name: "InFlightOps/RestoreIncrementalBackupAtTable"}];
225+
COUNTER_BACKUP_COLLECTION_COUNT = 178 [(CounterOpts) = {Name: "BackupCollectionCount"}];
226+
COUNTER_IN_FLIGHT_OPS_TxCreateBackupCollection = 179 [(CounterOpts) = {Name: "InFlightOps/CreateBackupCollection"}];
227+
COUNTER_IN_FLIGHT_OPS_TxAlterBackupCollection = 180 [(CounterOpts) = {Name: "InFlightOps/AlterBackupCollection"}];
228+
COUNTER_IN_FLIGHT_OPS_TxDropBackupCollection = 181 [(CounterOpts) = {Name: "InFlightOps/DropBackupCollection"}];
225229
}
226230

227231
enum ECumulativeCounters {
@@ -355,6 +359,10 @@ enum ECumulativeCounters {
355359
COUNTER_FINISHED_OPS_TxAlterResourcePool = 105 [(CounterOpts) = {Name: "FinishedOps/AlterResourcePool"}];
356360

357361
COUNTER_FINISHED_OPS_TxRestoreIncrementalBackupAtTable = 106 [(CounterOpts) = {Name: "FinishedOps/RestoreIncrementalBackupAtTable"}];
362+
363+
COUNTER_FINISHED_OPS_TxCreateBackupCollection = 107 [(CounterOpts) = {Name: "FinishedOps/CreateBackupCollection"}];
364+
COUNTER_FINISHED_OPS_TxAlterBackupCollection = 108 [(CounterOpts) = {Name: "FinishedOps/AlterBackupCollection"}];
365+
COUNTER_FINISHED_OPS_TxDropBackupCollection = 109 [(CounterOpts) = {Name: "FinishedOps/DropBackupCollection"}];
358366
}
359367

360368
enum EPercentileCounters {
@@ -578,4 +586,8 @@ enum ETxTypes {
578586
TXTYPE_UPDATE_DOMAIN_REPLY = 85 [(TxTypeOpts) = {Name: "TxUpdateDomainReply"}];
579587

580588
TXTYPE_SEQUENCESHARD_GET_SEQUENCE_RESULT = 86 [(TxTypeOpts) = {Name: "TxSequenceShardGetSequenceResult"}];
589+
590+
TXTYPE_CREATE_BACKUP_COLLECTION_RESULT = 87 [(TxTypeOpts) = {Name: "TxCreateBackupCollectionResult"}];
591+
TXTYPE_ALTER_BACKUP_COLLECTION_RESULT = 88 [(TxTypeOpts) = {Name: "TxAlterBackupCollectionResult"}];
592+
TXTYPE_DROP_BACKUP_COLLECTION_RESULT = 89 [(TxTypeOpts) = {Name: "TxDropBackupCollectionResult"}];
581593
}

‎ydb/core/protos/flat_scheme_op.proto

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import "ydb/core/tx/columnshard/engines/scheme/defaults/protos/data.proto";
2323
import "ydb/core/tx/columnshard/common/protos/snapshot.proto";
2424
import "ydb/library/formats/arrow/protos/accessor.proto";
2525

26+
import "google/protobuf/empty.proto";
2627
import "google/protobuf/struct.proto";
2728

2829
package NKikimrSchemeOp;
@@ -1669,6 +1670,11 @@ enum EOperationType {
16691670

16701671
ESchemeOpRestoreIncrementalBackup = 103;
16711672
ESchemeOpRestoreIncrementalBackupAtTable = 104;
1673+
1674+
// BackupCollection
1675+
ESchemeOpCreateBackupCollection = 105;
1676+
ESchemeOpAlterBackupCollection = 106;
1677+
ESchemeOpDropBackupCollection = 107;
16721678
}
16731679

16741680
message TApplyIf {
@@ -1852,6 +1858,10 @@ message TModifyScheme {
18521858
optional TResourcePoolDescription CreateResourcePool = 72;
18531859

18541860
optional TRestoreIncrementalBackup RestoreIncrementalBackup = 73;
1861+
1862+
optional TBackupCollectionDescription CreateBackupCollection = 74;
1863+
optional TBackupCollectionDescription AlterBackupCollection = 75;
1864+
optional TBackupCollectionDescription DropBackupCollection = 76;
18551865
}
18561866

18571867
message TCopySequence {
@@ -1917,6 +1927,7 @@ enum EPathType {
19171927
EPathTypeExternalDataSource = 19;
19181928
EPathTypeView = 20;
19191929
EPathTypeResourcePool = 21;
1930+
EPathTypeBackupCollection = 22;
19201931
}
19211932

19221933
enum EPathSubType {
@@ -1973,6 +1984,7 @@ message TPathVersion {
19731984
optional uint64 ExternalDataSourceVersion = 28;
19741985
optional uint64 ViewVersion = 29;
19751986
optional uint64 ResourcePoolVersion = 30;
1987+
optional uint64 BackupCollectionVersion = 31;
19761988
}
19771989

19781990
// Describes single path
@@ -2062,6 +2074,7 @@ message TPathDescription {
20622074
optional TExternalDataSourceDescription ExternalDataSourceDescription = 28;
20632075
optional TViewDescription ViewDescription = 29;
20642076
optional TResourcePoolDescription ResourcePoolDescription = 30;
2077+
optional TBackupCollectionDescription BackupCollectionDescription = 31;
20652078
}
20662079

20672080
// For persisting AlterTable Tx description in Schemeshard internal DB
@@ -2202,3 +2215,39 @@ message TResourcePoolDescription {
22022215
optional uint64 Version = 3;
22032216
optional TResourcePoolProperties Properties = 4;
22042217
}
2218+
2219+
message TBackupCollectionDescription {
2220+
optional string Name = 1;
2221+
optional NKikimrProto.TPathID PathId = 2;
2222+
optional uint64 Version = 3;
2223+
2224+
message TBackupEntry {
2225+
enum EType {
2226+
ETypeInvalid = 0;
2227+
ETypeTable = 1;
2228+
}
2229+
2230+
optional EType Type = 1;
2231+
optional string Path = 2;
2232+
}
2233+
2234+
message TExplicitEntryList {
2235+
repeated TBackupEntry Entries = 1;
2236+
}
2237+
2238+
message TIncrementalBackupConfig {
2239+
2240+
}
2241+
2242+
oneof Entries {
2243+
google.protobuf.Empty Database = 4;
2244+
TExplicitEntryList ExplicitEntryList = 5;
2245+
}
2246+
2247+
// non-empty enables data-collection
2248+
optional TIncrementalBackupConfig IncrementalBackupConfig = 6;
2249+
2250+
oneof Storage {
2251+
google.protobuf.Empty Cluster = 7;
2252+
}
2253+
}

‎ydb/core/tx/scheme_board/cache.cpp

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,7 @@ namespace {
244244
entry.BlobDepotInfo.Drop();
245245
entry.BlockStoreVolumeInfo.Drop();
246246
entry.FileStoreInfo.Drop();
247+
entry.BackupCollectionInfo.Drop();
247248
}
248249

249250
static void SetErrorAndClear(TResolveContext* context, TResolve::TEntry& entry, const bool isDescribeDenied) {
@@ -763,6 +764,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
763764
FileStoreInfo.Drop();
764765
ViewInfo.Drop();
765766
ResourcePoolInfo.Drop();
767+
BackupCollectionInfo.Drop();
766768
}
767769

768770
void FillTableInfo(const NKikimrSchemeOp::TPathDescription& pathDesc) {
@@ -1278,6 +1280,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
12781280
DESCRIPTION_PART(FileStoreInfo);
12791281
DESCRIPTION_PART(ViewInfo);
12801282
DESCRIPTION_PART(ResourcePoolInfo);
1283+
DESCRIPTION_PART(BackupCollectionInfo);
12811284

12821285
#undef DESCRIPTION_PART
12831286

@@ -1610,6 +1613,10 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
16101613
Kind = TNavigate::KindResourcePool;
16111614
FillInfo(Kind, ResourcePoolInfo, std::move(*pathDesc.MutableResourcePoolDescription()));
16121615
break;
1616+
case NKikimrSchemeOp::EPathTypeBackupCollection:
1617+
Kind = TNavigate::KindBackupCollection;
1618+
FillInfo(Kind, BackupCollectionInfo, std::move(*pathDesc.MutableBackupCollectionDescription()));
1619+
break;
16131620
case NKikimrSchemeOp::EPathTypeInvalid:
16141621
Y_DEBUG_ABORT("Invalid path type");
16151622
break;
@@ -1683,6 +1690,9 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
16831690
case NKikimrSchemeOp::EPathTypeResourcePool:
16841691
ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindResourcePool);
16851692
break;
1693+
case NKikimrSchemeOp::EPathTypeBackupCollection:
1694+
ListNodeEntry->Children.emplace_back(name, pathId, TNavigate::KindBackupCollection);
1695+
break;
16861696
case NKikimrSchemeOp::EPathTypeTableIndex:
16871697
case NKikimrSchemeOp::EPathTypeInvalid:
16881698
Y_DEBUG_ABORT("Invalid path type");
@@ -1904,6 +1914,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
19041914
entry.FileStoreInfo = FileStoreInfo;
19051915
entry.ViewInfo = ViewInfo;
19061916
entry.ResourcePoolInfo = ResourcePoolInfo;
1917+
entry.BackupCollectionInfo = BackupCollectionInfo;
19071918
}
19081919

19091920
bool CheckColumns(TResolveContext* context, TResolve::TEntry& entry,
@@ -2202,6 +2213,8 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> {
22022213
// ResourcePool specific
22032214
TIntrusivePtr<TNavigate::TResourcePoolInfo> ResourcePoolInfo;
22042215

2216+
// BackupCollection specific
2217+
TIntrusivePtr<TNavigate::TBackupCollectionInfo> BackupCollectionInfo;
22052218
}; // TCacheItem
22062219

22072220
struct TMerger {

‎ydb/core/tx/scheme_cache/scheme_cache.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,7 @@ struct TSchemeCacheNavigate {
159159
KindFileStore = 20,
160160
KindView = 21,
161161
KindResourcePool = 22,
162+
KindBackupCollection = 23,
162163
};
163164

164165
struct TListNodeEntry : public TAtomicRefCount<TListNodeEntry> {
@@ -275,6 +276,11 @@ struct TSchemeCacheNavigate {
275276
NKikimrSchemeOp::TResourcePoolDescription Description;
276277
};
277278

279+
struct TBackupCollectionInfo : public TAtomicRefCount<TBackupCollectionInfo> {
280+
EKind Kind = KindUnknown;
281+
NKikimrSchemeOp::TBackupCollectionDescription Description;
282+
};
283+
278284
struct TEntry {
279285
enum class ERequestType : ui8 {
280286
ByPath,
@@ -327,6 +333,7 @@ struct TSchemeCacheNavigate {
327333
TIntrusiveConstPtr<TFileStoreInfo> FileStoreInfo;
328334
TIntrusiveConstPtr<TViewInfo> ViewInfo;
329335
TIntrusiveConstPtr<TResourcePoolInfo> ResourcePoolInfo;
336+
TIntrusiveConstPtr<TBackupCollectionInfo> BackupCollectionInfo;
330337

331338
TString ToString() const;
332339
TString ToString(const NScheme::TTypeRegistry& typeRegistry) const;

‎ydb/core/tx/schemeshard/schemeshard__backup_collection_common.cpp

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
#include "schemeshard__backup_collection_common.h"
2+
3+
namespace NKikimr::NSchemeShard {
4+
5+
std::optional<TBackupCollectionPaths> ResolveBackupCollectionPaths(
6+
const TString& rootPathStr,
7+
const TString& name,
8+
bool validateFeatureFlag,
9+
TOperationContext& context,
10+
THolder<TProposeResponse>& result)
11+
{
12+
bool backupServiceEnabled = AppData()->FeatureFlags.GetEnableBackupService();
13+
if (!backupServiceEnabled && validateFeatureFlag) {
14+
result->SetError(NKikimrScheme::StatusPreconditionFailed, "Backup collections are disabled. Please contact your system administrator to enable it");
15+
return std::nullopt;
16+
}
17+
18+
const TPath& rootPath = TPath::Resolve(rootPathStr, context.SS);
19+
{
20+
const auto checks = rootPath.Check();
21+
checks
22+
.NotEmpty()
23+
.NotUnderDomainUpgrade()
24+
.IsAtLocalSchemeShard()
25+
.IsResolved()
26+
.NotDeleted()
27+
.NotUnderDeleting()
28+
.IsCommonSensePath()
29+
.IsLikeDirectory()
30+
.FailOnRestrictedCreateInTempZone();
31+
32+
if (!checks) {
33+
result->SetError(checks.GetStatus(), checks.GetError());
34+
return std::nullopt;
35+
}
36+
}
37+
38+
const TString& backupCollectionsDir = JoinPath({rootPath.GetDomainPathString(), ".backups/collections"});
39+
40+
TPathSplitUnix absPathSplit(name);
41+
42+
if (absPathSplit.size() > 1 && !absPathSplit.IsAbsolute) {
43+
result->SetError(NKikimrScheme::EStatus::StatusSchemeError, TStringBuilder() << "Backup collections must be placed directly in " << backupCollectionsDir);
44+
return std::nullopt;
45+
}
46+
47+
const TPath& backupCollectionsPath = TPath::Resolve(backupCollectionsDir, context.SS);
48+
{
49+
const auto checks = backupCollectionsPath.Check();
50+
checks.NotUnderDomainUpgrade()
51+
.IsAtLocalSchemeShard()
52+
.IsResolved()
53+
.NotDeleted()
54+
.NotUnderDeleting()
55+
.IsCommonSensePath()
56+
.IsLikeDirectory();
57+
58+
if (!checks) {
59+
result->SetError(checks.GetStatus(), checks.GetError());
60+
return std::nullopt;
61+
}
62+
}
63+
64+
std::optional<TPath> parentPath;
65+
if (absPathSplit.size() > 1) {
66+
TString realParent = "/" + JoinRange("/", absPathSplit.begin(), absPathSplit.end() - 1);
67+
parentPath = TPath::Resolve(realParent, context.SS);
68+
}
69+
70+
TPath dstPath = absPathSplit.IsAbsolute && parentPath ? parentPath->Child(TString(absPathSplit.back())) : rootPath.Child(name);
71+
if (!dstPath.PathString().StartsWith(backupCollectionsDir + "/")) {
72+
result->SetError(NKikimrScheme::EStatus::StatusSchemeError, TStringBuilder() << "Backup collections must be placed in " << backupCollectionsDir);
73+
return std::nullopt;
74+
}
75+
76+
return TBackupCollectionPaths{rootPath, dstPath};
77+
}
78+
79+
} // namespace NKikimr::NSchemeShard

‎ydb/core/tx/schemeshard/schemeshard__backup_collection_common.h

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#pragma once
2+
3+
#include "schemeshard__operation_common.h"
4+
#include "schemeshard_impl.h"
5+
6+
#include <util/generic/string.h>
7+
8+
#include <optional>
9+
10+
namespace NKikimr::NSchemeShard {
11+
12+
struct TBackupCollectionPaths {
13+
TPath RootPath;
14+
TPath DstPath;
15+
};
16+
17+
std::optional<TBackupCollectionPaths> ResolveBackupCollectionPaths(
18+
const TString& rootPathStr,
19+
const TString& name,
20+
bool preValidateDst,
21+
TOperationContext& context,
22+
THolder<TProposeResponse>& result);
23+
24+
} // namespace NKikimr::NSchemeShard

‎ydb/core/tx/schemeshard/schemeshard__init.cpp

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1956,6 +1956,29 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
19561956
}
19571957
}
19581958

1959+
// Read backup collections
1960+
{
1961+
auto rowset = db.Table<Schema::BackupCollection>().Range().Select();
1962+
if (!rowset.IsReady()) {
1963+
return false;
1964+
}
1965+
1966+
while (!rowset.EndOfSet()) {
1967+
TOwnerId ownerPathId = rowset.GetValue<Schema::BackupCollection::OwnerPathId>();
1968+
TLocalPathId localPathId = rowset.GetValue<Schema::BackupCollection::LocalPathId>();
1969+
TPathId pathId(ownerPathId, localPathId);
1970+
1971+
auto& backupCollection = Self->BackupCollections[pathId] = new TBackupCollectionInfo();
1972+
backupCollection->AlterVersion = rowset.GetValue<Schema::BackupCollection::AlterVersion>();
1973+
Y_PROTOBUF_SUPPRESS_NODISCARD backupCollection->Description.ParseFromString(rowset.GetValue<Schema::BackupCollection::Description>());
1974+
Self->IncrementPathDbRefCount(pathId);
1975+
1976+
if (!rowset.Next()) {
1977+
return false;
1978+
}
1979+
}
1980+
}
1981+
19591982
// Read table columns
19601983
{
19611984
TColumnRows columnRows;

0 commit comments

Comments
 (0)