Skip to content

Commit 4330ae8

Browse files
SammyVimesSammyVimes
authored
Multiple channels for external blobs (#10998)
1 parent 8a77b3d commit 4330ae8

File tree

48 files changed

+1630
-312
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+1630
-312
lines changed

‎ydb/core/base/storage_pools.h

Lines changed: 48 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
#include <util/system/types.h>
77
#include <util/generic/vector.h>
88
#include <util/generic/map.h>
9+
#include <util/generic/set.h>
910

1011
namespace NKikimr {
1112

@@ -40,6 +41,7 @@ struct TStorageRoom: public TThrRefBase {
4041
private:
4142
ui32 RoomId;
4243
TMap<EPurpose, ui32> Purposes;
44+
TSet<ui32> ExternalChannels;
4345

4446
public:
4547
TStorageRoom(ui32 id)
@@ -49,14 +51,29 @@ struct TStorageRoom: public TThrRefBase {
4951
TStorageRoom(const NKikimrStorageSettings::TStorageRoom& room)
5052
: RoomId(room.GetRoomId())
5153
{
52-
for (auto& explanation: room.GetExplanation()) {
53-
AssignChannel(explanation.GetPurpose(), explanation.GetChannel());
54+
if (room.ChannelsSize() > 0) {
55+
for (auto& explanation: room.GetChannels()) {
56+
AssignChannel(explanation.GetPurpose(), explanation.GetChannel());
57+
}
58+
} else {
59+
// Fallback for old format
60+
for (auto& explanation: room.GetExplanation()) {
61+
AssignChannel(explanation.GetPurpose(), explanation.GetChannel());
62+
}
5463
}
5564
}
5665

5766
void AssignChannel(EPurpose purpose, ui32 channel) {
58-
bool repeated = Purposes.emplace(purpose, channel).second;
59-
Y_ABORT_UNLESS(repeated, "reassign is forbided, channle purpose was %s chennel id was %d",
67+
if (purpose == EPurpose::TChannelPurpose_EPurpose_External) {
68+
bool inserted = ExternalChannels.emplace(channel).second;
69+
Y_ABORT_UNLESS(inserted, "reassign is forbided, channel purpose was %s channel id was %d",
70+
NKikimrStorageSettings::TChannelPurpose::EPurpose_Name(purpose).c_str(),
71+
channel);
72+
return;
73+
}
74+
75+
bool inserted = Purposes.emplace(purpose, channel).second;
76+
Y_ABORT_UNLESS(inserted, "reassign is forbided, channel purpose was %s channel id was %d",
6077
NKikimrStorageSettings::TChannelPurpose::EPurpose_Name(purpose).c_str(),
6178
channel);
6279
}
@@ -71,13 +88,36 @@ struct TStorageRoom: public TThrRefBase {
7188
return it->second;
7289
}
7390

91+
const TSet<ui32> GetExternalChannels(ui32 defaultChannel) const {
92+
if (ExternalChannels.empty()) {
93+
return {defaultChannel};
94+
}
95+
96+
return ExternalChannels;
97+
}
98+
7499
operator NKikimrStorageSettings::TStorageRoom() const {
75100
NKikimrStorageSettings::TStorageRoom room;
76101
room.SetRoomId(RoomId);
77102
for (auto& item: Purposes) {
78-
auto layout = room.AddExplanation();
79-
layout->SetChannel(item.second);
80-
layout->SetPurpose(item.first);
103+
auto newLayout = room.AddChannels();
104+
newLayout->SetChannel(item.second);
105+
newLayout->SetPurpose(item.first);
106+
107+
auto oldLayout = room.AddExplanation();
108+
oldLayout->SetChannel(item.second);
109+
oldLayout->SetPurpose(item.first);
110+
}
111+
if (!ExternalChannels.empty()) {
112+
const auto& externalChannel = ExternalChannels.begin();
113+
auto oldLayout = room.AddExplanation();
114+
oldLayout->SetChannel(*externalChannel);
115+
oldLayout->SetPurpose(EPurpose::TChannelPurpose_EPurpose_External);
116+
}
117+
for (auto& item: ExternalChannels) {
118+
auto newLayout = room.AddChannels();
119+
newLayout->SetChannel(item);
120+
newLayout->SetPurpose(EPurpose::TChannelPurpose_EPurpose_External);
81121
}
82122
return room;
83123
}
@@ -87,7 +127,7 @@ struct TStorageRoom: public TThrRefBase {
87127
}
88128

89129
operator bool() const {
90-
return !!Purposes;
130+
return !!Purposes || !!ExternalChannels;
91131
}
92132
};
93133

‎ydb/core/base/tablet.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -375,6 +375,7 @@ struct TEvTablet {
375375
const ui32 ConfirmedOnSend;
376376
TVector<ui32> YellowMoveChannels;
377377
TVector<ui32> YellowStopChannels;
378+
THashMap<ui32, float> ApproximateFreeSpaceShareByChannel;
378379
NMetrics::TTabletThroughputRawValue GroupWrittenBytes;
379380
NMetrics::TTabletIopsRawValue GroupWrittenOps;
380381

@@ -386,6 +387,7 @@ struct TEvTablet {
386387
ui32 confirmedOnSend,
387388
TVector<ui32>&& yellowMoveChannels,
388389
TVector<ui32>&& yellowStopChannels,
390+
THashMap<ui32, float>&& approximateFreeSpaceShareByChannel,
389391
NMetrics::TTabletThroughputRawValue&& written,
390392
NMetrics::TTabletIopsRawValue&& writtenOps)
391393
: Status(status)
@@ -395,6 +397,7 @@ struct TEvTablet {
395397
, ConfirmedOnSend(confirmedOnSend)
396398
, YellowMoveChannels(std::move(yellowMoveChannels))
397399
, YellowStopChannels(std::move(yellowStopChannels))
400+
, ApproximateFreeSpaceShareByChannel(std::move(approximateFreeSpaceShareByChannel))
398401
, GroupWrittenBytes(std::move(written))
399402
, GroupWrittenOps(std::move(writtenOps))
400403
{}

‎ydb/core/protos/channel_purpose.proto

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,5 +14,6 @@ message TChannelPurpose {
1414

1515
message TStorageRoom {
1616
optional uint32 RoomId = 1;
17-
repeated TChannelPurpose Explanation = 2;
17+
repeated TChannelPurpose Explanation = 2 [deprecated = true];
18+
repeated TChannelPurpose Channels = 3;
1819
}

‎ydb/core/protos/flat_scheme_op.proto

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ message TStorageConfig {
114114
optional TStorageSettings External = 4;
115115
optional uint32 DataThreshold = 5;
116116
optional uint32 ExternalThreshold = 6;
117+
optional uint32 ExternalChannelsCount = 7 [default = 1];
117118
}
118119

119120
message TKeyValueStorageConfig {

‎ydb/core/protos/scheme_log.proto

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,9 @@ message TAlterRecord {
4343
//_ Table: Attributes of storage (page collection)
4444

4545
optional uint32 Main = 9; // Default channel for main data
46-
optional uint32 Blobs = 15; // Channel for external blobs
46+
optional uint32 Blobs = 15 [deprecated = true]; // Channel for external blobs
4747
optional uint32 Outer = 19; // Channel for outer packed values
48+
repeated uint32 ExternalBlobs = 26; // Channels for external blobs
4849

4950
//_ Table: Attributes of family columns
5051

‎ydb/core/tablet/tablet_impl.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ struct TEvTabletBase {
109109
const TLogoBlobID EntryId;
110110
TVector<ui32> YellowMoveChannels;
111111
TVector<ui32> YellowStopChannels;
112+
THashMap<ui32, float> ApproximateFreeSpaceShareByChannel;
112113
NMetrics::TTabletThroughputRawValue GroupWrittenBytes;
113114
NMetrics::TTabletIopsRawValue GroupWrittenOps;
114115
const TString ErrorReason;
@@ -122,13 +123,15 @@ struct TEvTabletBase {
122123
const TLogoBlobID &entryId,
123124
TVector<ui32>&& yellowMoveChannels,
124125
TVector<ui32>&& yellowStopChannels,
126+
THashMap<ui32, float>&& approximateFreeSpaceShareByChannel,
125127
NMetrics::TTabletThroughputRawValue&& written,
126128
NMetrics::TTabletIopsRawValue&& writtenOps,
127129
const TString &reason = TString())
128130
: Status(status)
129131
, EntryId(entryId)
130132
, YellowMoveChannels(std::move(yellowMoveChannels))
131133
, YellowStopChannels(std::move(yellowStopChannels))
134+
, ApproximateFreeSpaceShareByChannel(std::move(approximateFreeSpaceShareByChannel))
132135
, GroupWrittenBytes(std::move(written))
133136
, GroupWrittenOps(std::move(writtenOps))
134137
, ErrorReason(reason)

‎ydb/core/tablet/tablet_req_writelog.cpp

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ class TTabletReqWriteLog : public TActorBootstrapped<TTabletReqWriteLog> {
2727
ui32 RepliesToWait;
2828
TVector<ui32> YellowMoveChannels;
2929
TVector<ui32> YellowStopChannels;
30+
THashMap<ui32, float> ApproximateFreeSpaceShareByChannel;
3031

3132
NWilson::TSpan RequestSpan;
3233
TMap<TLogoBlobID, NWilson::TSpan> BlobSpans;
@@ -38,19 +39,22 @@ class TTabletReqWriteLog : public TActorBootstrapped<TTabletReqWriteLog> {
3839
void Handle(TEvBlobStorage::TEvPutResult::TPtr &ev, const TActorContext &ctx) {
3940
TEvBlobStorage::TEvPutResult *msg = ev->Get();
4041

42+
ui32 channel = msg->Id.Channel();
43+
4144
if (msg->StatusFlags.Check(NKikimrBlobStorage::StatusDiskSpaceLightYellowMove)) {
42-
YellowMoveChannels.push_back(msg->Id.Channel());
45+
YellowMoveChannels.push_back(channel);
4346
}
4447
if (msg->StatusFlags.Check(NKikimrBlobStorage::StatusDiskSpaceYellowStop)) {
45-
YellowStopChannels.push_back(msg->Id.Channel());
48+
YellowStopChannels.push_back(channel);
4649
}
50+
ApproximateFreeSpaceShareByChannel[channel] = msg->ApproximateFreeSpaceShare;
4751

4852
switch (msg->Status) {
4953
case NKikimrProto::OK:
5054
LOG_DEBUG_S(ctx, NKikimrServices::TABLET_MAIN, "Put Result: " << msg->Print(false));
5155

52-
GroupWrittenBytes[std::make_pair(msg->Id.Channel(), msg->GroupId)] += msg->Id.BlobSize();
53-
GroupWrittenOps[std::make_pair(msg->Id.Channel(), msg->GroupId)] += 1;
56+
GroupWrittenBytes[std::make_pair(channel, msg->GroupId)] += msg->Id.BlobSize();
57+
GroupWrittenOps[std::make_pair(channel, msg->GroupId)] += 1;
5458

5559
ResponseCookies ^= ev->Cookie;
5660

@@ -120,6 +124,7 @@ class TTabletReqWriteLog : public TActorBootstrapped<TTabletReqWriteLog> {
120124
LogEntryID,
121125
std::move(YellowMoveChannels),
122126
std::move(YellowStopChannels),
127+
std::move(ApproximateFreeSpaceShareByChannel),
123128
std::move(GroupWrittenBytes),
124129
std::move(GroupWrittenOps),
125130
reason));

‎ydb/core/tablet/tablet_sys.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1241,6 +1241,7 @@ void TTablet::CheckEntry(TGraph::TIndex::iterator it) {
12411241
entry->ConfirmedOnSend,
12421242
std::move(entry->YellowMoveChannels),
12431243
std::move(entry->YellowStopChannels),
1244+
std::move(entry->ApproximateFreeSpaceShareByChannel),
12441245
std::move(entry->GroupWrittenBytes),
12451246
std::move(entry->GroupWrittenOps)),
12461247
0, entry->SourceCookie);
@@ -1560,6 +1561,7 @@ void TTablet::Handle(TEvTabletBase::TEvWriteLogResult::TPtr &ev) {
15601561
entry->BlobStorageConfirmed = true;
15611562
entry->YellowMoveChannels = std::move(msg->YellowMoveChannels);
15621563
entry->YellowStopChannels = std::move(msg->YellowStopChannels);
1564+
entry->ApproximateFreeSpaceShareByChannel = std::move(msg->ApproximateFreeSpaceShareByChannel);
15631565
entry->GroupWrittenBytes = std::move(msg->GroupWrittenBytes);
15641566
entry->GroupWrittenOps = std::move(msg->GroupWrittenOps);
15651567

‎ydb/core/tablet/tablet_sys.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,8 @@ class TTablet : public TActor<TTablet> {
9393
TVector<ui32> YellowMoveChannels;
9494
TVector<ui32> YellowStopChannels;
9595

96+
THashMap<ui32, float> ApproximateFreeSpaceShareByChannel;
97+
9698
size_t ByteSize;
9799

98100
TLogEntry(ui32 step, ui32 confirmedOnSend, ui64 sourceCookie)

‎ydb/core/tablet_flat/flat_database.cpp

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -432,7 +432,7 @@ void TDatabase::Begin(TTxStamp stamp, IPages& env)
432432
{
433433
Y_ABORT_UNLESS(!Redo, "Transaction already in progress");
434434
Y_ABORT_UNLESS(!Env);
435-
Annex = new TAnnex(*DatabaseImpl->Scheme);
435+
Annex = new TAnnex(*DatabaseImpl->Scheme, DatabaseImpl->Stats.NormalizedFreeSpaceShareByChannel);
436436
Redo = new NRedo::TWriter;
437437
DatabaseImpl->BeginTransaction();
438438
Change = MakeHolder<TChange>(stamp, DatabaseImpl->Serial());
@@ -511,6 +511,13 @@ const TDbStats& TDatabase::Counters() const noexcept
511511
return DatabaseImpl->Stats;
512512
}
513513

514+
void TDatabase::UpdateApproximateFreeSharesByChannel(const THashMap<ui32, float>& approximateFreeSpaceShareByChannel)
515+
{
516+
for (auto& [channel, value] : approximateFreeSpaceShareByChannel) {
517+
DatabaseImpl->Stats.NormalizedFreeSpaceShareByChannel[channel] = NUtil::NormalizeFreeSpaceShare(value);
518+
}
519+
}
520+
514521
void TDatabase::SetTableObserver(ui32 table, TIntrusivePtr<ITableObserver> ptr) noexcept
515522
{
516523
Require(table)->SetTableObserver(std::move(ptr));

0 commit comments

Comments
 (0)