Skip to content

Commit 85359f5

Browse files
swalrus1swalrus1
authored
store all versions of TTL in memory (#14677)
1 parent 805f251 commit 85359f5

File tree

3 files changed

+63
-41
lines changed

3 files changed

+63
-41
lines changed

‎ydb/core/tx/columnshard/columnshard_impl.cpp

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1617,11 +1617,7 @@ void TColumnShard::OnTieringModified(const std::optional<ui64> pathId) {
16171617
StoragesManager->OnTieringModified(Tiers);
16181618
if (TablesManager.HasPrimaryIndex()) {
16191619
if (pathId) {
1620-
std::optional<NOlap::TTiering> tableTtl;
1621-
if (auto* findTtl = TablesManager.GetTtl().FindPtr(*pathId)) {
1622-
tableTtl = *findTtl;
1623-
}
1624-
TablesManager.MutablePrimaryIndex().OnTieringModified(tableTtl, *pathId);
1620+
TablesManager.MutablePrimaryIndex().OnTieringModified(TablesManager.GetTableTtl(*pathId), *pathId);
16251621
} else {
16261622
TablesManager.MutablePrimaryIndex().OnTieringModified(TablesManager.GetTtl());
16271623
}

‎ydb/core/tx/columnshard/tables_manager.cpp

Lines changed: 6 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,6 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) {
112112
return false;
113113
}
114114

115-
THashMap<ui64, NOlap::TSnapshot> lastVersion;
116115
while (!rowset.EndOfSet()) {
117116
const ui64 pathId = rowset.GetValue<Schema::TableVersionInfo::PathId>();
118117
Y_ABORT_UNLESS(Tables.contains(pathId));
@@ -126,22 +125,8 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) {
126125
AFL_VERIFY(preset);
127126
AFL_VERIFY(preset->Id == versionInfo.GetSchemaPresetId())("preset", preset->Id)("table", versionInfo.GetSchemaPresetId());
128127

129-
if (!table.IsDropped() && versionInfo.HasTtlSettings()) {
130-
auto& ttlSettings = versionInfo.GetTtlSettings();
131-
auto vIt = lastVersion.find(pathId);
132-
if (vIt == lastVersion.end()) {
133-
vIt = lastVersion.emplace(pathId, version).first;
134-
}
135-
if (vIt->second <= version) {
136-
if (ttlSettings.HasEnabled()) {
137-
NOlap::TTiering deserializedTtl;
138-
AFL_VERIFY(deserializedTtl.DeserializeFromProto(ttlSettings.GetEnabled()).IsSuccess());
139-
Ttl[pathId] = std::move(deserializedTtl);
140-
} else {
141-
Ttl.erase(pathId);
142-
}
143-
vIt->second = version;
144-
}
128+
if (versionInfo.HasTtlSettings()) {
129+
Ttl.AddVersionFromProto(pathId, version, versionInfo.GetTtlSettings());
145130
}
146131
table.AddVersion(version);
147132
if (!rowset.Next()) {
@@ -230,7 +215,7 @@ const TTableInfo& TTablesManager::GetTable(const ui64 pathId) const {
230215
}
231216

232217
ui64 TTablesManager::GetMemoryUsage() const {
233-
ui64 memory = Tables.size() * sizeof(TTableInfo) + PathsToDrop.size() * sizeof(ui64) + Ttl.size() * sizeof(NOlap::TTiering);
218+
ui64 memory = Tables.size() * sizeof(TTableInfo) + PathsToDrop.size() * sizeof(ui64) + Ttl.GetMemoryUsage();
234219
if (PrimaryIndex) {
235220
memory += PrimaryIndex->MemoryUsage();
236221
}
@@ -242,7 +227,6 @@ void TTablesManager::DropTable(const ui64 pathId, const NOlap::TSnapshot& versio
242227
auto& table = Tables[pathId];
243228
table.SetDropVersion(version);
244229
AFL_VERIFY(PathsToDrop[version].emplace(pathId).second);
245-
Ttl.erase(pathId);
246230
Schema::SaveTableDropVersion(db, pathId, version.GetPlanStep(), version.GetTxId());
247231
}
248232

@@ -299,7 +283,7 @@ void TTablesManager::AddSchemaVersion(
299283
for (auto&& i : Tables) {
300284
PrimaryIndex->RegisterTable(i.first);
301285
}
302-
PrimaryIndex->OnTieringModified(Ttl);
286+
PrimaryIndex->OnTieringModified(GetTtl());
303287
} else {
304288
PrimaryIndex->RegisterSchemaVersion(version, presetId, NOlap::IColumnEngine::TSchemaInitializationData(versionInfo));
305289
}
@@ -319,14 +303,7 @@ void TTablesManager::AddTableVersion(const ui64 pathId, const NOlap::TSnapshot&
319303
bool isTtlModified = false;
320304
if (versionInfo.HasTtlSettings()) {
321305
isTtlModified = true;
322-
const auto& ttlSettings = versionInfo.GetTtlSettings();
323-
if (ttlSettings.HasEnabled()) {
324-
NOlap::TTiering deserializedTtl;
325-
AFL_VERIFY(deserializedTtl.DeserializeFromProto(ttlSettings.GetEnabled()).IsSuccess());
326-
Ttl[pathId] = std::move(deserializedTtl);
327-
} else {
328-
Ttl.erase(pathId);
329-
}
306+
Ttl.AddVersionFromProto(pathId, version, versionInfo.GetTtlSettings());
330307
}
331308

332309
if (versionInfo.HasSchemaPresetId()) {
@@ -344,11 +321,7 @@ void TTablesManager::AddTableVersion(const ui64 pathId, const NOlap::TSnapshot&
344321

345322
if (isTtlModified) {
346323
if (PrimaryIndex) {
347-
if (auto findTtl = Ttl.FindPtr(pathId)) {
348-
PrimaryIndex->OnTieringModified(*findTtl, pathId);
349-
} else {
350-
PrimaryIndex->OnTieringModified({}, pathId);
351-
}
324+
PrimaryIndex->OnTieringModified(GetTableTtl(pathId), pathId);
352325
}
353326
}
354327
Schema::SaveTableVersionInfo(db, pathId, version, versionInfo);

‎ydb/core/tx/columnshard/tables_manager.h

Lines changed: 56 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -147,13 +147,53 @@ class TTableInfo {
147147
}
148148
};
149149

150+
class TTtlVersions {
151+
private:
152+
THashMap<ui64, std::map<NOlap::TSnapshot, std::optional<NOlap::TTiering>>> Ttl;
153+
154+
void AddVersion(const ui64 pathId, const NOlap::TSnapshot& snapshot, std::optional<NOlap::TTiering> ttl) {
155+
AFL_VERIFY(Ttl[pathId].emplace(snapshot, ttl).second)("snapshot", snapshot);
156+
}
157+
158+
public:
159+
void AddVersionFromProto(const ui64 pathId, const NOlap::TSnapshot& snapshot, const NKikimrSchemeOp::TColumnDataLifeCycle& ttlSettings) {
160+
std::optional<NOlap::TTiering> ttlVersion;
161+
if (ttlSettings.HasEnabled()) {
162+
NOlap::TTiering deserializedTtl;
163+
AFL_VERIFY(deserializedTtl.DeserializeFromProto(ttlSettings.GetEnabled()).IsSuccess());
164+
ttlVersion.emplace(std::move(deserializedTtl));
165+
}
166+
AddVersion(pathId, snapshot, ttlVersion);
167+
}
168+
169+
std::optional<NOlap::TTiering> GetTableTtl(const ui64 pathId, const NOlap::TSnapshot& snapshot = NOlap::TSnapshot::Max()) const {
170+
auto findTable = Ttl.FindPtr(pathId);
171+
if (!findTable) {
172+
return std::nullopt;
173+
}
174+
const auto findTtl = findTable->upper_bound(snapshot);
175+
if (findTtl == findTable->begin()) {
176+
return std::nullopt;
177+
}
178+
return std::prev(findTtl)->second;
179+
}
180+
181+
ui64 GetMemoryUsage() const {
182+
ui64 memory = 0;
183+
for (const auto& [_, ttlVersions] : Ttl) {
184+
memory += ttlVersions.size() * sizeof(NOlap::TTiering);
185+
}
186+
return memory;
187+
}
188+
};
189+
150190
class TTablesManager {
151191
private:
152192
THashMap<ui64, TTableInfo> Tables;
153193
THashSet<ui32> SchemaPresetsIds;
154194
THashMap<ui32, NKikimrSchemeOp::TColumnTableSchema> ActualSchemaForPreset;
155195
std::map<NOlap::TSnapshot, THashSet<ui64>> PathsToDrop;
156-
THashMap<ui64, NOlap::TTiering> Ttl;
196+
TTtlVersions Ttl;
157197
std::unique_ptr<NOlap::IColumnEngine> PrimaryIndex;
158198
std::shared_ptr<NOlap::IStoragesManager> StoragesManager;
159199
std::shared_ptr<NOlap::NDataAccessorControl::IDataAccessorsManager> DataAccessorsManager;
@@ -177,8 +217,21 @@ class TTablesManager {
177217
bool TryFinalizeDropPathOnExecute(NTable::TDatabase& dbTable, const ui64 pathId) const;
178218
bool TryFinalizeDropPathOnComplete(const ui64 pathId);
179219

180-
const THashMap<ui64, NOlap::TTiering>& GetTtl() const {
181-
return Ttl;
220+
THashMap<ui64, NOlap::TTiering> GetTtl(const NOlap::TSnapshot& snapshot = NOlap::TSnapshot::Max()) const {
221+
THashMap<ui64, NOlap::TTiering> ttl;
222+
for (const auto& [pathId, info] : Tables) {
223+
if (info.IsDropped(snapshot)) {
224+
continue;
225+
}
226+
if (auto tableTtl = Ttl.GetTableTtl(pathId, snapshot)) {
227+
ttl.emplace(pathId, std::move(*tableTtl));
228+
}
229+
}
230+
return ttl;
231+
}
232+
233+
std::optional<NOlap::TTiering> GetTableTtl(const ui64 pathId, const NOlap::TSnapshot& snapshot = NOlap::TSnapshot::Max()) const {
234+
return Ttl.GetTableTtl(pathId, snapshot);
182235
}
183236

184237
const std::map<NOlap::TSnapshot, THashSet<ui64>>& GetPathsToDrop() const {

0 commit comments

Comments
 (0)