Skip to content

Commit 262bf50

Browse files
mregrockmregrock
authored
Persist config in nodewarden (#14363)
1 parent 8c57cab commit 262bf50

23 files changed

+487
-61
lines changed

‎ydb/core/blobstorage/nodewarden/node_warden.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ namespace NKikimr {
2323
std::optional<NKikimrConfig::TDomainsConfig> DomainsConfig;
2424
std::optional<NKikimrConfig::TSelfManagementConfig> SelfManagementConfig;
2525
TString ConfigStorePath;
26+
std::optional<NKikimrBlobStorage::TYamlConfig> YamlConfig;
2627
TIntrusivePtr<IPDiskServiceFactory> PDiskServiceFactory;
2728
TIntrusivePtr<TAllVDiskKinds> AllVDiskKinds;
2829
TIntrusivePtr<NPDisk::TDriveModelDb> AllDriveModels;

‎ydb/core/blobstorage/nodewarden/node_warden_cache.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ void TNodeWarden::Handle(TEvents::TEvInvokeResult::TPtr ev) {
1717
// process message
1818
try {
1919
ev->Get()->Process(ActorContext());
20-
} catch (const std::exception&) {
20+
} catch (const std::exception&) {
2121
Y_ABORT("Exception while executing sync callback: %s", CurrentExceptionMessage().data());
2222
}
2323

‎ydb/core/blobstorage/nodewarden/node_warden_impl.cpp

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,10 @@
1111
#include <ydb/core/blobstorage/pdisk/drivedata_serializer.h>
1212
#include <ydb/core/blobstorage/vdisk/repl/blobstorage_replbroker.h>
1313
#include <ydb/library/pdisk_io/file_params.h>
14+
#include <ydb/core/mind/bscontroller/yaml_config_helpers.h>
1415
#include <ydb/core/base/nameservice.h>
1516
#include <ydb/core/protos/key.pb.h>
17+
#include <util/folder/dirut.h>
1618

1719
#include <library/cpp/lwtrace/mon/mon_lwtrace.h>
1820

@@ -121,6 +123,8 @@ STATEFN(TNodeWarden::StateOnline) {
121123
hFunc(TEvBlobStorage::TEvControllerGroupMetricsExchange, Handle);
122124
hFunc(TEvPrivate::TEvSendDiskMetrics, Handle);
123125
hFunc(TEvPrivate::TEvUpdateNodeDrives, Handle);
126+
hFunc(TEvPrivate::TEvRetrySaveConfig, Handle);
127+
124128
hFunc(NMon::TEvHttpInfo, Handle);
125129
cFunc(NActors::TEvents::TSystem::Poison, PassAway);
126130

@@ -437,6 +441,12 @@ void TNodeWarden::Bootstrap() {
437441
const bool success = DeriveStorageConfig(appConfig, &StorageConfig, &errorReason);
438442
Y_VERIFY_S(success, "failed to generate initial TStorageConfig: " << errorReason);
439443

444+
//LoadConfigVersion();
445+
if (Cfg->YamlConfig) {
446+
YamlConfig.emplace();
447+
YamlConfig->CopyFrom(*Cfg->YamlConfig);
448+
}
449+
440450
// Start a statically configured set
441451
if (Cfg->BlobStorageConfig.HasServiceSet()) {
442452
const auto& serviceSet = Cfg->BlobStorageConfig.GetServiceSet();
@@ -625,6 +635,95 @@ void TNodeWarden::ProcessShredStatus(ui64 cookie, ui64 generation, std::optional
625635
}
626636
}
627637

638+
void TNodeWarden::PersistConfig(const TString& configYaml, ui64 version, std::optional<TString> storageYaml) {
639+
if (!Cfg->ConfigStorePath) {
640+
return;
641+
}
642+
643+
struct TSaveContext {
644+
TString ConfigStorePath;
645+
TString ConfigYaml;
646+
ui64 Version;
647+
std::optional<TString> StorageYaml;
648+
bool Success = true;
649+
TString ErrorMessage;
650+
TActorId SelfId;
651+
};
652+
653+
auto saveCtx = std::make_shared<TSaveContext>();
654+
saveCtx->ConfigStorePath = Cfg->ConfigStorePath;
655+
saveCtx->ConfigYaml = std::move(configYaml);
656+
saveCtx->StorageYaml = std::move(storageYaml);
657+
saveCtx->Version = std::move(version);
658+
saveCtx->SelfId = SelfId();
659+
660+
EnqueueSyncOp([this, saveCtx](const TActorContext&) {
661+
bool success = true;
662+
try {
663+
MakePathIfNotExist(saveCtx->ConfigStorePath.c_str());
664+
} catch (const yexception& e) {
665+
STLOG(PRI_ERROR, BS_NODE, NW91, "Failed to create config store path", (Error, e.what()));
666+
success = false;
667+
}
668+
669+
auto saveConfig = [&](const TString& yaml, const TString& configFileName) -> bool {
670+
try {
671+
TString tempPath = TStringBuilder() << saveCtx->ConfigStorePath << "/temp_" << configFileName;
672+
TString configPath = TStringBuilder() << saveCtx->ConfigStorePath << "/" << configFileName;
673+
674+
{
675+
TFileOutput tempFile(tempPath);
676+
tempFile << yaml;
677+
tempFile.Flush();
678+
}
679+
680+
if (!NFs::Rename(tempPath, configPath)) {
681+
STLOG(PRI_ERROR, BS_NODE, NW92, "Failed to rename temporary file", (Error, LastSystemErrorText()));
682+
success = false;
683+
return false;
684+
}
685+
return true;
686+
} catch (const std::exception& e) {
687+
STLOG(PRI_ERROR, BS_NODE, NW93, "Failed to save config file", (Error, e.what()));
688+
success = false;
689+
return false;
690+
}
691+
};
692+
693+
if (success) {
694+
success = saveConfig(saveCtx->ConfigYaml, YamlConfigFileName);
695+
if (success) {
696+
STLOG(PRI_INFO, BS_NODE, NW94, "Yaml config saved");
697+
}
698+
}
699+
700+
if (success && saveCtx->StorageYaml) {
701+
success = saveConfig(*saveCtx->StorageYaml, StorageConfigFileName);
702+
if (success) {
703+
STLOG(PRI_INFO, BS_NODE, NW95, "Storage config saved");
704+
}
705+
}
706+
707+
return [this, saveCtx, success]() {
708+
if (success) {
709+
if (!YamlConfig) {
710+
YamlConfig.emplace();
711+
}
712+
YamlConfig->SetYAML(saveCtx->ConfigYaml);
713+
YamlConfig->SetConfigVersion(saveCtx->Version);
714+
ConfigSaveTimer.Reset();
715+
} else {
716+
NKikimrBlobStorage::TYamlConfig yamlConfig;
717+
yamlConfig.SetYAML(saveCtx->ConfigYaml);
718+
yamlConfig.SetConfigVersion(saveCtx->Version);
719+
TActivationContext::Schedule(TDuration::MilliSeconds(ConfigSaveTimer.NextBackoffMs()),
720+
new IEventHandle(SelfId(), SelfId(),
721+
new TEvPrivate::TEvRetrySaveConfig(yamlConfig), 0, ExpectedSaveConfigCookie));
722+
}
723+
};
724+
});
725+
}
726+
628727
void TNodeWarden::Handle(TEvRegisterPDiskLoadActor::TPtr ev) {
629728
Send(ev.Get()->Sender, new TEvRegisterPDiskLoadActorResult(NextLocalPDiskInitOwnerRound()));
630729
}
@@ -709,6 +808,16 @@ void TNodeWarden::Handle(TEvBlobStorage::TEvControllerNodeServiceSetUpdate::TPtr
709808
}
710809
}
711810
}
811+
812+
if (record.HasYamlConfig()) {
813+
const auto& request = record.GetYamlConfig();
814+
if (request.HasYAML()) {
815+
TString yaml = NYamlConfig::DecompressYamlString(request.GetYAML());
816+
ui64 version = request.GetConfigVersion();
817+
PersistConfig(yaml, version);
818+
ExpectedSaveConfigCookie++;
819+
}
820+
}
712821
}
713822

714823
void TNodeWarden::SendDropDonorQuery(ui32 nodeId, ui32 pdiskId, ui32 vslotId, const TVDiskID& vdiskId, TDuration backoff) {
@@ -932,6 +1041,15 @@ void TNodeWarden::Handle(TEvPrivate::TEvUpdateNodeDrives::TPtr&) {
9321041
Schedule(TDuration::Seconds(10), new TEvPrivate::TEvUpdateNodeDrives());
9331042
}
9341043

1044+
void TNodeWarden::Handle(TEvPrivate::TEvRetrySaveConfig::TPtr& ev) {
1045+
STLOG(PRI_TRACE, BS_NODE, NW97, "Handle(TEvRetrySaveConfig)");
1046+
if (ev->Cookie == ExpectedSaveConfigCookie) {
1047+
const auto& yamlConfig = ev->Get()->YamlConfig;
1048+
PersistConfig(yamlConfig.GetYAML(), yamlConfig.GetConfigVersion());
1049+
ExpectedSaveConfigCookie++;
1050+
}
1051+
}
1052+
9351053
void TNodeWarden::SendDiskMetrics(bool reportMetrics) {
9361054
STLOG(PRI_TRACE, BS_NODE, NW45, "SendDiskMetrics", (ReportMetrics, reportMetrics));
9371055

‎ydb/core/blobstorage/nodewarden/node_warden_impl.h

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#include <ydb/core/blobstorage/incrhuge/incrhuge.h>
1010
#include <ydb/core/node_whiteboard/node_whiteboard.h>
1111
#include <ydb/core/protos/blobstorage_distributed_config.pb.h>
12+
#include <ydb/core/util/backoff.h>
1213

1314
namespace NKikimr {
1415
struct TNodeWardenConfig;
@@ -25,6 +26,8 @@ namespace NKikimr::NStorage {
2526
constexpr TDuration BackoffMin = TDuration::MilliSeconds(20);
2627
constexpr TDuration BackoffMax = TDuration::Seconds(5);
2728
constexpr const char *MockDevicesPath = "/Berkanavt/kikimr/testing/mock_devices.txt";
29+
constexpr const char *YamlConfigFileName = "config.yaml";
30+
constexpr const char *StorageConfigFileName = "storage.yaml";
2831

2932
template<typename T, typename TPred>
3033
T *FindOrCreateProtoItem(google::protobuf::RepeatedPtrField<T> *collection, TPred&& pred) {
@@ -140,6 +143,10 @@ namespace NKikimr::NStorage {
140143
ui64 NextConfigCookie = 1;
141144
std::unordered_map<ui64, std::function<void(TEvBlobStorage::TEvControllerConfigResponse*)>> ConfigInFlight;
142145

146+
TBackoffTimer ConfigSaveTimer{BackoffMin.MilliSeconds(), BackoffMax.MilliSeconds()};
147+
std::optional<NKikimrBlobStorage::TYamlConfig> YamlConfig;
148+
ui64 ExpectedSaveConfigCookie = 0;
149+
143150
TVector<NPDisk::TDriveData> WorkingLocalDrives;
144151

145152
NPDisk::TOwnerRound LocalPDiskInitOwnerRound = 1;
@@ -157,6 +164,8 @@ namespace NKikimr::NStorage {
157164
EvGetGroup,
158165
EvGroupPendingQueueTick,
159166
EvDereferencePDisk,
167+
EvSaveConfigResult,
168+
EvRetrySaveConfig,
160169
};
161170

162171
struct TEvSendDiskMetrics : TEventLocal<TEvSendDiskMetrics, EvSendDiskMetrics> {};
@@ -166,6 +175,13 @@ namespace NKikimr::NStorage {
166175
TPDiskKey PDiskKey;
167176
TEvDereferencePDisk(TPDiskKey pdiskKey) : PDiskKey(pdiskKey) {}
168177
};
178+
179+
struct TEvRetrySaveConfig : TEventLocal<TEvRetrySaveConfig, EvRetrySaveConfig> {
180+
NKikimrBlobStorage::TYamlConfig YamlConfig;
181+
TEvRetrySaveConfig(const NKikimrBlobStorage::TYamlConfig& yamlConfig)
182+
: YamlConfig(yamlConfig)
183+
{}
184+
};
169185
};
170186

171187
TControlWrapper EnablePutBatching;
@@ -550,6 +566,10 @@ namespace NKikimr::NStorage {
550566
void Handle(NPDisk::TEvShredPDiskResult::TPtr ev);
551567
void Handle(NPDisk::TEvShredPDisk::TPtr ev);
552568
void ProcessShredStatus(ui64 cookie, ui64 generation, std::optional<TString> error);
569+
570+
void PersistConfig(const TString& yaml, ui64 version, std::optional<TString> storageYaml = std::nullopt);
571+
void LoadConfigVersion();
572+
553573
void Handle(TEvRegisterPDiskLoadActor::TPtr ev);
554574
void Handle(TEvBlobStorage::TEvControllerNodeServiceSetUpdate::TPtr ev);
555575

@@ -565,6 +585,8 @@ namespace NKikimr::NStorage {
565585
void Handle(TEvBlobStorage::TEvControllerGroupMetricsExchange::TPtr ev);
566586
void Handle(TEvPrivate::TEvSendDiskMetrics::TPtr&);
567587
void Handle(TEvPrivate::TEvUpdateNodeDrives ::TPtr&);
588+
void Handle(TEvPrivate::TEvRetrySaveConfig::TPtr&);
589+
568590
void Handle(NMon::TEvHttpInfo::TPtr&);
569591
void RenderJsonGroupInfo(IOutputStream& out, const std::set<ui32>& groupIds);
570592
void RenderWholePage(IOutputStream&);

‎ydb/core/blobstorage/nodewarden/node_warden_pipe.cpp

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
11
#include "node_warden_impl.h"
2+
#include "node_warden.h"
3+
#include <util/system/fs.h>
4+
#include <ydb/library/yaml_config/yaml_config.h>
5+
#include <ydb/library/yaml_config/yaml_config_helpers.h>
26

37
using namespace NKikimr;
48
using namespace NStorage;
@@ -98,6 +102,11 @@ void TNodeWarden::SendRegisterNode() {
98102
}
99103
}
100104

105+
if (!Cfg->ConfigStorePath.empty() && YamlConfig) {
106+
ev->Record.SetConfigVersion(YamlConfig->GetConfigVersion());
107+
ev->Record.SetConfigHash(NKikimr::NYaml::GetConfigHash(YamlConfig->GetYAML()));
108+
}
109+
101110
SendToController(std::move(ev));
102111
}
103112

‎ydb/core/config/init/init.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -597,6 +597,12 @@ void LoadMainYamlConfig(TConfigRefs refs, const TString& mainYamlConfigFile, NKi
597597
appConfig.MutableSelfManagementConfig()->SetInitialConfigYaml(mainYamlConfigString);
598598
}
599599

600+
if (appConfig.GetConfigLoadedFromStore()) {
601+
auto* yamlConfig = appConfig.MutableStoredConfigYaml();
602+
yamlConfig->SetYAML(mainYamlConfigString);
603+
yamlConfig->SetConfigVersion(NYamlConfig::GetVersion(mainYamlConfigString));
604+
}
605+
600606
/*
601607
* FIXME: if (ErrorCollector.HasFatal()) { return; }
602608
*/

‎ydb/core/config/init/init_impl.h

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1070,10 +1070,11 @@ class TInitialConfiguratorImpl
10701070
TString yamlConfigFile = CommonAppOptions.YamlConfigFile;
10711071
if (!CommonAppOptions.ConfigStorePath.empty()) {
10721072
AppConfig.SetConfigStorePath(CommonAppOptions.ConfigStorePath);
1073-
1073+
10741074
const TString autoConfigPath = TStringBuilder() << CommonAppOptions.ConfigStorePath << "/" << CONFIG_NAME;
10751075
fs::path path(autoConfigPath.c_str());
10761076
if (IsFileExists(path)) {
1077+
AppConfig.SetConfigLoadedFromStore(true);
10771078
yamlConfigFile = autoConfigPath;
10781079
}
10791080
}
@@ -1099,7 +1100,7 @@ class TInitialConfiguratorImpl
10991100
InitDynamicNode();
11001101
}
11011102

1102-
LoadMainYamlConfig(refs, CommonAppOptions.YamlConfigFile, AppConfig);
1103+
LoadMainYamlConfig(refs, yamlConfigFile, AppConfig);
11031104

11041105
Option("sys-file", TCfg::TActorSystemConfigFieldTag{});
11051106

‎ydb/core/driver_lib/run/kikimr_services_initializers.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -951,6 +951,11 @@ void TBSNodeWardenInitializer::InitializeServices(NActors::TActorSystemSetup* se
951951
if (Config.HasConfigStorePath()) {
952952
nodeWardenConfig->ConfigStorePath = Config.GetConfigStorePath();
953953
}
954+
955+
if (Config.HasStoredConfigYaml()) {
956+
nodeWardenConfig->YamlConfig.emplace(Config.GetStoredConfigYaml());
957+
}
958+
954959
ObtainTenantKey(&nodeWardenConfig->TenantKey, Config.GetKeyConfig());
955960
ObtainStaticKey(&nodeWardenConfig->StaticKey);
956961
ObtainPDiskKey(&nodeWardenConfig->PDiskKey, Config.GetPDiskKeyConfig());

‎ydb/core/mind/bscontroller/bsc.cpp

Lines changed: 0 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -618,40 +618,6 @@ ui32 TBlobStorageController::GetEventPriority(IEventHandle *ev) {
618618
Y_ABORT();
619619
}
620620

621-
TString TBlobStorageController::CompressYamlConfig(const TYamlConfig& yamlConfig) {
622-
TStringStream s;
623-
{
624-
TZstdCompress zstd(&s);
625-
Save(&zstd, yamlConfig);
626-
}
627-
return s.Str();
628-
}
629-
630-
TString TBlobStorageController::CompressStorageYamlConfig(const TString& storageYamlConfig) {
631-
TStringStream s;
632-
{
633-
TZstdCompress zstd(&s);
634-
Save(&zstd, storageYamlConfig);
635-
}
636-
return s.Str();
637-
}
638-
639-
TBlobStorageController::TYamlConfig TBlobStorageController::DecompressYamlConfig(const TString& buffer) {
640-
TStringInput s(buffer);
641-
TZstdDecompress zstd(&s);
642-
TYamlConfig res;
643-
Load(&zstd, res);
644-
return res;
645-
}
646-
647-
TString TBlobStorageController::DecompressStorageYamlConfig(const TString& buffer) {
648-
TStringInput s(buffer);
649-
TZstdDecompress zstd(&s);
650-
TString res;
651-
Load(&zstd, res);
652-
return res;
653-
}
654-
655621
} // NBsController
656622

657623
} // NKikimr

‎ydb/core/mind/bscontroller/commit_config.cpp

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,17 @@ namespace NKikimr::NBsController {
6060
}
6161
if (YamlConfig) {
6262
Self->YamlConfig = std::move(YamlConfig);
63+
const auto& configVersion = GetVersion(*Self->YamlConfig);
64+
const auto& compressedConfig = CompressSingleConfig(*Self->YamlConfig);
65+
for (auto& node: Self->Nodes) {
66+
if (node.second.ConnectedServerId) {
67+
auto configPersistEv = std::make_unique<TEvBlobStorage::TEvControllerNodeServiceSetUpdate>();
68+
auto* yamlConfig = configPersistEv->Record.MutableYamlConfig();
69+
yamlConfig->SetYAML(compressedConfig);
70+
yamlConfig->SetConfigVersion(configVersion);
71+
Self->SendToWarden(node.first, std::move(configPersistEv), 0);
72+
}
73+
}
6374
}
6475
if (StorageYamlConfig) {
6576
Self->StorageYamlConfig = std::move(*StorageYamlConfig);

0 commit comments

Comments
 (0)