Skip to content

Commit fe62452

Browse files
alexvrualexvru
authored
Implement dual-config mode for distconf (#14825)
1 parent 2c7f23f commit fe62452

19 files changed

+423
-192
lines changed

‎ydb/core/blobstorage/nodewarden/distconf_generate.cpp

Lines changed: 28 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -43,29 +43,44 @@ namespace NKikimr::NStorage {
4343

4444
// initial config YAML is taken from the Cfg->SelfManagementConfig as it is cleared in TStorageConfig while
4545
// deriving it from NodeWarden configuration
46-
if (!Cfg->SelfManagementConfig || !Cfg->SelfManagementConfig->HasInitialConfigYaml()) {
46+
if (!Cfg->StartupConfigYaml) {
4747
return "missing initial config YAML";
4848
}
49-
TStringStream ss;
50-
TString yaml = Cfg->SelfManagementConfig->GetInitialConfigYaml();
51-
ui32 version = 0;
49+
50+
ui64 version = 0;
5251
try {
53-
auto json = NYaml::Yaml2Json(YAML::Load(yaml), true);
54-
if (json.Has("metadata")) {
55-
if (auto& metadata = json["metadata"]; metadata.Has("version")) {
56-
version = metadata["version"].GetUIntegerRobust();
57-
}
58-
}
52+
version = NYamlConfig::GetMainMetadata(Cfg->StartupConfigYaml).Version.value_or(0);
5953
} catch (const std::exception& ex) {
60-
return TStringBuilder() << "failed to parse initial config YAML: " << ex.what();
54+
return TStringBuilder() << "failed to parse initial main YAML: " << ex.what();
6155
}
6256
if (version) {
63-
return TStringBuilder() << "initial config version must be zero";
57+
return TStringBuilder() << "initial main config version must be zero";
6458
}
65-
if (const auto& error = UpdateConfigComposite(*config, yaml, std::nullopt)) {
59+
60+
if (const auto& error = UpdateConfigComposite(*config, Cfg->StartupConfigYaml, std::nullopt)) {
6661
return TStringBuilder() << "failed to update config yaml: " << *error;
6762
}
6863

64+
if (Cfg->StartupStorageYaml) {
65+
ui64 storageVersion = 0;
66+
try {
67+
storageVersion = NYamlConfig::GetStorageMetadata(*Cfg->StartupStorageYaml).Version.value_or(0);
68+
} catch (const std::exception& ex) {
69+
return TStringBuilder() << "failed to parse initial storage YAML: " << ex.what();
70+
}
71+
if (storageVersion) {
72+
return TStringBuilder() << "initial storage config version must be zero";
73+
}
74+
75+
TString s;
76+
if (TStringOutput output(s); true) {
77+
TZstdCompress zstd(&output);
78+
zstd << *Cfg->StartupStorageYaml;
79+
}
80+
config->SetCompressedStorageYaml(s);
81+
config->SetExpectedStorageYamlVersion(storageVersion + 1);
82+
}
83+
6984
if (!Cfg->DomainsConfig) { // no automatic configuration required
7085
} else if (Cfg->DomainsConfig->StateStorageSize() == 1) { // the StateStorage config is already defined explicitly, just migrate it
7186
const auto& ss = Cfg->DomainsConfig->GetStateStorage(0);

‎ydb/core/blobstorage/nodewarden/distconf_invoke.cpp

Lines changed: 89 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -665,11 +665,19 @@ namespace NKikimr::NStorage {
665665
? std::make_optional(request.GetSwitchDedicatedStorageSection())
666666
: std::nullopt;
667667

668-
const TString *storageYamlPtr = newStorageYaml ? &newStorageYaml.value() :
669-
Self->StorageConfigYaml ? &Self->StorageConfigYaml.value() : nullptr;
670-
671668
const bool targetDedicatedStorageSection = switchDedicatedStorageSection.value_or(Self->StorageConfigYaml.has_value());
672669

670+
if (switchDedicatedStorageSection) {
671+
// check that configs are explicitly defined when we are switching dual-config mode
672+
if (!NewYaml) {
673+
return FinishWithError(TResult::ERROR, "main config must be specified when switching dedicated"
674+
" storage section mode");
675+
} else if (*switchDedicatedStorageSection && !newStorageYaml) {
676+
return FinishWithError(TResult::ERROR, "storage config must be specified when turning on dedicated"
677+
" storage section mode");
678+
}
679+
}
680+
673681
if (request.GetDedicatedStorageSectionConfigMode() != targetDedicatedStorageSection) {
674682
return FinishWithError(TResult::ERROR, "DedicatedStorageSectionConfigMode does not match target state");
675683
} else if (newStorageYaml && !targetDedicatedStorageSection) {
@@ -678,89 +686,109 @@ namespace NKikimr::NStorage {
678686
} else if (switchDedicatedStorageSection && *switchDedicatedStorageSection == Self->StorageConfigYaml.has_value()) {
679687
// this enable/disable command does not change the state
680688
return FinishWithError(TResult::ERROR, "dedicated storage config section is already in requested state");
681-
} else if (targetDedicatedStorageSection && !storageYamlPtr) {
682-
// we are going to turn on dual-config mode, but no storage config provided
683-
return FinishWithError(TResult::ERROR, "no dedicated storage config section provided");
684689
}
685690

686-
const TString *mainYamlPtr = NewYaml ? &NewYaml.value() : &Self->MainConfigYaml;
687-
688-
std::optional<ui64> newYamlVersion;
689-
std::optional<ui64> newStorageYamlVersion;
691+
TString state;
692+
NKikimrBlobStorage::TStorageConfig config(*Self->StorageConfig);
693+
std::optional<ui64> newExpectedStorageYamlVersion;
690694

691-
NKikimrConfig::TAppConfig appConfig;
692-
const char *state = "";
695+
if (config.HasExpectedStorageYamlVersion()) {
696+
newExpectedStorageYamlVersion.emplace(config.GetExpectedStorageYamlVersion());
697+
}
693698

694699
try {
695-
if (storageYamlPtr) { // parse the storage yaml first
696-
state = "loading storage YAML";
697-
auto json = NYaml::Yaml2Json(YAML::Load(*storageYamlPtr), true);
698-
state = "parsing storage YAML";
699-
NYaml::Parse(json, NYaml::GetJsonToProtoConfig(), appConfig, true);
700-
state = "extracting storage YAML metadata";
701-
if (json.Has("metadata")) {
702-
if (auto& metadata = json["metadata"]; metadata.Has("version")) {
703-
newStorageYamlVersion = metadata["version"].GetUIntegerRobust();
704-
}
700+
auto load = [&](const TString& yaml, ui64& version, const char *expectedKind) {
701+
state = TStringBuilder() << "loading " << expectedKind << " YAML";
702+
NJson::TJsonValue json = NYaml::Yaml2Json(YAML::Load(yaml), true);
703+
704+
state = TStringBuilder() << "extracting " << expectedKind << " metadata";
705+
if (!json.Has("metadata") || !json["metadata"].IsMap()) {
706+
throw yexception() << "no metadata section";
705707
}
708+
auto& metadata = json["metadata"];
709+
NYaml::ValidateMetadata(metadata);
710+
if (!metadata.Has("kind") || metadata["kind"] != expectedKind) {
711+
throw yexception() << "missing or invalid kind provided";
712+
}
713+
version = metadata["version"].GetUIntegerRobust();
714+
715+
state = TStringBuilder() << "validating " << expectedKind << " config section";
716+
if (!json.Has("config") || !json["config"].IsMap()) {
717+
throw yexception() << "missing config section";
718+
}
719+
720+
return json;
721+
};
722+
723+
NJson::TJsonValue main;
724+
NJson::TJsonValue storage;
725+
const NJson::TJsonValue *effective = nullptr;
726+
727+
if (newStorageYaml) {
728+
ui64 version = 0;
729+
storage = load(*newStorageYaml, version, "StorageConfig");
730+
if (const ui64 expected = Self->StorageConfig->GetExpectedStorageYamlVersion(); version != expected) {
731+
return FinishWithError(TResult::ERROR, TStringBuilder()
732+
<< "storage config version must be increasing by one"
733+
<< " new version# " << version
734+
<< " expected version# " << expected);
735+
}
736+
737+
newExpectedStorageYamlVersion = version + 1;
738+
effective = &storage;
706739
}
707740

708-
state = "loading main YAML";
709-
auto json = NYaml::Yaml2Json(YAML::Load(*mainYamlPtr), true);
710-
state = "parsing main YAML";
711-
NYaml::Parse(json, NYaml::GetJsonToProtoConfig(), appConfig, true);
712-
state = "extracting main YAML metadata";
713-
if (json.Has("metadata")) {
714-
if (auto& metadata = json["metadata"]; metadata.Has("version")) {
715-
newYamlVersion = metadata["version"].GetUIntegerRobust();
741+
if (NewYaml) {
742+
ui64 version = 0;
743+
main = load(*NewYaml, version, "MainConfig");
744+
if (const ui64 expected = *Self->MainConfigYamlVersion + 1; version != expected) {
745+
return FinishWithError(TResult::ERROR, TStringBuilder()
746+
<< "main config version must be increasing by one"
747+
<< " new version# " << version
748+
<< " expected version# " << expected);
749+
}
750+
751+
if (!effective && !Self->StorageConfigYaml) {
752+
effective = &main;
716753
}
717754
}
718-
} catch (const std::exception& ex) {
719-
return FinishWithError(TResult::ERROR, TStringBuilder() << "exception while " << state
720-
<< ": " << ex.what());
721-
}
722755

723-
if (newYamlVersion && *newYamlVersion != *Self->MainConfigYamlVersion + 1) {
724-
return FinishWithError(TResult::ERROR, TStringBuilder() << "version must be increasing by one"
725-
<< " new version# " << *newYamlVersion << " expected version# " << *Self->MainConfigYamlVersion + 1);
726-
} else if (newStorageYamlVersion && *newStorageYamlVersion != Self->StorageConfig->GetExpectedStorageYamlVersion()) {
727-
return FinishWithError(TResult::ERROR, TStringBuilder() << "version must be increasing by one"
728-
<< " new version# " << *newStorageYamlVersion
729-
<< " expected version# " << Self->StorageConfig->GetExpectedStorageYamlVersion());
730-
}
756+
if (effective) {
757+
state = "parsing final config";
731758

732-
TString errorReason;
733-
NKikimrBlobStorage::TStorageConfig config(*Self->StorageConfig);
734-
const bool success = DeriveStorageConfig(appConfig, &config, &errorReason);
735-
if (!success) {
736-
return FinishWithError(TResult::ERROR, TStringBuilder() << "error while deriving StorageConfig: "
737-
<< errorReason);
759+
NKikimrConfig::TAppConfig appConfig;
760+
NYaml::Parse(*effective, NYaml::GetJsonToProtoConfig(), appConfig, true);
761+
762+
if (TString errorReason; !DeriveStorageConfig(appConfig, &config, &errorReason)) {
763+
return FinishWithError(TResult::ERROR, TStringBuilder()
764+
<< "error while deriving StorageConfig: " << errorReason);
765+
}
766+
}
767+
} catch (const std::exception& ex) {
768+
return FinishWithError(TResult::ERROR, TStringBuilder() << "exception while " << state
769+
<< ": " << ex.what());
738770
}
739771

740772
if (NewYaml) {
741773
if (const auto& error = UpdateConfigComposite(config, *NewYaml, std::nullopt)) {
742774
return FinishWithError(TResult::ERROR, TStringBuilder() << "failed to update config yaml: " << *error);
743775
}
744-
} else {
745-
config.SetConfigComposite(Self->StorageConfig->GetConfigComposite());
776+
}
777+
778+
if (newExpectedStorageYamlVersion) {
779+
config.SetExpectedStorageYamlVersion(*newExpectedStorageYamlVersion);
746780
}
747781

748782
if (newStorageYaml) {
749783
// make new compressed storage yaml section
750784
TString s;
751785
if (TStringOutput output(s); true) {
752786
TZstdCompress zstd(&output);
753-
::Save(&zstd, *newStorageYaml);
754-
::Save(&zstd, *newStorageYamlVersion);
787+
zstd << *newStorageYaml;
755788
}
756789
config.SetCompressedStorageYaml(s);
757-
config.SetExpectedStorageYamlVersion(*newStorageYamlVersion + 1);
758-
} else if (switchDedicatedStorageSection && !*switchDedicatedStorageSection) {
759-
// delete compressed storage yaml section as this request turns off dedicated storage yaml
760-
} else if (Self->StorageConfig->HasCompressedStorageYaml()) {
761-
// retain current storage yaml
762-
config.SetCompressedStorageYaml(Self->StorageConfig->GetCompressedStorageYaml());
763-
config.SetExpectedStorageYamlVersion(Self->StorageConfig->GetExpectedStorageYamlVersion());
790+
} else if (!targetDedicatedStorageSection) {
791+
config.ClearCompressedStorageYaml();
764792
}
765793

766794
// advance the config generation
@@ -774,9 +802,7 @@ namespace NKikimr::NStorage {
774802
<< "ReplaceStorageConfig config validation failed: " << *error);
775803
}
776804

777-
const bool pushToConsole = true;
778-
779-
if (!pushToConsole || !request.GetSkipConsoleValidation()) {
805+
if (request.GetSkipConsoleValidation() || !NewYaml) {
780806
return StartProposition(&config);
781807
}
782808

@@ -785,7 +811,7 @@ namespace NKikimr::NStorage {
785811
!Self->SelfManagementEnabled &&
786812
config.GetSelfManagementConfig().GetEnabled();
787813

788-
if (NewYaml && !Self->EnqueueConsoleConfigValidation(SelfId(), enablingDistconf, *NewYaml)) {
814+
if (!Self->EnqueueConsoleConfigValidation(SelfId(), enablingDistconf, *NewYaml)) {
789815
FinishWithError(TResult::ERROR, "console pipe is not available");
790816
} else {
791817
ProposedStorageConfig = std::move(config);

‎ydb/core/blobstorage/nodewarden/distconf_mon.cpp

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#include "distconf.h"
2+
#include "node_warden_impl.h"
23

34
#include <google/protobuf/util/json_util.h>
45

@@ -171,7 +172,9 @@ namespace NKikimr::NStorage {
171172
if (config) {
172173
TString s;
173174
NProtoBuf::TextFormat::PrintToString(*config, &s);
174-
out << "<pre>" << s << "</pre>";
175+
out << "<pre>";
176+
EscapeHtmlString(out, s);
177+
out << "</pre>";
175178
} else {
176179
out << "not defined";
177180
}

‎ydb/core/blobstorage/nodewarden/node_warden.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ namespace NKikimr {
2424
std::optional<NKikimrConfig::TSelfManagementConfig> SelfManagementConfig;
2525
TString ConfigStorePath;
2626
std::optional<NKikimrBlobStorage::TYamlConfig> YamlConfig;
27+
TString StartupConfigYaml;
28+
std::optional<TString> StartupStorageYaml;
2729
TIntrusivePtr<IPDiskServiceFactory> PDiskServiceFactory;
2830
TIntrusivePtr<TAllVDiskKinds> AllVDiskKinds;
2931
TIntrusivePtr<NPDisk::TDriveModelDb> AllDriveModels;

0 commit comments

Comments
 (0)