Skip to content

Commit 0b791fa

Browse files
kobzonegakobzonega
authored
YDBOPS-9608 support dynconfig (#200)
1 parent d91bb38 commit 0b791fa

File tree

10 files changed

+318
-104
lines changed

10 files changed

+318
-104
lines changed

‎Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ kind-load:
7979

8080
.PHONY: unit-test
8181
unit-test: manifests generate fmt vet envtest ## Run unit tests
82-
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use --arch=amd64 $(ENVTEST_K8S_VERSION) -p path)" go test -v -timeout 1800s -p 1 ./internal/controllers/... -ginkgo.v -coverprofile cover.out
82+
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use --arch=amd64 $(ENVTEST_K8S_VERSION) -p path)" go test -v -timeout 1800s -p 1 ./internal/... -ginkgo.v -coverprofile cover.out
8383

8484
.PHONY: e2e-test
8585
e2e-test: manifests generate fmt vet docker-build kind-init kind-load ## Run e2e tests

‎api/v1alpha1/configuration.go

Lines changed: 45 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package v1alpha1
22

33
import (
4+
"bytes"
45
"crypto/sha256"
56
"fmt"
67
"path"
@@ -24,7 +25,7 @@ func hash(text string) string {
2425
return fmt.Sprintf("%x", h.Sum(nil))
2526
}
2627

27-
func generateSomeDefaults(cr *Storage, crDB *Database) schema.Configuration {
28+
func generateHosts(cr *Storage) []schema.Host {
2829
var hosts []schema.Host
2930

3031
for i := 0; i < int(cr.Spec.Nodes); i++ {
@@ -57,6 +58,10 @@ func generateSomeDefaults(cr *Storage, crDB *Database) schema.Configuration {
5758
}
5859
}
5960

61+
return hosts
62+
}
63+
64+
func generateKeyConfig(cr *Storage, crDB *Database) *schema.KeyConfig {
6065
var keyConfig *schema.KeyConfig
6166
if crDB != nil && crDB.Spec.Encryption != nil && crDB.Spec.Encryption.Enabled {
6267
keyConfig = &schema.KeyConfig{
@@ -71,25 +76,10 @@ func generateSomeDefaults(cr *Storage, crDB *Database) schema.Configuration {
7176
}
7277
}
7378

74-
return schema.Configuration{
75-
Hosts: hosts,
76-
KeyConfig: keyConfig,
77-
}
79+
return keyConfig
7880
}
7981

80-
func tryFillMissingSections(
81-
resultConfig map[string]interface{},
82-
generatedConfig schema.Configuration,
83-
) {
84-
if resultConfig["hosts"] == nil {
85-
resultConfig["hosts"] = generatedConfig.Hosts
86-
}
87-
if generatedConfig.KeyConfig != nil {
88-
resultConfig["key_config"] = generatedConfig.KeyConfig
89-
}
90-
}
91-
92-
func BuildConfiguration(cr *Storage, crDB *Database) (string, error) {
82+
func BuildConfiguration(cr *Storage, crDB *Database) ([]byte, error) {
9383
config := make(map[string]interface{})
9484

9585
// If any kind of configuration exists on Database object, then
@@ -103,18 +93,47 @@ func BuildConfiguration(cr *Storage, crDB *Database) (string, error) {
10393
rawYamlConfiguration = cr.Spec.Configuration
10494
}
10595

106-
err := yaml.Unmarshal([]byte(rawYamlConfiguration), &config)
96+
dynconfig, err := ParseDynconfig(rawYamlConfiguration)
97+
if err == nil {
98+
if dynconfig.Config["hosts"] == nil {
99+
hosts := generateHosts(cr)
100+
dynconfig.Config["hosts"] = hosts
101+
}
102+
103+
return yaml.Marshal(dynconfig)
104+
}
105+
106+
err = yaml.Unmarshal([]byte(rawYamlConfiguration), &config)
107107
if err != nil {
108-
return "", err
108+
return nil, err
109109
}
110110

111-
generatedConfig := generateSomeDefaults(cr, crDB)
112-
tryFillMissingSections(config, generatedConfig)
111+
if config["hosts"] == nil {
112+
hosts := generateHosts(cr)
113+
config["hosts"] = hosts
114+
}
113115

114-
data, err := yaml.Marshal(config)
115-
if err != nil {
116-
return "", err
116+
// Will be removed by YDBOPS-9692
117+
keyConfig := generateKeyConfig(cr, crDB)
118+
if keyConfig != nil {
119+
config["key_config"] = keyConfig
117120
}
118121

119-
return string(data), nil
122+
return yaml.Marshal(config)
123+
}
124+
125+
func ParseConfig(rawYamlConfiguration string) (schema.Configuration, error) {
126+
config := schema.Configuration{}
127+
dec := yaml.NewDecoder(bytes.NewReader([]byte(rawYamlConfiguration)))
128+
dec.KnownFields(false)
129+
err := dec.Decode(&config)
130+
return config, err
131+
}
132+
133+
func ParseDynconfig(rawYamlConfiguration string) (schema.Dynconfig, error) {
134+
dynconfig := schema.Dynconfig{}
135+
dec := yaml.NewDecoder(bytes.NewReader([]byte(rawYamlConfiguration)))
136+
dec.KnownFields(true)
137+
err := dec.Decode(&dynconfig)
138+
return dynconfig, err
120139
}

‎api/v1alpha1/database_webhook.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,10 @@ func (r *DatabaseDefaulter) Default(ctx context.Context, obj runtime.Object) err
5858
database := obj.(*Database)
5959
databaselog.Info("default", "name", database.Name)
6060

61+
if !database.Spec.OperatorSync {
62+
return nil
63+
}
64+
6165
if database.Spec.StorageClusterRef.Namespace == "" {
6266
database.Spec.StorageClusterRef.Namespace = database.Namespace
6367
}
@@ -150,7 +154,7 @@ func (r *DatabaseDefaulter) Default(ctx context.Context, obj runtime.Object) err
150154
if err != nil {
151155
return err
152156
}
153-
database.Spec.Configuration = configuration
157+
database.Spec.Configuration = string(configuration)
154158
}
155159

156160
return nil

‎api/v1alpha1/storage_webhook.go

Lines changed: 53 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -61,18 +61,15 @@ func (r *Storage) GetGRPCServiceEndpoint() string {
6161
return fmt.Sprintf("%s:%d", host, GRPCPort)
6262
}
6363

64-
// +k8s:deepcopy-gen=false
65-
type PartialHostsConfig struct {
66-
Hosts []schema.Host `yaml:"hosts,flow"`
67-
}
68-
6964
func (r *Storage) GetHostFromConfigEndpoint() string {
65+
configuration := make(map[string]interface{})
66+
7067
// skip handle error because we already checked in webhook
71-
hostsConfig := PartialHostsConfig{}
72-
_ = yaml.Unmarshal([]byte(r.Spec.Configuration), &hostsConfig)
68+
_ = yaml.Unmarshal([]byte(r.Spec.Configuration), &configuration)
69+
hostsConfig := configuration["hosts"].([]schema.Host)
7370

7471
randNum := rand.Int31n(r.Spec.Nodes) // #nosec G404
75-
host := hostsConfig.Hosts[randNum].Host
72+
host := hostsConfig[randNum].Host
7673
return fmt.Sprintf("%s:%d", host, GRPCPort)
7774
}
7875

@@ -97,15 +94,6 @@ func (r *Storage) IsRemoteNodeSetsOnly() bool {
9794
return true
9895
}
9996

100-
// +k8s:deepcopy-gen=false
101-
type PartialDomainsConfig struct {
102-
DomainsConfig struct {
103-
SecurityConfig struct {
104-
EnforceUserTokenRequirement bool `yaml:"enforce_user_token_requirement"`
105-
} `yaml:"security_config"`
106-
} `yaml:"domains_config"`
107-
}
108-
10997
// StorageDefaulter mutates Storages
11098
// +k8s:deepcopy-gen=false
11199
type StorageDefaulter struct {
@@ -119,6 +107,10 @@ func (r *StorageDefaulter) Default(ctx context.Context, obj runtime.Object) erro
119107
storage := obj.(*Storage)
120108
storagelog.Info("default", "name", storage.Name)
121109

110+
if !storage.Spec.OperatorSync {
111+
return nil
112+
}
113+
122114
if storage.Spec.Image == nil {
123115
storage.Spec.Image = &PodImage{}
124116
}
@@ -174,7 +166,7 @@ func (r *StorageDefaulter) Default(ctx context.Context, obj runtime.Object) erro
174166
if err != nil {
175167
return err
176168
}
177-
storage.Spec.Configuration = configuration
169+
storage.Spec.Configuration = string(configuration)
178170

179171
return nil
180172
}
@@ -187,23 +179,28 @@ var _ webhook.Validator = &Storage{}
187179
func (r *Storage) ValidateCreate() error {
188180
storagelog.Info("validate create", "name", r.Name)
189181

190-
configuration := make(map[string]interface{})
191-
err := yaml.Unmarshal([]byte(r.Spec.Configuration), &configuration)
192-
if err != nil {
193-
return fmt.Errorf("failed to parse .spec.configuration, error: %w", err)
182+
var configuration schema.Configuration
183+
184+
rawYamlConfiguration := r.Spec.Configuration
185+
dynconfig, err := ParseDynconfig(r.Spec.Configuration)
186+
if err == nil {
187+
config, err := yaml.Marshal(dynconfig.Config)
188+
if err != nil {
189+
return fmt.Errorf("failed to parse .config from dynconfig, error: %w", err)
190+
}
191+
rawYamlConfiguration = string(config)
194192
}
195193

196-
hostsConfig := PartialHostsConfig{}
197-
err = yaml.Unmarshal([]byte(r.Spec.Configuration), &hostsConfig)
194+
configuration, err = ParseConfig(rawYamlConfiguration)
198195
if err != nil {
199-
return fmt.Errorf("failed to parse YAML to determine `hosts`, error: %w", err)
196+
return fmt.Errorf("failed to parse .spec.configuration, error: %w", err)
200197
}
201198

202199
var nodesNumber int32
203-
if len(hostsConfig.Hosts) == 0 {
200+
if len(configuration.Hosts) == 0 {
204201
nodesNumber = r.Spec.Nodes
205202
} else {
206-
nodesNumber = int32(len(hostsConfig.Hosts))
203+
nodesNumber = int32(len(configuration.Hosts))
207204
}
208205

209206
minNodesPerErasure := map[ErasureType]int32{
@@ -215,15 +212,11 @@ func (r *Storage) ValidateCreate() error {
215212
return fmt.Errorf("erasure type %v requires at least %v storage nodes", r.Spec.Erasure, minNodesPerErasure[r.Spec.Erasure])
216213
}
217214

218-
yamlConfig := PartialDomainsConfig{}
219-
err = yaml.Unmarshal([]byte(r.Spec.Configuration), &yamlConfig)
220-
if err != nil {
221-
return fmt.Errorf("failed to parse YAML to determine `enforce_user_token_requirement`, error: %w", err)
222-
}
223-
224215
var authEnabled bool
225-
if yamlConfig.DomainsConfig.SecurityConfig.EnforceUserTokenRequirement {
226-
authEnabled = true
216+
if configuration.DomainsConfig.SecurityConfig != nil {
217+
if configuration.DomainsConfig.SecurityConfig.EnforceUserTokenRequirement != nil {
218+
authEnabled = *configuration.DomainsConfig.SecurityConfig.EnforceUserTokenRequirement
219+
}
227220
}
228221

229222
if (authEnabled && r.Spec.OperatorConnection == nil) || (!authEnabled && r.Spec.OperatorConnection != nil) {
@@ -285,23 +278,28 @@ func hasUpdatesBesidesFrozen(oldStorage, newStorage *Storage) (bool, string) {
285278
func (r *Storage) ValidateUpdate(old runtime.Object) error {
286279
storagelog.Info("validate update", "name", r.Name)
287280

288-
configuration := make(map[string]interface{})
289-
err := yaml.Unmarshal([]byte(r.Spec.Configuration), &configuration)
290-
if err != nil {
291-
return fmt.Errorf("failed to parse .spec.configuration, error: %w", err)
281+
var configuration schema.Configuration
282+
283+
rawYamlConfiguration := r.Spec.Configuration
284+
dynconfig, err := ParseDynconfig(r.Spec.Configuration)
285+
if err == nil {
286+
config, err := yaml.Marshal(dynconfig.Config)
287+
if err != nil {
288+
return fmt.Errorf("failed to parse .config from dynconfig, error: %w", err)
289+
}
290+
rawYamlConfiguration = string(config)
292291
}
293292

294-
hostsConfig := PartialHostsConfig{}
295-
err = yaml.Unmarshal([]byte(r.Spec.Configuration), &hostsConfig)
293+
configuration, err = ParseConfig(rawYamlConfiguration)
296294
if err != nil {
297-
return fmt.Errorf("failed to parse YAML to determine `hosts`, error: %w", err)
295+
return fmt.Errorf("failed to parse .spec.configuration, error: %w", err)
298296
}
299297

300298
var nodesNumber int32
301-
if len(hostsConfig.Hosts) == 0 {
299+
if len(configuration.Hosts) == 0 {
302300
nodesNumber = r.Spec.Nodes
303301
} else {
304-
nodesNumber = int32(len(hostsConfig.Hosts))
302+
nodesNumber = int32(len(configuration.Hosts))
305303
}
306304

307305
minNodesPerErasure := map[ErasureType]int32{
@@ -313,6 +311,17 @@ func (r *Storage) ValidateUpdate(old runtime.Object) error {
313311
return fmt.Errorf("erasure type %v requires at least %v storage nodes", r.Spec.Erasure, minNodesPerErasure[r.Spec.Erasure])
314312
}
315313

314+
var authEnabled bool
315+
if configuration.DomainsConfig.SecurityConfig != nil {
316+
if configuration.DomainsConfig.SecurityConfig.EnforceUserTokenRequirement != nil {
317+
authEnabled = *configuration.DomainsConfig.SecurityConfig.EnforceUserTokenRequirement
318+
}
319+
}
320+
321+
if (authEnabled && r.Spec.OperatorConnection == nil) || (!authEnabled && r.Spec.OperatorConnection != nil) {
322+
return fmt.Errorf("field 'spec.operatorConnection' does not align with config option `enforce_user_token_requirement: %t`", authEnabled)
323+
}
324+
316325
if !r.Spec.OperatorSync {
317326
oldStorage := old.(*Storage)
318327

@@ -331,21 +340,6 @@ func (r *Storage) ValidateUpdate(old runtime.Object) error {
331340
}
332341
}
333342

334-
yamlConfig := PartialDomainsConfig{}
335-
err = yaml.Unmarshal([]byte(r.Spec.Configuration), &yamlConfig)
336-
if err != nil {
337-
return fmt.Errorf("failed to parse YAML to determine `enforce_user_token_requirement`, error: %w", err)
338-
}
339-
340-
var authEnabled bool
341-
if yamlConfig.DomainsConfig.SecurityConfig.EnforceUserTokenRequirement {
342-
authEnabled = true
343-
}
344-
345-
if (authEnabled && r.Spec.OperatorConnection == nil) || (!authEnabled && r.Spec.OperatorConnection != nil) {
346-
return fmt.Errorf("field 'spec.operatorConnection' does not align with config option `enforce_user_token_requirement: %t`", authEnabled)
347-
}
348-
349343
if r.Spec.NodeSets != nil {
350344
var nodesInSetsCount int32
351345
for _, nodeSetInline := range r.Spec.NodeSets {

‎e2e/tests/data/storage-block-4-2-config-tls.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ channel_profile_config:
102102
grpc_config:
103103
start_grpc_proxy: true
104104
ssl_port: 2135
105-
ca: /tls/grpc/ca.crt
105+
ca: /etc/ssl/certs/ca-certificates.crt
106106
cert: /tls/grpc/tls.crt
107107
key: /tls/grpc/tls.key
108108
grpc_memory_quota_bytes: '1073741824'

‎internal/configuration/schema/configuration.go

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,26 @@
11
package schema
22

3+
type Dynconfig struct {
4+
Metadata *Metadata `yaml:"metadata"`
5+
Config map[string]interface{} `yaml:"config"`
6+
AllowedLabels map[string]interface{} `yaml:"allowed_labels"`
7+
SelectorConfig []SelectorConfig `yaml:"selector_config"`
8+
}
39
type Configuration struct {
4-
Hosts []Host `yaml:"hosts"`
5-
KeyConfig *KeyConfig `yaml:"key_config,omitempty"`
10+
DomainsConfig *DomainsConfig `yaml:"domains_config"`
11+
Hosts []Host `yaml:"hosts,omitempty"`
12+
KeyConfig *KeyConfig `yaml:"key_config,omitempty"`
13+
}
14+
15+
type Metadata struct {
16+
Kind string `yaml:"kind"`
17+
Cluster string `yaml:"cluster"`
18+
Version uint64 `yaml:"version"`
19+
ID uint64 `yaml:"id,omitempty"`
20+
}
21+
22+
type SelectorConfig struct {
23+
Description string `yaml:"description"`
24+
Selector map[string]interface{} `yaml:"selector"`
25+
Config map[string]interface{} `yaml:"config"`
626
}

‎internal/configuration/schema/domains.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
package schema
2+
3+
type DomainsConfig struct {
4+
SecurityConfig *SecurityConfig `yaml:"security_config,omitempty"`
5+
}
6+
7+
type SecurityConfig struct {
8+
EnforceUserTokenRequirement *bool `yaml:"enforce_user_token_requirement,omitempty"`
9+
}

0 commit comments

Comments
 (0)