Skip to content

Commit 4facf60

Browse files
JorresJorres
committed
feat: enable medium test webhooks
1 parent 6e8d37c commit 4facf60

File tree

9 files changed

+330
-15
lines changed

9 files changed

+330
-15
lines changed

‎api/v1alpha1/storage_webhook.go

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import (
44
"context"
55
"fmt"
66
"math/rand"
7+
"reflect"
8+
"sort"
79

810
"github.com/golang-jwt/jwt/v4"
911
"github.com/google/go-cmp/cmp"
@@ -443,6 +445,49 @@ func (r *Storage) ValidateUpdate(old runtime.Object) error {
443445
return crdCheckError
444446
}
445447

448+
if err := r.validateGrpcPorts(); err != nil {
449+
return err
450+
}
451+
452+
return nil
453+
}
454+
455+
func (r *Storage) validateGrpcPorts() error {
456+
servicePorts := []int32{}
457+
458+
firstPort := int32(GRPCPort)
459+
if r.Spec.Service.GRPC.Port != 0 {
460+
firstPort = r.Spec.Service.GRPC.Port
461+
}
462+
servicePorts = append(servicePorts, firstPort)
463+
if r.Spec.Service.GRPC.AdditionalPort != 0 {
464+
servicePorts = append(servicePorts, r.Spec.Service.GRPC.AdditionalPort)
465+
}
466+
configuration, err := ParseConfiguration(r.Spec.Configuration)
467+
if err != nil {
468+
return fmt.Errorf("failed to parse configuration immediately after building it, should not happen, %w", err)
469+
}
470+
471+
configurationPorts := []int32{}
472+
if configuration.GrpcConfig.Port != 0 {
473+
configurationPorts = append(configurationPorts, configuration.GrpcConfig.Port)
474+
}
475+
if configuration.GrpcConfig.SslPort != 0 {
476+
configurationPorts = append(configurationPorts, configuration.GrpcConfig.SslPort)
477+
}
478+
479+
sort.Slice(servicePorts, func(i, j int) bool {
480+
return servicePorts[i] < servicePorts[j]
481+
})
482+
483+
sort.Slice(configurationPorts, func(i, j int) bool {
484+
return configurationPorts[i] < configurationPorts[j]
485+
})
486+
487+
if !reflect.DeepEqual(servicePorts, configurationPorts) {
488+
return fmt.Errorf("grpc port mismatch: %v in spec.service.grpc, %v in YDB configuration", servicePorts, configurationPorts)
489+
}
490+
446491
return nil
447492
}
448493

‎internal/configuration/schema/configuration.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ type Configuration struct {
1010
DomainsConfig *DomainsConfig `yaml:"domains_config"`
1111
Hosts []Host `yaml:"hosts,omitempty"`
1212
KeyConfig *KeyConfig `yaml:"key_config,omitempty"`
13+
GrpcConfig *GrpcConfig `yaml:"grpc_config,omitempty"`
1314
}
1415

1516
type Metadata struct {

‎internal/configuration/schema/grpc.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
package schema
2+
3+
type GrpcConfig struct {
4+
Port int32 `yaml:"port,omitempty"`
5+
SslPort int32 `yaml:"ssl_port,omitempty"`
6+
}

‎internal/controllers/databasenodeset/controller_test.go

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ var _ = Describe("DatabaseNodeSet controller medium tests", func() {
7171
}
7272
Expect(k8sClient.Create(ctx, &namespace)).Should(Succeed())
7373

74-
storageSample = *testobjects.DefaultStorage(filepath.Join("..", "..", "..", "tests", "data", "storage-mirror-3-dc-config.yaml"))
74+
storageSample = *testobjects.DefaultBlock42Storage(filepath.Join("..", "..", "..", "tests", "data", "storage-block-4-2-config.yaml"))
7575
Expect(k8sClient.Create(ctx, &storageSample)).Should(Succeed())
7676

7777
By("checking that Storage created on local cluster...")
@@ -100,10 +100,11 @@ var _ = Describe("DatabaseNodeSet controller medium tests", func() {
100100
}, test.Timeout, test.Interval).ShouldNot(HaveOccurred())
101101

102102
databaseSample = *testobjects.DefaultDatabase()
103+
databaseSample.Spec.DatabaseNodeSpec.Nodes = 4
103104
databaseSample.Spec.NodeSets = append(databaseSample.Spec.NodeSets, v1alpha1.DatabaseNodeSetSpecInline{
104105
Name: testNodeSetName,
105106
DatabaseNodeSpec: v1alpha1.DatabaseNodeSpec{
106-
Nodes: 1,
107+
Nodes: 4,
107108
},
108109
})
109110

@@ -154,23 +155,23 @@ var _ = Describe("DatabaseNodeSet controller medium tests", func() {
154155
v1alpha1.AnnotationUpdateStrategyOnDelete: "true",
155156
}
156157

157-
foundDatabase.Spec.NodeSets = append(foundDatabase.Spec.NodeSets, v1alpha1.DatabaseNodeSetSpecInline{
158+
foundDatabase.Spec.NodeSets = []v1alpha1.DatabaseNodeSetSpecInline{{
158159
Name: testNodeSetName + "-labeled",
159160
Labels: map[string]string{
160161
testNodeSetLabel: "true",
161162
},
162163
DatabaseNodeSpec: v1alpha1.DatabaseNodeSpec{
163-
Nodes: 1,
164+
Nodes: 2,
164165
},
165-
})
166+
}}
166167

167168
foundDatabase.Spec.NodeSets = append(foundDatabase.Spec.NodeSets, v1alpha1.DatabaseNodeSetSpecInline{
168169
Name: testNodeSetName + "-annotated",
169170
Annotations: map[string]string{
170171
v1alpha1.AnnotationDataCenter: "envtest",
171172
},
172173
DatabaseNodeSpec: v1alpha1.DatabaseNodeSpec{
173-
Nodes: 1,
174+
Nodes: 2,
174175
},
175176
})
176177

‎internal/controllers/storage/controller_test.go

Lines changed: 59 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ import (
77
"strings"
88
"testing"
99

10+
"gopkg.in/yaml.v3"
11+
1012
. "github.com/onsi/ginkgo/v2"
1113
. "github.com/onsi/gomega"
1214
appsv1 "k8s.io/api/apps/v1"
@@ -315,6 +317,10 @@ var _ = Describe("Storage controller medium tests", func() {
315317

316318
storage.Spec.Service.GRPC.Port = 2137
317319

320+
configWithNewPorts, err := patchGRPCPortsInConfiguration(storage.Spec.Configuration, 2137, -1)
321+
Expect(err).To(BeNil())
322+
storage.Spec.Configuration = configWithNewPorts
323+
318324
Expect(k8sClient.Update(ctx, &storage)).Should(Succeed())
319325

320326
var svc corev1.Service
@@ -349,9 +355,13 @@ var _ = Describe("Storage controller medium tests", func() {
349355
Namespace: testobjects.YdbNamespace,
350356
}, &storage)).Should(Succeed())
351357

352-
storage.Spec.Service.GRPC.Port = 2135
358+
storage.Spec.Service.GRPC.Port = v1alpha1.GRPCPort
353359
storage.Spec.Service.GRPC.AdditionalPort = 2136
354360

361+
configWithNewPorts, err := patchGRPCPortsInConfiguration(storage.Spec.Configuration, 2135, 2136)
362+
Expect(err).To(BeNil())
363+
storage.Spec.Configuration = configWithNewPorts
364+
355365
Expect(k8sClient.Update(ctx, &storage)).Should(Succeed())
356366

357367
var svc corev1.Service
@@ -370,7 +380,7 @@ var _ = Describe("Storage controller medium tests", func() {
370380

371381
ports := svc.Spec.Ports
372382
g.Expect(len(ports)).To(Equal(2), "expected 2 ports but got %d", len(ports))
373-
g.Expect(ports[0].Port).To(Equal(int32(2135)))
383+
g.Expect(ports[0].Port).To(Equal(int32(v1alpha1.GRPCPort)))
374384
g.Expect(ports[0].Name).To(Equal(v1alpha1.GRPCServicePortName))
375385
g.Expect(ports[1].Port).To(Equal(storage.Spec.Service.GRPC.AdditionalPort))
376386
g.Expect(ports[1].Name).To(Equal(v1alpha1.GRPCServiceAdditionalPortName))
@@ -380,5 +390,52 @@ var _ = Describe("Storage controller medium tests", func() {
380390
"Service %s/%s should eventually have proper ports", testobjects.YdbNamespace, serviceName,
381391
)
382392
})
393+
394+
By("Forbid to edit grpc ports, when out of sync with YDB config...", func() {
395+
storage := v1alpha1.Storage{}
396+
Expect(k8sClient.Get(ctx, types.NamespacedName{
397+
Name: testobjects.StorageName,
398+
Namespace: testobjects.YdbNamespace,
399+
}, &storage)).Should(Succeed())
400+
401+
storage.Spec.Service.GRPC.Port = v1alpha1.GRPCPort
402+
By("Specify 2136 in manifest spec...")
403+
storage.Spec.Service.GRPC.AdditionalPort = 2136
404+
405+
By("And then specify 2137 in manifest spec...")
406+
configWithNewPorts, err := patchGRPCPortsInConfiguration(storage.Spec.Configuration, v1alpha1.GRPCPort, 2137)
407+
Expect(err).To(BeNil())
408+
storage.Spec.Configuration = configWithNewPorts
409+
410+
err = k8sClient.Update(ctx, &storage)
411+
Expect(err).To(MatchError(ContainSubstring("grpc port mismatch")))
412+
})
383413
})
384414
})
415+
416+
func patchGRPCPortsInConfiguration(in string, port, sslPort int) (string, error) {
417+
m := make(map[string]interface{})
418+
if err := yaml.Unmarshal([]byte(in), &m); err != nil {
419+
return "", err
420+
}
421+
422+
cfg, _ := m["grpc_config"].(map[string]interface{})
423+
if cfg == nil {
424+
cfg = make(map[string]interface{})
425+
}
426+
427+
if sslPort != -1 {
428+
cfg["ssl_port"] = sslPort
429+
}
430+
if port != -1 {
431+
cfg["port"] = port
432+
}
433+
m["grpc_config"] = cfg
434+
435+
res, err := yaml.Marshal(m)
436+
if err != nil {
437+
return "", err
438+
}
439+
440+
return string(res), nil
441+
}

‎internal/resources/storage_statefulset.go

Lines changed: 33 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -356,6 +356,38 @@ func (b *StorageStatefulSetBuilder) buildCaStorePatchingInitContainerVolumeMount
356356
return volumeMounts
357357
}
358358

359+
func (b *StorageStatefulSetBuilder) buildContainerPorts() []corev1.ContainerPort {
360+
podPorts := []corev1.ContainerPort{{
361+
Name: "grpc", ContainerPort: api.GRPCPort,
362+
}, {
363+
Name: "interconnect", ContainerPort: api.InterconnectPort,
364+
}, {
365+
Name: "status", ContainerPort: api.StatusPort,
366+
}}
367+
368+
firstGRPCPort := corev1.ContainerPort{
369+
Name: "grpc",
370+
ContainerPort: api.GRPCPort,
371+
}
372+
373+
overrideGRPCPort := b.Spec.StorageClusterSpec.Service.GRPC.Port
374+
if overrideGRPCPort != 0 {
375+
firstGRPCPort.ContainerPort = overrideGRPCPort
376+
}
377+
378+
podPorts = append(podPorts, firstGRPCPort)
379+
380+
additionalPort := b.Spec.StorageClusterSpec.Service.GRPC.AdditionalPort
381+
if additionalPort != 0 {
382+
podPorts = append(podPorts, corev1.ContainerPort{
383+
Name: "additional-grpc",
384+
ContainerPort: additionalPort,
385+
})
386+
}
387+
388+
return podPorts
389+
}
390+
359391
func (b *StorageStatefulSetBuilder) buildContainer() corev1.Container { // todo add init container for sparse files?
360392
command, args := b.buildContainerArgs()
361393
containerResources := corev1.ResourceRequirements{}
@@ -376,13 +408,7 @@ func (b *StorageStatefulSetBuilder) buildContainer() corev1.Container { // todo
376408

377409
SecurityContext: mergeSecurityContextWithDefaults(b.Spec.SecurityContext),
378410

379-
Ports: []corev1.ContainerPort{{
380-
Name: "grpc", ContainerPort: api.GRPCPort,
381-
}, {
382-
Name: "interconnect", ContainerPort: api.InterconnectPort,
383-
}, {
384-
Name: "status", ContainerPort: api.StatusPort,
385-
}},
411+
Ports: b.buildContainerPorts(),
386412

387413
VolumeMounts: b.buildVolumeMounts(),
388414
Resources: containerResources,

‎internal/test/k8s_helpers.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,13 +149,20 @@ func SetupK8STestManager(testCtx *context.Context, k8sClient *client.Client, con
149149

150150
// FIXME: find a better way?
151151
_, curfile, _, _ := runtime.Caller(0) //nolint:dogsled
152+
webhookDir := filepath.Join(curfile, "..", "..", "..", "config", "webhook")
152153
testEnv := &envtest.Environment{
153154
CRDDirectoryPaths: []string{
154155
filepath.Join(curfile, "..", "..", "..", "deploy", "ydb-operator", "crds"),
155156
filepath.Join(filepath.Dir(curfile), "extra_crds"),
156157
},
157158
ErrorIfCRDPathMissing: true,
158159
UseExistingCluster: &useExistingCluster,
160+
WebhookInstallOptions: envtest.WebhookInstallOptions{
161+
Paths: []string{webhookDir},
162+
LocalServingHost: "127.0.0.1",
163+
LocalServingPort: 9443,
164+
LocalServingCertDir: "",
165+
},
159166
}
160167

161168
BeforeSuite(func() {
@@ -174,6 +181,9 @@ func SetupK8STestManager(testCtx *context.Context, k8sClient *client.Client, con
174181
mgr, err := ctrl.NewManager(cfg, ctrl.Options{
175182
MetricsBindAddress: "0",
176183
Scheme: scheme.Scheme,
184+
Host: testEnv.WebhookInstallOptions.LocalServingHost,
185+
Port: testEnv.WebhookInstallOptions.LocalServingPort,
186+
CertDir: testEnv.WebhookInstallOptions.LocalServingCertDir,
177187
})
178188
Expect(err).ToNot(HaveOccurred())
179189

@@ -183,6 +193,11 @@ func SetupK8STestManager(testCtx *context.Context, k8sClient *client.Client, con
183193
Expect(c.SetupWithManager(mgr)).To(Succeed())
184194
}
185195

196+
// Setup webhooks
197+
Expect((&v1alpha1.Storage{}).SetupWebhookWithManager(mgr)).To(Succeed())
198+
Expect((&v1alpha1.Database{}).SetupWebhookWithManager(mgr)).To(Succeed())
199+
Expect(v1alpha1.RegisterMonitoringValidatingWebhook(mgr, true)).To(Succeed())
200+
186201
go func() {
187202
defer GinkgoRecover()
188203
err = mgr.Start(ctx)

0 commit comments

Comments
 (0)