Skip to content

Commit 5bc28b1

Browse files
inelpandzichors
inelpandzic
and
hors
authored
K8SPSMDB-951: Fix custom PBM config cr.yaml example (#1517)
* Fix cr.yaml * Fix default values in cr and update priority config. * Fix retryer CR spec. * Add unit test getting pbm config. * Minor refactor. --------- Co-authored-by: Viacheslav Sarzhan <slava.sarzhan@percona.com>
1 parent 4f2461e commit 5bc28b1

File tree

11 files changed

+327
-44
lines changed

11 files changed

+327
-44
lines changed

‎config/crd/bases/psmdb.percona.com_perconaservermongodbs.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ spec:
5454
type: number
5555
priority:
5656
additionalProperties:
57-
type: string
57+
type: number
5858
type: object
5959
timeouts:
6060
properties:

‎deploy/bundle.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -727,7 +727,7 @@ spec:
727727
type: number
728728
priority:
729729
additionalProperties:
730-
type: string
730+
type: number
731731
type: object
732732
timeouts:
733733
properties:

‎deploy/cr.yaml

Lines changed: 20 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -530,8 +530,8 @@ spec:
530530
# sseCustomerKey: Y3VzdG9tZXIta2V5
531531
# retryer:
532532
# numMaxRetries: 3
533-
# minRetryDelay: 30
534-
# maxRetryDelay: 5
533+
# minRetryDelay: 30ms
534+
# maxRetryDelay: 5m
535535
# region: us-west-2
536536
# prefix: ""
537537
# uploadPartSize: 10485760
@@ -560,25 +560,24 @@ spec:
560560
# oplogSpanMin: 10
561561
compressionType: gzip
562562
compressionLevel: 6
563-
# backup:
564-
# priority:
565-
# "localhost:28019": 2.5
566-
# "localhost:27018": 2.5
567-
# compression: gzip
568-
# compressionLevel: 0
569-
# timeouts:
570-
# startingStatus: 33
571-
# oplogSpanMin:
572-
# restore:
573-
# batchSize: 500
574-
# numInsertionWorkers: 10
575-
# numDownloadWorkers: 4
576-
# maxDownloadBufferMb: 0
577-
# downloadChunkMb: 32
578-
# mongodLocation: /usr/bin/mongo
579-
# mongodLocationMap:
580-
# "node01:2017": /usr/bin/mongo
581-
# "node03:27017": /usr/bin/mongo
563+
# configuration:
564+
# backupOptions:
565+
# priority:
566+
# "localhost:28019": 2.5
567+
# "localhost:27018": 2.5
568+
# timeouts:
569+
# startingStatus: 33
570+
# oplogSpanMin: 10
571+
# restoreOptions:
572+
# batchSize: 500
573+
# numInsertionWorkers: 10
574+
# numDownloadWorkers: 4
575+
# maxDownloadBufferMb: 0
576+
# downloadChunkMb: 32
577+
# mongodLocation: /usr/bin/mongo
578+
# mongodLocationMap:
579+
# "node01:2017": /usr/bin/mongo
580+
# "node03:27017": /usr/bin/mongo
582581
# tasks:
583582
# - name: daily-s3-us-west
584583
# enabled: true

‎deploy/crd.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -727,7 +727,7 @@ spec:
727727
type: number
728728
priority:
729729
additionalProperties:
730-
type: string
730+
type: number
731731
type: object
732732
timeouts:
733733
properties:

‎deploy/cw-bundle.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -727,7 +727,7 @@ spec:
727727
type: number
728728
priority:
729729
additionalProperties:
730-
type: string
730+
type: number
731731
type: object
732732
timeouts:
733733
properties:

‎e2e-tests/version-service/conf/crd.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -727,7 +727,7 @@ spec:
727727
type: number
728728
priority:
729729
additionalProperties:
730-
type: string
730+
type: number
731731
type: object
732732
timeouts:
733733
properties:

‎pkg/apis/psmdb/v1/psmdb_types.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -838,9 +838,9 @@ type BackupTimeouts struct {
838838
}
839839

840840
type BackupOptions struct {
841-
OplogSpanMin float64 `json:"oplogSpanMin"`
842-
Priority map[string]string `json:"priority,omitempty"`
843-
Timeouts *BackupTimeouts `json:"timeouts,omitempty"`
841+
OplogSpanMin float64 `json:"oplogSpanMin"`
842+
Priority map[string]float64 `json:"priority,omitempty"`
843+
Timeouts *BackupTimeouts `json:"timeouts,omitempty"`
844844
}
845845

846846
type RestoreOptions struct {

‎pkg/apis/psmdb/v1/zz_generated.deepcopy.go

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/psmdb/backup/pbm.go

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -228,28 +228,30 @@ func GetPriorities(ctx context.Context, k8sclient client.Client, cluster *api.Pe
228228
}
229229

230230
func GetPBMConfig(ctx context.Context, k8sclient client.Client, cluster *api.PerconaServerMongoDB, stg api.BackupStorageSpec) (config.Config, error) {
231-
conf := config.Config{}
232-
233-
priority, err := GetPriorities(ctx, k8sclient, cluster)
234-
if err != nil {
235-
return conf, errors.Wrap(err, "get priorities")
236-
}
237-
238-
conf = config.Config{
231+
conf := config.Config{
239232
PITR: config.PITRConf{
240233
Enabled: cluster.Spec.Backup.PITR.Enabled,
241234
Compression: cluster.Spec.Backup.PITR.CompressionType,
242235
CompressionLevel: cluster.Spec.Backup.PITR.CompressionLevel,
243236
},
244-
Backup: config.BackupConf{
245-
Priority: priority,
246-
},
247237
}
248238

249239
if cluster.Spec.Backup.Configuration.BackupOptions != nil {
250-
conf.Backup.OplogSpanMin = cluster.Spec.Backup.Configuration.BackupOptions.OplogSpanMin
251-
conf.Backup.Timeouts = &config.BackupTimeouts{
252-
Starting: cluster.Spec.Backup.Configuration.BackupOptions.Timeouts.Starting,
240+
conf.Backup = config.BackupConf{
241+
OplogSpanMin: cluster.Spec.Backup.Configuration.BackupOptions.OplogSpanMin,
242+
Timeouts: &config.BackupTimeouts{
243+
Starting: cluster.Spec.Backup.Configuration.BackupOptions.Timeouts.Starting,
244+
},
245+
}
246+
247+
if cluster.Spec.Backup.Configuration.BackupOptions.Priority != nil {
248+
conf.Backup.Priority = cluster.Spec.Backup.Configuration.BackupOptions.Priority
249+
} else {
250+
priority, err := GetPriorities(ctx, k8sclient, cluster)
251+
if err != nil {
252+
return conf, errors.Wrap(err, "get priorities")
253+
}
254+
conf.Backup.Priority = priority
253255
}
254256
}
255257

‎pkg/psmdb/backup/pbm_test.go

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
package backup
2+
3+
import (
4+
"context"
5+
"os"
6+
"testing"
7+
8+
corev1 "k8s.io/api/core/v1"
9+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
10+
// "k8s.io/apimachinery/pkg/runtime/schema"
11+
"k8s.io/client-go/kubernetes/scheme"
12+
"sigs.k8s.io/controller-runtime/pkg/client"
13+
"sigs.k8s.io/controller-runtime/pkg/client/fake" // nolint
14+
"sigs.k8s.io/yaml"
15+
16+
api "github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1"
17+
)
18+
19+
func TestApplyCustomPBMConfig(t *testing.T) {
20+
cr := expectedCR(t)
21+
22+
storage := cr.Spec.Backup.Storages["test-s3-storage"]
23+
24+
cli := buildFakeClient(t)
25+
26+
config, err := GetPBMConfig(context.Background(), cli, cr, storage)
27+
if err != nil {
28+
t.Fatal(err)
29+
}
30+
31+
expectedS3Retryer := storage.S3.Retryer
32+
if expectedS3Retryer.NumMaxRetries != config.Storage.S3.Retryer.NumMaxRetries {
33+
t.Errorf("expected %d, got %d", expectedS3Retryer.NumMaxRetries, config.Storage.S3.Retryer.NumMaxRetries)
34+
}
35+
if expectedS3Retryer.MaxRetryDelay.Duration != config.Storage.S3.Retryer.MaxRetryDelay {
36+
t.Errorf("expected %d, got %d", expectedS3Retryer.MaxRetryDelay.Duration, config.Storage.S3.Retryer.MaxRetryDelay)
37+
}
38+
if expectedS3Retryer.MinRetryDelay.Duration != config.Storage.S3.Retryer.MinRetryDelay {
39+
t.Errorf("expected %d, got %d", expectedS3Retryer.MinRetryDelay.Duration, config.Storage.S3.Retryer.MinRetryDelay)
40+
}
41+
42+
expectedBackupOptions := cr.Spec.Backup.Configuration.BackupOptions
43+
if expectedBackupOptions.OplogSpanMin != config.Backup.OplogSpanMin {
44+
t.Errorf("expected %f, got %f", expectedBackupOptions.OplogSpanMin, config.Backup.OplogSpanMin)
45+
}
46+
if len(expectedBackupOptions.Priority) != len(config.Backup.Priority) {
47+
t.Errorf("expected %d, got %d", len(expectedBackupOptions.Priority), len(config.Backup.Priority))
48+
}
49+
if expectedBackupOptions.Priority["localhost:28019"] != config.Backup.Priority["localhost:28019"] {
50+
t.Errorf("expected %f, got %f", expectedBackupOptions.Priority["localhost:28019"], config.Backup.Priority["localhost:28019"])
51+
}
52+
if expectedBackupOptions.Timeouts.Starting != config.Backup.Timeouts.Starting {
53+
t.Errorf("expected %d, got %d", expectedBackupOptions.Timeouts.Starting, config.Backup.Timeouts.Starting)
54+
}
55+
56+
expectedRestoreOptions := cr.Spec.Backup.Configuration.RestoreOptions
57+
if expectedRestoreOptions.BatchSize != config.Restore.BatchSize {
58+
t.Errorf("expected %d, got %d", expectedRestoreOptions.BatchSize, config.Restore.BatchSize)
59+
}
60+
if expectedRestoreOptions.NumInsertionWorkers != config.Restore.NumInsertionWorkers {
61+
t.Errorf("expected %d, got %d", expectedRestoreOptions.NumInsertionWorkers, config.Restore.NumInsertionWorkers)
62+
}
63+
if expectedRestoreOptions.NumDownloadWorkers != config.Restore.NumDownloadWorkers {
64+
t.Errorf("expected %d, got %d", expectedRestoreOptions.NumDownloadWorkers, config.Restore.NumDownloadWorkers)
65+
}
66+
if expectedRestoreOptions.MaxDownloadBufferMb != config.Restore.MaxDownloadBufferMb {
67+
t.Errorf("expected %d, got %d", expectedRestoreOptions.MaxDownloadBufferMb, config.Restore.MaxDownloadBufferMb)
68+
}
69+
if expectedRestoreOptions.DownloadChunkMb != config.Restore.DownloadChunkMb {
70+
t.Errorf("expected %d, got %d", expectedRestoreOptions.DownloadChunkMb, config.Restore.DownloadChunkMb)
71+
}
72+
if expectedRestoreOptions.MongodLocation != config.Restore.MongodLocation {
73+
t.Errorf("expected %s, got %s", expectedRestoreOptions.MongodLocation, config.Restore.MongodLocation)
74+
}
75+
if len(expectedRestoreOptions.MongodLocationMap) != len(config.Restore.MongodLocationMap) {
76+
t.Errorf("expected %d, got %d", len(expectedRestoreOptions.MongodLocationMap), len(config.Restore.MongodLocationMap))
77+
}
78+
if expectedRestoreOptions.MongodLocationMap["node01:1017"] != config.Restore.MongodLocationMap["node01:1017"] {
79+
t.Errorf("expected %s, got %s", expectedRestoreOptions.MongodLocationMap["node01:1017"], config.Restore.MongodLocationMap["node01:1017"])
80+
}
81+
}
82+
83+
func buildFakeClient(t *testing.T) client.WithWatch {
84+
t.Helper()
85+
86+
s := scheme.Scheme
87+
if err := scheme.AddToScheme(s); err != nil {
88+
t.Fatal(err, "failed to add client-go scheme")
89+
}
90+
91+
secret := &corev1.Secret{
92+
TypeMeta: metav1.TypeMeta{
93+
APIVersion: "/v1",
94+
Kind: "Secret",
95+
},
96+
ObjectMeta: metav1.ObjectMeta{
97+
Name: "test-secret",
98+
Namespace: "test-namespace",
99+
},
100+
}
101+
return fake.NewClientBuilder().WithScheme(s).WithObjects(secret).Build()
102+
}
103+
104+
func expectedCR(t *testing.T) *api.PerconaServerMongoDB {
105+
t.Helper()
106+
107+
data, err := os.ReadFile("testdata/cr-with-pbm-config.yaml")
108+
if err != nil {
109+
t.Fatal(err)
110+
}
111+
cr := new(api.PerconaServerMongoDB)
112+
113+
if err := yaml.Unmarshal(data, cr); err != nil {
114+
t.Fatal(err)
115+
}
116+
117+
return cr
118+
}

0 commit comments

Comments
 (0)