Skip to content

Commit 388aa92

Browse files
pooknullhors
pooknull
and
hors
authored
K8SPSMDB-1072: add retention section to backup tasks (#1928)
* K8SPSMDB-1072: add `retention` section to backup tasks https://perconadev.atlassian.net/browse/K8SPSMDB-1072 * add retention to tests * fix unit test * remove unused comment * fix test * small improvement * fix copypaste * fix lint * fix bundle * fix tests * fix test --------- Co-authored-by: Viacheslav Sarzhan <slava.sarzhan@percona.com>
1 parent 1f3b686 commit 388aa92

File tree

14 files changed

+235
-58
lines changed

14 files changed

+235
-58
lines changed

‎config/crd/bases/psmdb.percona.com_perconaservermongodbs.yaml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -389,6 +389,22 @@ spec:
389389
type: integer
390390
name:
391391
type: string
392+
retention:
393+
properties:
394+
count:
395+
minimum: 0
396+
type: integer
397+
deleteFromStorage:
398+
default: true
399+
type: boolean
400+
type:
401+
enum:
402+
- count
403+
type: string
404+
required:
405+
- deleteFromStorage
406+
- type
407+
type: object
392408
schedule:
393409
type: string
394410
storageName:

‎deploy/bundle.yaml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1108,6 +1108,22 @@ spec:
11081108
type: integer
11091109
name:
11101110
type: string
1111+
retention:
1112+
properties:
1113+
count:
1114+
minimum: 0
1115+
type: integer
1116+
deleteFromStorage:
1117+
default: true
1118+
type: boolean
1119+
type:
1120+
enum:
1121+
- count
1122+
type: string
1123+
required:
1124+
- deleteFromStorage
1125+
- type
1126+
type: object
11111127
schedule:
11121128
type: string
11131129
storageName:

‎deploy/cr.yaml

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -770,21 +770,30 @@ spec:
770770
# - name: daily-s3-us-west
771771
# enabled: true
772772
# schedule: "0 0 * * *"
773-
# keep: 3
773+
# retention:
774+
# count: 3
775+
# type: count
776+
# deleteFromStorage: true
774777
# storageName: s3-us-west
775778
# compressionType: gzip
776779
# compressionLevel: 6
777780
# - name: weekly-s3-us-west
778781
# enabled: false
779782
# schedule: "0 0 * * 0"
780-
# keep: 5
783+
# retention:
784+
# count: 3
785+
# type: count
786+
# deleteFromStorage: true
781787
# storageName: s3-us-west
782788
# compressionType: gzip
783789
# compressionLevel: 6
784790
# - name: weekly-s3-us-west-physical
785791
# enabled: false
786792
# schedule: "0 5 * * 0"
787-
# keep: 5
793+
# retention:
794+
# count: 3
795+
# type: count
796+
# deleteFromStorage: true
788797
# type: physical
789798
# storageName: s3-us-west
790799
# compressionType: gzip
@@ -799,7 +808,10 @@ spec:
799808
# - name: weekly-s3-us-west-incremental-base
800809
# enabled: false
801810
# schedule: "0 5 * * 0"
802-
# keep: 5
811+
# retention:
812+
# count: 3
813+
# type: count
814+
# deleteFromStorage: true
803815
# type: incremental-base
804816
# storageName: s3-us-west
805817
# compressionType: gzip

‎deploy/crd.yaml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1108,6 +1108,22 @@ spec:
11081108
type: integer
11091109
name:
11101110
type: string
1111+
retention:
1112+
properties:
1113+
count:
1114+
minimum: 0
1115+
type: integer
1116+
deleteFromStorage:
1117+
default: true
1118+
type: boolean
1119+
type:
1120+
enum:
1121+
- count
1122+
type: string
1123+
required:
1124+
- deleteFromStorage
1125+
- type
1126+
type: object
11111127
schedule:
11121128
type: string
11131129
storageName:

‎deploy/cw-bundle.yaml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1108,6 +1108,22 @@ spec:
11081108
type: integer
11091109
name:
11101110
type: string
1111+
retention:
1112+
properties:
1113+
count:
1114+
minimum: 0
1115+
type: integer
1116+
deleteFromStorage:
1117+
default: true
1118+
type: boolean
1119+
type:
1120+
enum:
1121+
- count
1122+
type: string
1123+
required:
1124+
- deleteFromStorage
1125+
- type
1126+
type: object
11111127
schedule:
11121128
type: string
11131129
storageName:

‎e2e-tests/scheduled-backup/conf/some-name-rs0-2.yml

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -49,31 +49,46 @@ spec:
4949
tasks:
5050
- name: every-min-minio
5151
enabled: true
52-
keep: 1
52+
retention:
53+
type: count
54+
count: 1
55+
deleteFromStorage: true
5356
schedule: "* * * * *"
5457
compressionType: gzip
5558
storageName: minio
5659
- name: every-min-aws-s3
5760
enabled: true
58-
keep: 1
61+
retention:
62+
type: count
63+
count: 1
64+
deleteFromStorage: true
5965
schedule: "* * * * *"
6066
compressionType: gzip
6167
storageName: aws-s3
6268
- name: every-min-gcp-cs
6369
enabled: true
64-
keep: 1
70+
retention:
71+
type: count
72+
count: 1
73+
deleteFromStorage: true
6574
schedule: "* * * * *"
6675
compressionType: gzip
6776
storageName: gcp-cs
6877
- name: every-min-azure-blob
6978
enabled: true
70-
keep: 1
79+
retention:
80+
type: count
81+
count: 1
82+
deleteFromStorage: true
7183
schedule: "* * * * *"
7284
compressionType: gzip
7385
storageName: azure-blob
7486
- name: weekly
7587
enabled: true
76-
keep: 1
88+
retention:
89+
type: count
90+
count: 1
91+
deleteFromStorage: true
7792
schedule: "0 0 * * 0"
7893
compressionType: gzip
7994
storageName: aws-s3

‎e2e-tests/scheduled-backup/conf/some-name-rs0-3.yml

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,13 +49,20 @@ spec:
4949
tasks:
5050
- name: every-min-minio
5151
enabled: true
52-
keep: 1
52+
retention:
53+
type: count
54+
count: 1
55+
deleteFromStorage: true
5356
type: physical
5457
schedule: "* * * * *"
5558
compressionType: gzip
5659
storageName: minio
5760
- name: weekly
5861
enabled: true
62+
retention:
63+
type: count
64+
count: 1
65+
deleteFromStorage: true
5966
schedule: "0 0 * * 0"
6067
compressionType: gzip
6168
storageName: aws-s3

‎e2e-tests/version-service/conf/crd.yaml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1108,6 +1108,22 @@ spec:
11081108
type: integer
11091109
name:
11101110
type: string
1111+
retention:
1112+
properties:
1113+
count:
1114+
minimum: 0
1115+
type: integer
1116+
deleteFromStorage:
1117+
default: true
1118+
type: boolean
1119+
type:
1120+
enum:
1121+
- count
1122+
type: string
1123+
required:
1124+
- deleteFromStorage
1125+
- type
1126+
type: object
11111127
schedule:
11121128
type: string
11131129
storageName:

‎pkg/apis/psmdb/v1/psmdb_defaults.go

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -518,10 +518,20 @@ func (cr *PerconaServerMongoDB) CheckNSetDefaults(ctx context.Context, platform
518518
}
519519

520520
if cr.Spec.Backup.Enabled {
521-
for _, bkpTask := range cr.Spec.Backup.Tasks {
521+
for i := range cr.Spec.Backup.Tasks {
522+
bkpTask := &cr.Spec.Backup.Tasks[i]
523+
524+
if bkpTask.Name == "" {
525+
return errors.Errorf("backup task %d should have a name", i)
526+
}
522527
if string(bkpTask.CompressionType) == "" {
523528
bkpTask.CompressionType = compress.CompressionTypeGZIP
524529
}
530+
531+
if cr.CompareVersion("1.21.0") >= 0 && bkpTask.Keep > 0 && bkpTask.Retention != nil && bkpTask.Retention.Count == 0 {
532+
log.Info(".spec.backup.tasks[].keep will be deprecated in the future. Consider using .spec.backup.tasks[].retention.count instead", "task", bkpTask.Name)
533+
continue
534+
}
525535
}
526536
if len(cr.Spec.Backup.ServiceAccountName) == 0 && cr.CompareVersion("1.15.0") < 0 {
527537
cr.Spec.Backup.ServiceAccountName = "percona-server-mongodb-operator"

‎pkg/apis/psmdb/v1/psmdb_types.go

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -944,9 +944,12 @@ type MongodSpecInMemory struct {
944944
}
945945

946946
type BackupTaskSpec struct {
947-
Name string `json:"name"`
948-
Enabled bool `json:"enabled"`
949-
Keep int `json:"keep,omitempty"`
947+
Name string `json:"name"`
948+
Enabled bool `json:"enabled"`
949+
// Deprecated: Use Retention instead. This field will be removed in the future
950+
Keep int `json:"keep,omitempty"`
951+
// +optional
952+
Retention *BackupTaskSpecRetention `json:"retention,omitempty"`
950953
Schedule string `json:"schedule,omitempty"`
951954
StorageName string `json:"storageName,omitempty"`
952955
CompressionType compress.CompressionType `json:"compressionType,omitempty"`
@@ -956,6 +959,36 @@ type BackupTaskSpec struct {
956959
Type defs.BackupType `json:"type,omitempty"`
957960
}
958961

962+
func (task *BackupTaskSpec) GetRetention(cr *PerconaServerMongoDB) BackupTaskSpecRetention {
963+
if task.Retention != nil && cr.CompareVersion("1.21.0") >= 0 {
964+
return *task.Retention
965+
}
966+
return BackupTaskSpecRetention{
967+
Type: BackupTaskSpecRetentionTypeCount,
968+
Count: task.Keep,
969+
DeleteFromStorage: true,
970+
}
971+
}
972+
973+
type BackupTaskSpecRetentionType string
974+
975+
const (
976+
BackupTaskSpecRetentionTypeCount = "count"
977+
)
978+
979+
type BackupTaskSpecRetention struct {
980+
// +kubebuilder:validation:Minimum=0
981+
Count int `json:"count,omitempty"`
982+
983+
// +kubebuilder:validation:Required
984+
// +kubebuilder:validation:Enum={count}
985+
Type string `json:"type,omitempty"`
986+
987+
// +kubebuilder:validation:Required
988+
// +kubebuilder:default=true
989+
DeleteFromStorage bool `json:"deleteFromStorage,omitempty"`
990+
}
991+
959992
func (task *BackupTaskSpec) JobName(cr *PerconaServerMongoDB) string {
960993
return fmt.Sprintf("%s-backup-%s-%s", cr.Name, task.Name, cr.Namespace)
961994
}

0 commit comments

Comments
 (0)