diff --git a/apis/apps/v1alpha1/backuppolicytemplate_types.go b/apis/apps/v1alpha1/backuppolicytemplate_types.go index cf690df1748..e8759e6a3a3 100644 --- a/apis/apps/v1alpha1/backuppolicytemplate_types.go +++ b/apis/apps/v1alpha1/backuppolicytemplate_types.go @@ -55,11 +55,9 @@ type BackupPolicy struct { // +kubebuilder:validation:Pattern:=`^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$` ComponentDefRef string `json:"componentDefRef"` - // ttl is a time string ending with the 'd'|'D'|'h'|'H' character to describe how long - // the Backup should be retained. if not set, will be retained forever. - // +kubebuilder:validation:Pattern:=`^\d+[d|D|h|H]$` + // retention describe how long the Backup should be retained. if not set, will be retained forever. // +optional - TTL *string `json:"ttl,omitempty"` + Retention *RetentionSpec `json:"retention,omitempty"` // schedule policy for backup. // +optional @@ -69,30 +67,35 @@ type BackupPolicy struct { // +optional Snapshot *SnapshotPolicy `json:"snapshot,omitempty"` - // the policy for full backup. + // the policy for datafile backup. // +optional - Full *CommonBackupPolicy `json:"full,omitempty"` + Datafile *CommonBackupPolicy `json:"datafile,omitempty"` + + // the policy for logfile backup. + // +optional + Logfile *CommonBackupPolicy `json:"logfile,omitempty"` +} - // the policy for incremental backup. +type RetentionSpec struct { + // ttl is a time string ending with the 'd'|'D'|'h'|'H' character to describe how long + // the Backup should be retained. if not set, will be retained forever. + // +kubebuilder:validation:Pattern:=`^\d+[d|D|h|H]$` // +optional - Incremental *CommonBackupPolicy `json:"incremental,omitempty"` + TTL *string `json:"ttl,omitempty"` } type Schedule struct { - // schedule policy for base backup. + // schedule policy for snapshot backup. // +optional - BaseBackup *BaseBackupSchedulePolicy `json:"baseBackup,omitempty"` + Snapshot *SchedulePolicy `json:"snapshot,omitempty"` - // schedule policy for incremental backup. + // schedule policy for datafile backup. // +optional - Incremental *SchedulePolicy `json:"incremental,omitempty"` -} + Datafile *SchedulePolicy `json:"datafile,omitempty"` -type BaseBackupSchedulePolicy struct { - SchedulePolicy `json:",inline"` - // the type of base backup, only support full and snapshot. - // +kubebuilder:validation:Required - Type BaseBackupType `json:"type"` + // schedule policy for logfile backup. + // +optional + Logfile *SchedulePolicy `json:"logfile,omitempty"` } type SchedulePolicy struct { diff --git a/apis/dataprotection/v1alpha1/backup_types.go b/apis/dataprotection/v1alpha1/backup_types.go index 4f36ee44204..8b77fb99e0d 100644 --- a/apis/dataprotection/v1alpha1/backup_types.go +++ b/apis/dataprotection/v1alpha1/backup_types.go @@ -33,8 +33,8 @@ type BackupSpec struct { // +kubebuilder:validation:Pattern:=`^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$` BackupPolicyName string `json:"backupPolicyName"` - // Backup Type. full or incremental or snapshot. if unset, default is full. - // +kubebuilder:default=full + // Backup Type. datafile or logfile or snapshot. if unset, default is datafile. + // +kubebuilder:default=datafile BackupType BackupType `json:"backupType"` // if backupType is incremental, parentBackupName is required. @@ -195,13 +195,13 @@ func (r *BackupSpec) Validate(backupPolicy *BackupPolicy) error { if backupPolicy.Spec.Snapshot == nil { return fmt.Errorf(notSupportedMessage, r.BackupPolicyName, BackupTypeSnapshot) } - case BackupTypeFull: - if backupPolicy.Spec.Full == nil { - return fmt.Errorf(notSupportedMessage, r.BackupPolicyName, BackupTypeFull) + case BackupTypeDataFile: + if backupPolicy.Spec.Datafile == nil { + return fmt.Errorf(notSupportedMessage, r.BackupPolicyName, BackupTypeDataFile) } - case BackupTypeIncremental: - if backupPolicy.Spec.Incremental == nil { - return fmt.Errorf(notSupportedMessage, r.BackupPolicyName, BackupTypeIncremental) + case BackupTypeLogFile: + if backupPolicy.Spec.Logfile == nil { + return fmt.Errorf(notSupportedMessage, r.BackupPolicyName, BackupTypeLogFile) } } return nil @@ -217,9 +217,9 @@ func GetRecoverableTimeRange(backups []Backup) []BackupLogStatus { b.Status.Manifests.BackupLog.StopTime == nil { continue } - if b.Spec.BackupType == BackupTypeIncremental { + if b.Spec.BackupType == BackupTypeLogFile { incrementalBackup = &b - } else if b.Spec.BackupType != BackupTypeIncremental && b.Status.Phase == BackupCompleted { + } else if b.Spec.BackupType != BackupTypeLogFile && b.Status.Phase == BackupCompleted { baseBackups = append(baseBackups, b) } } diff --git a/apis/dataprotection/v1alpha1/backuppolicy_types.go b/apis/dataprotection/v1alpha1/backuppolicy_types.go index 773197781d2..8dc80baada4 100644 --- a/apis/dataprotection/v1alpha1/backuppolicy_types.go +++ b/apis/dataprotection/v1alpha1/backuppolicy_types.go @@ -30,11 +30,9 @@ import ( // BackupPolicySpec defines the desired state of BackupPolicy type BackupPolicySpec struct { - // ttl is a time string ending with the 'd'|'D'|'h'|'H' character to describe how long - // the Backup should be retained. if not set, will be retained forever. - // +kubebuilder:validation:Pattern:=`^\d+[d|D|h|H]$` + // retention describe how long the Backup should be retained. if not set, will be retained forever. // +optional - TTL *string `json:"ttl,omitempty"` + Retention *RetentionSpec `json:"retention,omitempty"` // schedule policy for backup. // +optional @@ -44,30 +42,35 @@ type BackupPolicySpec struct { // +optional Snapshot *SnapshotPolicy `json:"snapshot,omitempty"` - // the policy for full backup. + // the policy for datafile backup. // +optional - Full *CommonBackupPolicy `json:"full,omitempty"` + Datafile *CommonBackupPolicy `json:"datafile,omitempty"` + + // the policy for logfile backup. + // +optional + Logfile *CommonBackupPolicy `json:"logfile,omitempty"` +} - // the policy for incremental backup. +type RetentionSpec struct { + // ttl is a time string ending with the 'd'|'D'|'h'|'H' character to describe how long + // the Backup should be retained. if not set, will be retained forever. + // +kubebuilder:validation:Pattern:=`^\d+[d|D|h|H]$` // +optional - Incremental *CommonBackupPolicy `json:"incremental,omitempty"` + TTL *string `json:"ttl,omitempty"` } type Schedule struct { - // schedule policy for base backup. + // schedule policy for snapshot backup. // +optional - BaseBackup *BaseBackupSchedulePolicy `json:"baseBackup,omitempty"` + Snapshot *SchedulePolicy `json:"snapshot,omitempty"` - // schedule policy for incremental backup. + // schedule policy for datafile backup. // +optional - Incremental *SchedulePolicy `json:"incremental,omitempty"` -} + Datafile *SchedulePolicy `json:"datafile,omitempty"` -type BaseBackupSchedulePolicy struct { - SchedulePolicy `json:",inline"` - // the type of base backup, only support full and snapshot. - // +kubebuilder:validation:Required - Type BaseBackupType `json:"type"` + // schedule policy for logfile backup. + // +optional + Logfile *SchedulePolicy `json:"logfile,omitempty"` } type SchedulePolicy struct { @@ -301,10 +304,10 @@ func init() { func (r *BackupPolicySpec) GetCommonPolicy(backupType BackupType) *CommonBackupPolicy { switch backupType { - case BackupTypeFull: - return r.Full - case BackupTypeIncremental: - return r.Incremental + case BackupTypeDataFile: + return r.Datafile + case BackupTypeLogFile: + return r.Logfile } return nil } diff --git a/apis/dataprotection/v1alpha1/types.go b/apis/dataprotection/v1alpha1/types.go index 78b50a70003..f516ced249f 100644 --- a/apis/dataprotection/v1alpha1/types.go +++ b/apis/dataprotection/v1alpha1/types.go @@ -31,15 +31,15 @@ const ( BackupFailed BackupPhase = "Failed" ) -// BackupType the backup type, marked backup set is full or incremental or snapshot. +// BackupType the backup type, marked backup set is datafile or logfile or snapshot. // +enum -// +kubebuilder:validation:Enum={full,incremental,snapshot} +// +kubebuilder:validation:Enum={datafile,logfile,snapshot} type BackupType string const ( - BackupTypeFull BackupType = "full" - BackupTypeIncremental BackupType = "incremental" - BackupTypeSnapshot BackupType = "snapshot" + BackupTypeDataFile BackupType = "datafile" + BackupTypeLogFile BackupType = "logfile" + BackupTypeSnapshot BackupType = "snapshot" ) // BaseBackupType the base backup type. @@ -47,11 +47,6 @@ const ( // +kubebuilder:validation:Enum={full,snapshot} type BaseBackupType string -const ( - BaseBackupTypeFull BaseBackupType = "full" - BaseBackupTypeSnapshot BaseBackupType = "snapshot" -) - // CreatePVCPolicy the policy how to create the PersistentVolumeClaim for backup. // +enum // +kubebuilder:validation:Enum={IfNotPresent,Never} diff --git a/config/crd/bases/apps.kubeblocks.io_backuppolicytemplates.yaml b/config/crd/bases/apps.kubeblocks.io_backuppolicytemplates.yaml index abd20cd4340..51ba57b1c3a 100644 --- a/config/crd/bases/apps.kubeblocks.io_backuppolicytemplates.yaml +++ b/config/crd/bases/apps.kubeblocks.io_backuppolicytemplates.yaml @@ -59,8 +59,8 @@ spec: maxLength: 63 pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ type: string - full: - description: the policy for full backup. + datafile: + description: the policy for datafile backup. properties: backupStatusUpdates: description: define how to update metadata for backup status. @@ -147,8 +147,8 @@ spec: type: string type: object type: object - incremental: - description: the policy for incremental backup. + logfile: + description: the policy for logfile backup. properties: backupStatusUpdates: description: define how to update metadata for backup status. @@ -235,11 +235,22 @@ spec: type: string type: object type: object + retention: + description: retention describe how long the Backup should be + retained. if not set, will be retained forever. + properties: + ttl: + description: ttl is a time string ending with the 'd'|'D'|'h'|'H' + character to describe how long the Backup should be retained. + if not set, will be retained forever. + pattern: ^\d+[d|D|h|H]$ + type: string + type: object schedule: description: schedule policy for backup. properties: - baseBackup: - description: schedule policy for base backup. + datafile: + description: schedule policy for datafile backup. properties: cronExpression: description: the cron expression for schedule, the timezone @@ -248,20 +259,26 @@ spec: enable: description: enable or disable the schedule. type: boolean - type: - description: the type of base backup, only support full - and snapshot. - enum: - - full - - snapshot + required: + - cronExpression + - enable + type: object + logfile: + description: schedule policy for logfile backup. + properties: + cronExpression: + description: the cron expression for schedule, the timezone + is in UTC. see https://en.wikipedia.org/wiki/Cron. type: string + enable: + description: enable or disable the schedule. + type: boolean required: - cronExpression - enable - - type type: object - incremental: - description: schedule policy for incremental backup. + snapshot: + description: schedule policy for snapshot backup. properties: cronExpression: description: the cron expression for schedule, the timezone @@ -378,12 +395,6 @@ spec: type: string type: object type: object - ttl: - description: ttl is a time string ending with the 'd'|'D'|'h'|'H' - character to describe how long the Backup should be retained. - if not set, will be retained forever. - pattern: ^\d+[d|D|h|H]$ - type: string required: - componentDefRef type: object diff --git a/config/crd/bases/dataprotection.kubeblocks.io_backuppolicies.yaml b/config/crd/bases/dataprotection.kubeblocks.io_backuppolicies.yaml index 46a131acba0..b150563971e 100644 --- a/config/crd/bases/dataprotection.kubeblocks.io_backuppolicies.yaml +++ b/config/crd/bases/dataprotection.kubeblocks.io_backuppolicies.yaml @@ -50,8 +50,8 @@ spec: spec: description: BackupPolicySpec defines the desired state of BackupPolicy properties: - full: - description: the policy for full backup. + datafile: + description: the policy for datafile backup. properties: backupStatusUpdates: description: define how to update metadata for backup status. @@ -232,8 +232,8 @@ spec: - persistentVolumeClaim - target type: object - incremental: - description: the policy for incremental backup. + logfile: + description: the policy for logfile backup. properties: backupStatusUpdates: description: define how to update metadata for backup status. @@ -414,11 +414,22 @@ spec: - persistentVolumeClaim - target type: object + retention: + description: retention describe how long the Backup should be retained. + if not set, will be retained forever. + properties: + ttl: + description: ttl is a time string ending with the 'd'|'D'|'h'|'H' + character to describe how long the Backup should be retained. + if not set, will be retained forever. + pattern: ^\d+[d|D|h|H]$ + type: string + type: object schedule: description: schedule policy for backup. properties: - baseBackup: - description: schedule policy for base backup. + datafile: + description: schedule policy for datafile backup. properties: cronExpression: description: the cron expression for schedule, the timezone @@ -427,20 +438,26 @@ spec: enable: description: enable or disable the schedule. type: boolean - type: - description: the type of base backup, only support full and - snapshot. - enum: - - full - - snapshot + required: + - cronExpression + - enable + type: object + logfile: + description: schedule policy for logfile backup. + properties: + cronExpression: + description: the cron expression for schedule, the timezone + is in UTC. see https://en.wikipedia.org/wiki/Cron. type: string + enable: + description: enable or disable the schedule. + type: boolean required: - cronExpression - enable - - type type: object - incremental: - description: schedule policy for incremental backup. + snapshot: + description: schedule policy for snapshot backup. properties: cronExpression: description: the cron expression for schedule, the timezone @@ -596,12 +613,6 @@ spec: required: - target type: object - ttl: - description: ttl is a time string ending with the 'd'|'D'|'h'|'H' - character to describe how long the Backup should be retained. if - not set, will be retained forever. - pattern: ^\d+[d|D|h|H]$ - type: string type: object status: description: BackupPolicyStatus defines the observed state of BackupPolicy diff --git a/config/crd/bases/dataprotection.kubeblocks.io_backups.yaml b/config/crd/bases/dataprotection.kubeblocks.io_backups.yaml index 19138c606f3..c1c96731aa7 100644 --- a/config/crd/bases/dataprotection.kubeblocks.io_backups.yaml +++ b/config/crd/bases/dataprotection.kubeblocks.io_backups.yaml @@ -61,12 +61,12 @@ spec: pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ type: string backupType: - default: full - description: Backup Type. full or incremental or snapshot. if unset, - default is full. + default: datafile + description: Backup Type. datafile or logfile or snapshot. if unset, + default is datafile. enum: - - full - - incremental + - datafile + - logfile - snapshot type: string parentBackupName: diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index da18e9ed991..8c6be115d84 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -1460,7 +1460,7 @@ var _ = Describe("Cluster Controller", func() { By("creating backup") backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). SetBackupPolicyName(backupPolicyName). - SetBackupType(dataprotectionv1alpha1.BackupTypeFull). + SetBackupType(dataprotectionv1alpha1.BackupTypeDataFile). Create(&testCtx).GetObject() By("waiting for backup failed, because no backup policy exists") diff --git a/controllers/dataprotection/backup_controller.go b/controllers/dataprotection/backup_controller.go index 00749ae7c3c..1041edc405b 100644 --- a/controllers/dataprotection/backup_controller.go +++ b/controllers/dataprotection/backup_controller.go @@ -223,9 +223,9 @@ func (r *BackupReconciler) doNewPhaseAction( // update Phase to InProgress backup.Status.Phase = dataprotectionv1alpha1.BackupInProgress backup.Status.StartTimestamp = &metav1.Time{Time: r.clock.Now().UTC()} - if backupPolicy.Spec.TTL != nil { + if backupPolicy.Spec.Retention != nil && backupPolicy.Spec.Retention.TTL != nil { backup.Status.Expiration = &metav1.Time{ - Time: backup.Status.StartTimestamp.Add(dataprotectionv1alpha1.ToDuration(backupPolicy.Spec.TTL)), + Time: backup.Status.StartTimestamp.Add(dataprotectionv1alpha1.ToDuration(backupPolicy.Spec.Retention.TTL)), } } @@ -448,7 +448,7 @@ func (r *BackupReconciler) doInProgressPhaseAction( backup.Status.Phase = dataprotectionv1alpha1.BackupFailed backup.Status.FailureReason = job.Status.Conditions[0].Reason } - if backup.Spec.BackupType == dataprotectionv1alpha1.BackupTypeIncremental { + if backup.Spec.BackupType == dataprotectionv1alpha1.BackupTypeLogFile { if backup.Status.Manifests != nil && backup.Status.Manifests.BackupLog != nil && backup.Status.Manifests.BackupLog.StartTime == nil { diff --git a/controllers/dataprotection/backup_controller_test.go b/controllers/dataprotection/backup_controller_test.go index cbc0ff153d1..561eaf25735 100644 --- a/controllers/dataprotection/backup_controller_test.go +++ b/controllers/dataprotection/backup_controller_test.go @@ -158,7 +158,7 @@ var _ = Describe("Backup Controller test", func() { By("By creating a backup from backupPolicy: " + backupPolicyName) backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). SetBackupPolicyName(backupPolicyName). - SetBackupType(dataprotectionv1alpha1.BackupTypeFull). + SetBackupType(dataprotectionv1alpha1.BackupTypeDataFile). Create(&testCtx).GetObject() backupKey = client.ObjectKeyFromObject(backup) }) @@ -200,7 +200,7 @@ var _ = Describe("Backup Controller test", func() { By("creating a backup from backupPolicy: " + backupPolicyName) backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). SetBackupPolicyName(backupPolicyName). - SetBackupType(dataprotectionv1alpha1.BackupTypeFull). + SetBackupType(dataprotectionv1alpha1.BackupTypeDataFile). Create(&testCtx).GetObject() backupKey = client.ObjectKeyFromObject(backup) @@ -408,7 +408,7 @@ var _ = Describe("Backup Controller test", func() { By("By creating a backup from backupPolicy: " + backupPolicyName) backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). SetBackupPolicyName(backupPolicyName). - SetBackupType(dataprotectionv1alpha1.BackupTypeFull). + SetBackupType(dataprotectionv1alpha1.BackupTypeDataFile). Create(&testCtx).GetObject() backupKey = client.ObjectKeyFromObject(backup) } @@ -459,7 +459,7 @@ var _ = Describe("Backup Controller test", func() { By("set persistentVolumeConfigmap") configMapName := "pv-template-configmap" Expect(testapps.ChangeObj(&testCtx, backupPolicy, func(tmpObj *dataprotectionv1alpha1.BackupPolicy) { - tmpObj.Spec.Full.PersistentVolumeClaim.PersistentVolumeConfigMap = &dataprotectionv1alpha1.PersistentVolumeConfigMap{ + tmpObj.Spec.Datafile.PersistentVolumeClaim.PersistentVolumeConfigMap = &dataprotectionv1alpha1.PersistentVolumeConfigMap{ Name: configMapName, Namespace: testCtx.DefaultNamespace, } @@ -537,7 +537,7 @@ var _ = Describe("Backup Controller test", func() { By("By creating a backup from backupPolicy: " + backupPolicyName) backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). SetBackupPolicyName(backupPolicyName). - SetBackupType(dataprotectionv1alpha1.BackupTypeFull). + SetBackupType(dataprotectionv1alpha1.BackupTypeDataFile). Create(&testCtx).GetObject() backupKey = client.ObjectKeyFromObject(backup) }) diff --git a/controllers/dataprotection/backuppolicy_controller.go b/controllers/dataprotection/backuppolicy_controller.go index fe73e3b07cc..63ccfd57db2 100644 --- a/controllers/dataprotection/backuppolicy_controller.go +++ b/controllers/dataprotection/backuppolicy_controller.go @@ -131,8 +131,8 @@ func (r *BackupPolicyReconciler) deleteExternalResources(reqCtx intctrlutil.Requ // delete cronjob resource cronjob := &batchv1.CronJob{} - for _, v := range []dataprotectionv1alpha1.BackupType{dataprotectionv1alpha1.BackupTypeFull, - dataprotectionv1alpha1.BackupTypeIncremental, dataprotectionv1alpha1.BackupTypeSnapshot} { + for _, v := range []dataprotectionv1alpha1.BackupType{dataprotectionv1alpha1.BackupTypeDataFile, + dataprotectionv1alpha1.BackupTypeLogFile, dataprotectionv1alpha1.BackupTypeSnapshot} { key := types.NamespacedName{ Namespace: viper.GetString(constant.CfgKeyCtrlrMgrNS), Name: r.getCronJobName(backupPolicy.Name, backupPolicy.Namespace, v), @@ -279,8 +279,8 @@ func (r *BackupPolicyReconciler) buildCronJob( return nil, err } var ttl metav1.Duration - if backupPolicy.Spec.TTL != nil { - ttl = metav1.Duration{Duration: dataprotectionv1alpha1.ToDuration(backupPolicy.Spec.TTL)} + if backupPolicy.Spec.Retention != nil && backupPolicy.Spec.Retention.TTL != nil { + ttl = metav1.Duration{Duration: dataprotectionv1alpha1.ToDuration(backupPolicy.Spec.Retention.TTL)} } cueValue := intctrlutil.NewCUEBuilder(*cueTpl) options := backupPolicyOptions{ @@ -303,8 +303,8 @@ func (r *BackupPolicyReconciler) buildCronJob( return nil, err } cuePath := "cronjob" - if backType == dataprotectionv1alpha1.BackupTypeIncremental { - cuePath = "cronjob_incremental" + if backType == dataprotectionv1alpha1.BackupTypeLogFile { + cuePath = "cronjob_logfile" } cronjobByte, err := cueValue.Lookup(cuePath) if err != nil { @@ -399,47 +399,47 @@ func (r *BackupPolicyReconciler) handleSnapshotPolicy( return nil } var cronExpression string - schedule := backupPolicy.Spec.Schedule.BaseBackup - if schedule != nil && schedule.Enable && schedule.Type == dataprotectionv1alpha1.BaseBackupTypeSnapshot { + schedule := backupPolicy.Spec.Schedule.Snapshot + if schedule != nil && schedule.Enable { cronExpression = schedule.CronExpression } return r.handlePolicy(reqCtx, backupPolicy, backupPolicy.Spec.Snapshot.BasePolicy, cronExpression, dataprotectionv1alpha1.BackupTypeSnapshot) } -// handleFullPolicy handles full policy. +// handleFullPolicy handles datafile policy. func (r *BackupPolicyReconciler) handleFullPolicy( reqCtx intctrlutil.RequestCtx, backupPolicy *dataprotectionv1alpha1.BackupPolicy) error { - if backupPolicy.Spec.Full == nil { + if backupPolicy.Spec.Datafile == nil { // TODO delete cronjob if exists return nil } var cronExpression string - schedule := backupPolicy.Spec.Schedule.BaseBackup - if schedule != nil && schedule.Enable && schedule.Type == dataprotectionv1alpha1.BaseBackupTypeFull { + schedule := backupPolicy.Spec.Schedule.Datafile + if schedule != nil && schedule.Enable { cronExpression = schedule.CronExpression } - r.setGlobalPersistentVolumeClaim(backupPolicy.Spec.Full) - return r.handlePolicy(reqCtx, backupPolicy, backupPolicy.Spec.Full.BasePolicy, - cronExpression, dataprotectionv1alpha1.BackupTypeFull) + r.setGlobalPersistentVolumeClaim(backupPolicy.Spec.Datafile) + return r.handlePolicy(reqCtx, backupPolicy, backupPolicy.Spec.Datafile.BasePolicy, + cronExpression, dataprotectionv1alpha1.BackupTypeDataFile) } // handleIncrementalPolicy handles incremental policy. func (r *BackupPolicyReconciler) handleIncrementalPolicy( reqCtx intctrlutil.RequestCtx, backupPolicy *dataprotectionv1alpha1.BackupPolicy) error { - if backupPolicy.Spec.Incremental == nil { + if backupPolicy.Spec.Logfile == nil { return nil } var cronExpression string - schedule := backupPolicy.Spec.Schedule.Incremental + schedule := backupPolicy.Spec.Schedule.Logfile if schedule != nil && schedule.Enable { cronExpression = schedule.CronExpression } - r.setGlobalPersistentVolumeClaim(backupPolicy.Spec.Incremental) - return r.handlePolicy(reqCtx, backupPolicy, backupPolicy.Spec.Incremental.BasePolicy, - cronExpression, dataprotectionv1alpha1.BackupTypeIncremental) + r.setGlobalPersistentVolumeClaim(backupPolicy.Spec.Logfile) + return r.handlePolicy(reqCtx, backupPolicy, backupPolicy.Spec.Logfile.BasePolicy, + cronExpression, dataprotectionv1alpha1.BackupTypeLogFile) } // setGlobalPersistentVolumeClaim sets global config of pvc to common policy. diff --git a/controllers/dataprotection/backuppolicy_controller_test.go b/controllers/dataprotection/backuppolicy_controller_test.go index 9ee9ef7432f..344528408d6 100644 --- a/controllers/dataprotection/backuppolicy_controller_test.go +++ b/controllers/dataprotection/backuppolicy_controller_test.go @@ -150,7 +150,7 @@ var _ = Describe("Backup Policy Controller", func() { Eventually(testapps.CheckObj(&testCtx, backupPolicyKey, func(g Gomega, fetched *dpv1alpha1.BackupPolicy) { g.Expect(fetched.Status.Phase).To(Equal(dpv1alpha1.PolicyAvailable)) })).Should(Succeed()) - Eventually(testapps.CheckObj(&testCtx, getCronjobKey(dpv1alpha1.BackupTypeFull), func(g Gomega, fetched *batchv1.CronJob) { + Eventually(testapps.CheckObj(&testCtx, getCronjobKey(dpv1alpha1.BackupTypeDataFile), func(g Gomega, fetched *batchv1.CronJob) { g.Expect(fetched.Spec.Schedule).To(Equal(defaultSchedule)) })).Should(Succeed()) }) @@ -166,26 +166,26 @@ var _ = Describe("Backup Policy Controller", func() { autoBackupLabel := map[string]string{ dataProtectionLabelAutoBackupKey: "true", dataProtectionLabelBackupPolicyKey: backupPolicyName, - dataProtectionLabelBackupTypeKey: string(dpv1alpha1.BaseBackupTypeFull), + dataProtectionLabelBackupTypeKey: string(dpv1alpha1.BackupTypeDataFile), } By("create a expired backup") backupExpired := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupNamePrefix). WithRandomName().AddLabelsInMap(autoBackupLabel). SetBackupPolicyName(backupPolicyName). - SetBackupType(dpv1alpha1.BackupTypeFull). + SetBackupType(dpv1alpha1.BackupTypeDataFile). Create(&testCtx).GetObject() By("create 1st limit backup") backupOutLimit1 := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupNamePrefix). WithRandomName().AddLabelsInMap(autoBackupLabel). SetBackupPolicyName(backupPolicyName). - SetBackupType(dpv1alpha1.BackupTypeFull). + SetBackupType(dpv1alpha1.BackupTypeDataFile). Create(&testCtx).GetObject() By("create 2nd limit backup") backupOutLimit2 := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupNamePrefix). WithRandomName().AddLabelsInMap(autoBackupLabel). SetBackupPolicyName(backupPolicyName). - SetBackupType(dpv1alpha1.BackupTypeFull). + SetBackupType(dpv1alpha1.BackupTypeDataFile). Create(&testCtx).GetObject() By("waiting expired backup completed") @@ -225,11 +225,11 @@ var _ = Describe("Backup Policy Controller", func() { patchBackupStatus(backupStatus, client.ObjectKeyFromObject(backupOutLimit2)) // trigger the backup policy controller through update cronjob - patchCronJobStatus(getCronjobKey(dpv1alpha1.BackupTypeFull)) + patchCronJobStatus(getCronjobKey(dpv1alpha1.BackupTypeDataFile)) By("retain the latest backup") Eventually(testapps.List(&testCtx, intctrlutil.BackupSignature, - client.MatchingLabels(backupPolicy.Spec.Full.Target.LabelsSelector.MatchLabels), + client.MatchingLabels(backupPolicy.Spec.Datafile.Target.LabelsSelector.MatchLabels), client.InNamespace(backupPolicy.Namespace))).Should(HaveLen(1)) }) }) @@ -296,7 +296,7 @@ var _ = Describe("Backup Policy Controller", func() { backupPolicyKey := client.ObjectKeyFromObject(backupPolicy) Eventually(testapps.CheckObj(&testCtx, backupPolicyKey, func(g Gomega, fetched *dpv1alpha1.BackupPolicy) { g.Expect(fetched.Status.Phase).To(Equal(dpv1alpha1.PolicyAvailable)) - g.Expect(fetched.Spec.Full.Target.Secret.Name).To(Equal(randomSecretName)) + g.Expect(fetched.Spec.Datafile.Target.Secret.Name).To(Equal(randomSecretName)) })).Should(Succeed()) }) }) @@ -318,8 +318,8 @@ var _ = Describe("Backup Policy Controller", func() { backupPolicyKey := client.ObjectKeyFromObject(backupPolicy) Eventually(testapps.CheckObj(&testCtx, backupPolicyKey, func(g Gomega, fetched *dpv1alpha1.BackupPolicy) { g.Expect(fetched.Status.Phase).To(Equal(dpv1alpha1.PolicyAvailable)) - g.Expect(fetched.Spec.Full.PersistentVolumeClaim.Name).To(Equal(pvcName)) - g.Expect(fetched.Spec.Full.PersistentVolumeClaim.InitCapacity.String()).To(Equal(pvcInitCapacity)) + g.Expect(fetched.Spec.Datafile.PersistentVolumeClaim.Name).To(Equal(pvcName)) + g.Expect(fetched.Spec.Datafile.PersistentVolumeClaim.InitCapacity.String()).To(Equal(pvcInitCapacity)) })).Should(Succeed()) }) }) diff --git a/controllers/dataprotection/cue/cronjob.cue b/controllers/dataprotection/cue/cronjob.cue index e20d136eaff..ca98465b23f 100644 --- a/controllers/dataprotection/cue/cronjob.cue +++ b/controllers/dataprotection/cue/cronjob.cue @@ -78,7 +78,7 @@ EOF } } -cronjob_incremental: { +cronjob_logfile: { apiVersion: "batch/v1" kind: "CronJob" metadata: { diff --git a/controllers/dataprotection/restorejob_controller_test.go b/controllers/dataprotection/restorejob_controller_test.go index 11ef660b611..e777d772331 100644 --- a/controllers/dataprotection/restorejob_controller_test.go +++ b/controllers/dataprotection/restorejob_controller_test.go @@ -82,7 +82,7 @@ var _ = Describe("RestoreJob Controller", func() { By("By assure an backup obj") return testapps.NewBackupFactory(testCtx.DefaultNamespace, "backup-job-"). WithRandomName().SetBackupPolicyName(backupPolicy). - SetBackupType(dataprotectionv1alpha1.BackupTypeFull). + SetBackupType(dataprotectionv1alpha1.BackupTypeDataFile). Create(&testCtx).GetObject() } diff --git a/deploy/apecloud-mysql-scale/templates/backuppolicytemplate.yaml b/deploy/apecloud-mysql-scale/templates/backuppolicytemplate.yaml index c3a6bfab870..437548b7f73 100644 --- a/deploy/apecloud-mysql-scale/templates/backuppolicytemplate.yaml +++ b/deploy/apecloud-mysql-scale/templates/backuppolicytemplate.yaml @@ -11,12 +11,15 @@ spec: clusterDefinitionRef: apecloud-mysql-scale backupPolicies: - componentDefRef: mysql - ttl: 7d + retention: + ttl: 7d schedule: - baseBackup: - type: snapshot + snapshot: enable: false - cronExpression: "0 18 * * 0" + cronExpression: "0 18 * * *" + datafile: + enable: false + cronExpression: "0 18 * * *" snapshot: hooks: containerName: mysql @@ -26,5 +29,5 @@ spec: - "rm -f /data/mysql/data/.restore_new_cluster; sync" target: role: leader - full: + datafile: backupToolName: xtrabackup-for-apecloud-mysql-scale \ No newline at end of file diff --git a/deploy/apecloud-mysql/templates/backuppolicytemplate.yaml b/deploy/apecloud-mysql/templates/backuppolicytemplate.yaml index 9915dab7815..8a21fb0d8d6 100644 --- a/deploy/apecloud-mysql/templates/backuppolicytemplate.yaml +++ b/deploy/apecloud-mysql/templates/backuppolicytemplate.yaml @@ -11,12 +11,15 @@ spec: clusterDefinitionRef: apecloud-mysql backupPolicies: - componentDefRef: mysql - ttl: 7d + retention: + ttl: 7d schedule: - baseBackup: - type: snapshot + snapshot: enable: false - cronExpression: "0 18 * * 0" + cronExpression: "0 18 * * *" + datafile: + enable: false + cronExpression: "0 18 * * *" snapshot: hooks: containerName: mysql @@ -26,5 +29,5 @@ spec: - "rm -f /data/mysql/data/.restore_new_cluster; sync" target: role: leader - full: + datafile: backupToolName: xtrabackup-for-apecloud-mysql \ No newline at end of file diff --git a/deploy/helm/crds/apps.kubeblocks.io_backuppolicytemplates.yaml b/deploy/helm/crds/apps.kubeblocks.io_backuppolicytemplates.yaml index abd20cd4340..51ba57b1c3a 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_backuppolicytemplates.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_backuppolicytemplates.yaml @@ -59,8 +59,8 @@ spec: maxLength: 63 pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ type: string - full: - description: the policy for full backup. + datafile: + description: the policy for datafile backup. properties: backupStatusUpdates: description: define how to update metadata for backup status. @@ -147,8 +147,8 @@ spec: type: string type: object type: object - incremental: - description: the policy for incremental backup. + logfile: + description: the policy for logfile backup. properties: backupStatusUpdates: description: define how to update metadata for backup status. @@ -235,11 +235,22 @@ spec: type: string type: object type: object + retention: + description: retention describe how long the Backup should be + retained. if not set, will be retained forever. + properties: + ttl: + description: ttl is a time string ending with the 'd'|'D'|'h'|'H' + character to describe how long the Backup should be retained. + if not set, will be retained forever. + pattern: ^\d+[d|D|h|H]$ + type: string + type: object schedule: description: schedule policy for backup. properties: - baseBackup: - description: schedule policy for base backup. + datafile: + description: schedule policy for datafile backup. properties: cronExpression: description: the cron expression for schedule, the timezone @@ -248,20 +259,26 @@ spec: enable: description: enable or disable the schedule. type: boolean - type: - description: the type of base backup, only support full - and snapshot. - enum: - - full - - snapshot + required: + - cronExpression + - enable + type: object + logfile: + description: schedule policy for logfile backup. + properties: + cronExpression: + description: the cron expression for schedule, the timezone + is in UTC. see https://en.wikipedia.org/wiki/Cron. type: string + enable: + description: enable or disable the schedule. + type: boolean required: - cronExpression - enable - - type type: object - incremental: - description: schedule policy for incremental backup. + snapshot: + description: schedule policy for snapshot backup. properties: cronExpression: description: the cron expression for schedule, the timezone @@ -378,12 +395,6 @@ spec: type: string type: object type: object - ttl: - description: ttl is a time string ending with the 'd'|'D'|'h'|'H' - character to describe how long the Backup should be retained. - if not set, will be retained forever. - pattern: ^\d+[d|D|h|H]$ - type: string required: - componentDefRef type: object diff --git a/deploy/helm/crds/dataprotection.kubeblocks.io_backuppolicies.yaml b/deploy/helm/crds/dataprotection.kubeblocks.io_backuppolicies.yaml index 46a131acba0..b150563971e 100644 --- a/deploy/helm/crds/dataprotection.kubeblocks.io_backuppolicies.yaml +++ b/deploy/helm/crds/dataprotection.kubeblocks.io_backuppolicies.yaml @@ -50,8 +50,8 @@ spec: spec: description: BackupPolicySpec defines the desired state of BackupPolicy properties: - full: - description: the policy for full backup. + datafile: + description: the policy for datafile backup. properties: backupStatusUpdates: description: define how to update metadata for backup status. @@ -232,8 +232,8 @@ spec: - persistentVolumeClaim - target type: object - incremental: - description: the policy for incremental backup. + logfile: + description: the policy for logfile backup. properties: backupStatusUpdates: description: define how to update metadata for backup status. @@ -414,11 +414,22 @@ spec: - persistentVolumeClaim - target type: object + retention: + description: retention describe how long the Backup should be retained. + if not set, will be retained forever. + properties: + ttl: + description: ttl is a time string ending with the 'd'|'D'|'h'|'H' + character to describe how long the Backup should be retained. + if not set, will be retained forever. + pattern: ^\d+[d|D|h|H]$ + type: string + type: object schedule: description: schedule policy for backup. properties: - baseBackup: - description: schedule policy for base backup. + datafile: + description: schedule policy for datafile backup. properties: cronExpression: description: the cron expression for schedule, the timezone @@ -427,20 +438,26 @@ spec: enable: description: enable or disable the schedule. type: boolean - type: - description: the type of base backup, only support full and - snapshot. - enum: - - full - - snapshot + required: + - cronExpression + - enable + type: object + logfile: + description: schedule policy for logfile backup. + properties: + cronExpression: + description: the cron expression for schedule, the timezone + is in UTC. see https://en.wikipedia.org/wiki/Cron. type: string + enable: + description: enable or disable the schedule. + type: boolean required: - cronExpression - enable - - type type: object - incremental: - description: schedule policy for incremental backup. + snapshot: + description: schedule policy for snapshot backup. properties: cronExpression: description: the cron expression for schedule, the timezone @@ -596,12 +613,6 @@ spec: required: - target type: object - ttl: - description: ttl is a time string ending with the 'd'|'D'|'h'|'H' - character to describe how long the Backup should be retained. if - not set, will be retained forever. - pattern: ^\d+[d|D|h|H]$ - type: string type: object status: description: BackupPolicyStatus defines the observed state of BackupPolicy diff --git a/deploy/helm/crds/dataprotection.kubeblocks.io_backups.yaml b/deploy/helm/crds/dataprotection.kubeblocks.io_backups.yaml index 19138c606f3..c1c96731aa7 100644 --- a/deploy/helm/crds/dataprotection.kubeblocks.io_backups.yaml +++ b/deploy/helm/crds/dataprotection.kubeblocks.io_backups.yaml @@ -61,12 +61,12 @@ spec: pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ type: string backupType: - default: full - description: Backup Type. full or incremental or snapshot. if unset, - default is full. + default: datafile + description: Backup Type. datafile or logfile or snapshot. if unset, + default is datafile. enum: - - full - - incremental + - datafile + - logfile - snapshot type: string parentBackupName: diff --git a/deploy/milvus/templates/backuppolicytemplate.yaml b/deploy/milvus/templates/backuppolicytemplate.yaml index 154924259ff..c1be0359c64 100644 --- a/deploy/milvus/templates/backuppolicytemplate.yaml +++ b/deploy/milvus/templates/backuppolicytemplate.yaml @@ -9,10 +9,10 @@ spec: clusterDefinitionRef: milvus backupPolicies: - componentDefRef: milvus - ttl: 7d + retention: + ttl: 7d schedule: - baseBackup: - type: snapshot + snapshot: enable: false cronExpression: "0 18 * * 0" snapshot: diff --git a/deploy/mongodb/templates/backuppolicytemplate.yaml b/deploy/mongodb/templates/backuppolicytemplate.yaml index 177dd0fbda9..1002039f5f9 100644 --- a/deploy/mongodb/templates/backuppolicytemplate.yaml +++ b/deploy/mongodb/templates/backuppolicytemplate.yaml @@ -9,19 +9,22 @@ spec: clusterDefinitionRef: mongodb backupPolicies: - componentDefRef: mongodb - ttl: 7d + retention: + ttl: 7d schedule: - baseBackup: - type: full + snapshot: enable: false - cronExpression: "0 18 * * 0" + cronExpression: "0 18 * * *" + datafile: + enable: false + cronExpression: "0 18 * * *" snapshot: target: role: primary connectionCredentialKey: passwordKey: password usernameKey: username - full: + datafile: backupToolName: mongodb-physical-backup-tool backupsHistoryLimit: 7 target: diff --git a/deploy/postgresql/templates/backuppolicytemplate.yaml b/deploy/postgresql/templates/backuppolicytemplate.yaml index 77dec27e20e..6b896b6daef 100644 --- a/deploy/postgresql/templates/backuppolicytemplate.yaml +++ b/deploy/postgresql/templates/backuppolicytemplate.yaml @@ -9,13 +9,16 @@ spec: clusterDefinitionRef: postgresql backupPolicies: - componentDefRef: postgresql - ttl: 7d + retention: + ttl: 7d schedule: - baseBackup: - type: snapshot + snapshot: enable: false cronExpression: "0 18 * * *" - incremental: + datafile: + enable: false + cronExpression: "0 18 * * *" + logfile: enable: false cronExpression: "*/5 * * * *" snapshot: @@ -32,9 +35,9 @@ spec: containerName: postgresql script: /kb-scripts/backup-log-collector.sh true updateStage: post - full: + datafile: backupToolName: postgres-basebackup - incremental: + logfile: backupToolName: postgres-pitr target: role: primary diff --git a/deploy/qdrant/templates/backuppolicytemplate.yaml b/deploy/qdrant/templates/backuppolicytemplate.yaml index a8f4dc89d11..553f8ee163f 100644 --- a/deploy/qdrant/templates/backuppolicytemplate.yaml +++ b/deploy/qdrant/templates/backuppolicytemplate.yaml @@ -9,10 +9,10 @@ spec: clusterDefinitionRef: qdrant backupPolicies: - componentDefRef: qdrant - ttl: 7d + retention: + ttl: 7d schedule: - baseBackup: - type: snapshot + snapshot: enable: false cronExpression: "0 18 * * 0" snapshot: diff --git a/deploy/redis/templates/backuppolicytemplate.yaml b/deploy/redis/templates/backuppolicytemplate.yaml index f707cabf26b..dff9af63686 100644 --- a/deploy/redis/templates/backuppolicytemplate.yaml +++ b/deploy/redis/templates/backuppolicytemplate.yaml @@ -9,10 +9,10 @@ spec: clusterDefinitionRef: redis backupPolicies: - componentDefRef: redis - ttl: 7d + retention: + ttl: 7d schedule: - baseBackup: - type: snapshot + snapshot: enable: false cronExpression: "0 18 * * 0" snapshot: diff --git a/deploy/weaviate/templates/backuppolicytemplate.yaml b/deploy/weaviate/templates/backuppolicytemplate.yaml index 37b4c2faa25..579493cead8 100644 --- a/deploy/weaviate/templates/backuppolicytemplate.yaml +++ b/deploy/weaviate/templates/backuppolicytemplate.yaml @@ -9,10 +9,10 @@ spec: clusterDefinitionRef: weaviate backupPolicies: - componentDefRef: weaviate - ttl: 7d + retention: + ttl: 7d schedule: - baseBackup: - type: snapshot + snapshot: enable: false cronExpression: "0 18 * * 0" snapshot: diff --git a/docs/user_docs/cli/kbcli_cluster_backup.md b/docs/user_docs/cli/kbcli_cluster_backup.md index 6b0f5486480..c9c5f06dc49 100644 --- a/docs/user_docs/cli/kbcli_cluster_backup.md +++ b/docs/user_docs/cli/kbcli_cluster_backup.md @@ -15,10 +15,10 @@ kbcli cluster backup NAME [flags] kbcli cluster backup mycluster # create a snapshot backup - kbcli cluster backup mycluster --backup-type snapshot + kbcli cluster backup mycluster --type snapshot - # create a full backup - kbcli cluster backup mycluster --backup-type full + # create a datafile backup + kbcli cluster backup mycluster --type datafile # create a backup with specified backup policy kbcli cluster backup mycluster --backup-policy @@ -29,8 +29,8 @@ kbcli cluster backup NAME [flags] ``` --backup-name string Backup name --backup-policy string Backup policy name, this flag will be ignored when backup-type is snapshot - --backup-type string Backup type (default "snapshot") -h, --help help for backup + --type string Backup type (default "snapshot") ``` ### Options inherited from parent commands diff --git a/internal/cli/cmd/cluster/dataprotection.go b/internal/cli/cmd/cluster/dataprotection.go index 40127c0d3f0..671a3034703 100644 --- a/internal/cli/cmd/cluster/dataprotection.go +++ b/internal/cli/cmd/cluster/dataprotection.go @@ -72,10 +72,10 @@ var ( kbcli cluster backup mycluster # create a snapshot backup - kbcli cluster backup mycluster --backup-type snapshot + kbcli cluster backup mycluster --type snapshot - # create a full backup - kbcli cluster backup mycluster --backup-type full + # create a datafile backup + kbcli cluster backup mycluster --type datafile # create a backup with specified backup policy kbcli cluster backup mycluster --backup-policy @@ -213,7 +213,7 @@ func NewCreateBackupCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) }, } - cmd.Flags().StringVar(&o.BackupType, "backup-type", "snapshot", "Backup type") + cmd.Flags().StringVar(&o.BackupType, "type", "snapshot", "Backup type") cmd.Flags().StringVar(&o.BackupName, "backup-name", "", "Backup name") cmd.Flags().StringVar(&o.BackupPolicy, "backup-policy", "", "Backup policy name, this flag will be ignored when backup-type is snapshot") @@ -595,14 +595,19 @@ func printBackupPolicyList(o list.ListOptions) error { } tbl := printer.NewTablePrinter(o.Out) - tbl.SetHeader("NAME", "DEFAULT", "CLUSTER", "CREATE-TIME") + tbl.SetHeader("NAME", "DEFAULT", "CLUSTER", "CREATE-TIME", "STATUS") for _, obj := range backupPolicyList.Items { defaultPolicy, ok := obj.GetAnnotations()[constant.DefaultBackupPolicyAnnotationKey] + backupPolicy := &dataprotectionv1alpha1.BackupPolicy{} + if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, backupPolicy); err != nil { + return err + } if !ok { defaultPolicy = "false" } createTime := obj.GetCreationTimestamp() - tbl.AddRow(obj.GetName(), defaultPolicy, obj.GetLabels()[constant.AppInstanceLabelKey], util.TimeFormat(&createTime)) + tbl.AddRow(obj.GetName(), defaultPolicy, obj.GetLabels()[constant.AppInstanceLabelKey], + util.TimeFormat(&createTime), backupPolicy.Status.Phase) } tbl.Print() return nil diff --git a/internal/cli/cmd/cluster/dataprotection_test.go b/internal/cli/cmd/cluster/dataprotection_test.go index 76359bbad21..1ef63420240 100644 --- a/internal/cli/cmd/cluster/dataprotection_test.go +++ b/internal/cli/cmd/cluster/dataprotection_test.go @@ -258,7 +258,7 @@ var _ = Describe("DataProtection", func() { baseBackup.TypeMeta = backupTypeMeta baseBackup.Status.Phase = dataprotectionv1alpha1.BackupCompleted incrBackup := testapps.NewBackupFactory(testing.Namespace, backupName). - SetBackupType(dataprotectionv1alpha1.BackupTypeIncremental). + SetBackupType(dataprotectionv1alpha1.BackupTypeLogFile). SetBackLog(now.Add(-time.Minute), now.Add(time.Minute)). SetLabels(backupLabels).GetObject() incrBackup.TypeMeta = backupTypeMeta diff --git a/internal/cli/cmd/cluster/describe.go b/internal/cli/cmd/cluster/describe.go index 74de7d65a79..b62ace4e8b9 100644 --- a/internal/cli/cmd/cluster/describe.go +++ b/internal/cli/cmd/cluster/describe.go @@ -239,16 +239,22 @@ func showDataProtection(backupPolicies []dpv1alpha1.BackupPolicy, backups []dpv1 backupSchedule := printer.NoneString backupType := printer.NoneString scheduleEnable := "Disabled" - if policy.Spec.Schedule.BaseBackup != nil { - if policy.Spec.Schedule.BaseBackup.Enable { + if policy.Spec.Schedule.Snapshot != nil { + if policy.Spec.Schedule.Snapshot.Enable { scheduleEnable = "Enabled" + backupSchedule = policy.Spec.Schedule.Snapshot.CronExpression + backupType = string(dpv1alpha1.BackupTypeSnapshot) + } + } + if policy.Spec.Schedule.Datafile != nil { + if policy.Spec.Schedule.Datafile.Enable { + scheduleEnable = "Enabled" + backupSchedule = policy.Spec.Schedule.Datafile.CronExpression + backupType = string(dpv1alpha1.BackupTypeDataFile) } - backupSchedule = policy.Spec.Schedule.BaseBackup.CronExpression - backupType = string(policy.Spec.Schedule.BaseBackup.Type) - } - if policy.Spec.TTL != nil { - ttlString = *policy.Spec.TTL + if policy.Spec.Retention != nil && policy.Spec.Retention.TTL != nil { + ttlString = *policy.Spec.Retention.TTL } lastScheduleTime := printer.NoneString if policy.Status.LastScheduleTime != nil { diff --git a/internal/controller/component/restore_utils.go b/internal/controller/component/restore_utils.go index ba34a15c2f9..717546e5572 100644 --- a/internal/controller/component/restore_utils.go +++ b/internal/controller/component/restore_utils.go @@ -76,7 +76,7 @@ func BuildRestoredInfo2( return intctrlutil.NewErrorf(intctrlutil.ErrorTypeBackupNotCompleted, "backup %s is not completed", backup.Name) } switch backup.Spec.BackupType { - case dataprotectionv1alpha1.BackupTypeFull: + case dataprotectionv1alpha1.BackupTypeDataFile: return buildInitContainerWithFullBackup(component, backup, backupTool) case dataprotectionv1alpha1.BackupTypeSnapshot: return buildVolumeClaimTemplatesWithSnapshot(component, backup) diff --git a/internal/controller/component/restore_utils_test.go b/internal/controller/component/restore_utils_test.go index 514f3c4f77d..9a15aa900f8 100644 --- a/internal/controller/component/restore_utils_test.go +++ b/internal/controller/component/restore_utils_test.go @@ -84,7 +84,7 @@ var _ = Describe("probe_utils", func() { } backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). SetBackupPolicyName(backupPolicyName). - SetBackupType(dataprotectionv1alpha1.BackupTypeFull). + SetBackupType(dataprotectionv1alpha1.BackupTypeDataFile). Create(&testCtx).GetObject() updateBackupStatus(backup, backupToolName, dataprotectionv1alpha1.BackupCompleted) component := &SynthesizedComponent{ diff --git a/internal/controller/lifecycle/transformer_backup_policy_tpl.go b/internal/controller/lifecycle/transformer_backup_policy_tpl.go index aff78780327..449eb6c74d0 100644 --- a/internal/controller/lifecycle/transformer_backup_policy_tpl.go +++ b/internal/controller/lifecycle/transformer_backup_policy_tpl.go @@ -158,13 +158,13 @@ func (r *BackupPolicyTPLTransformer) syncBackupPolicy(backupPolicy *dataprotecti backupPolicy.Spec.Snapshot.Target = syncTheRoleLabel(backupPolicy.Spec.Snapshot.Target, policyTPL.Snapshot.BasePolicy) } - if backupPolicy.Spec.Full != nil && policyTPL.Full != nil { - backupPolicy.Spec.Full.Target = syncTheRoleLabel(backupPolicy.Spec.Full.Target, - policyTPL.Full.BasePolicy) + if backupPolicy.Spec.Datafile != nil && policyTPL.Datafile != nil { + backupPolicy.Spec.Datafile.Target = syncTheRoleLabel(backupPolicy.Spec.Datafile.Target, + policyTPL.Datafile.BasePolicy) } - if backupPolicy.Spec.Incremental != nil && policyTPL.Incremental != nil { - backupPolicy.Spec.Incremental.Target = syncTheRoleLabel(backupPolicy.Spec.Incremental.Target, - policyTPL.Incremental.BasePolicy) + if backupPolicy.Spec.Logfile != nil && policyTPL.Logfile != nil { + backupPolicy.Spec.Logfile.Target = syncTheRoleLabel(backupPolicy.Spec.Logfile.Target, + policyTPL.Logfile.BasePolicy) } } @@ -196,11 +196,17 @@ func (r *BackupPolicyTPLTransformer) buildBackupPolicy(policyTPL appsv1alpha1.Ba }, } bpSpec := backupPolicy.Spec - bpSpec.TTL = policyTPL.TTL - bpSpec.Schedule.BaseBackup = r.convertBaseBackupSchedulePolicy(policyTPL.Schedule.BaseBackup) - bpSpec.Schedule.Incremental = r.convertSchedulePolicy(policyTPL.Schedule.Incremental) - bpSpec.Full = r.convertCommonPolicy(policyTPL.Full, cluster.Name, *component, workloadType) - bpSpec.Incremental = r.convertCommonPolicy(policyTPL.Incremental, cluster.Name, *component, workloadType) + if policyTPL.Retention != nil { + bpSpec.Retention = &dataprotectionv1alpha1.RetentionSpec{ + TTL: policyTPL.Retention.TTL, + } + } + + bpSpec.Schedule.Snapshot = r.convertSchedulePolicy(policyTPL.Schedule.Snapshot) + bpSpec.Schedule.Datafile = r.convertSchedulePolicy(policyTPL.Schedule.Datafile) + bpSpec.Schedule.Logfile = r.convertSchedulePolicy(policyTPL.Schedule.Logfile) + bpSpec.Datafile = r.convertCommonPolicy(policyTPL.Datafile, cluster.Name, *component, workloadType) + bpSpec.Logfile = r.convertCommonPolicy(policyTPL.Logfile, cluster.Name, *component, workloadType) bpSpec.Snapshot = r.convertSnapshotPolicy(policyTPL.Snapshot, cluster.Name, *component, workloadType) backupPolicy.Spec = bpSpec return backupPolicy @@ -228,18 +234,6 @@ func (r *BackupPolicyTPLTransformer) convertSchedulePolicy(sp *appsv1alpha1.Sche } } -// convertBaseBackupSchedulePolicy converts the baseBackupSchedulePolicy from backupPolicyTemplate. -func (r *BackupPolicyTPLTransformer) convertBaseBackupSchedulePolicy(sp *appsv1alpha1.BaseBackupSchedulePolicy) *dataprotectionv1alpha1.BaseBackupSchedulePolicy { - if sp == nil { - return nil - } - schedulePolicy := r.convertSchedulePolicy(&sp.SchedulePolicy) - return &dataprotectionv1alpha1.BaseBackupSchedulePolicy{ - Type: dataprotectionv1alpha1.BaseBackupType(sp.Type), - SchedulePolicy: *schedulePolicy, - } -} - // convertBasePolicy converts the basePolicy from backupPolicyTemplate. func (r *BackupPolicyTPLTransformer) convertBasePolicy(bp appsv1alpha1.BasePolicy, clusterName string, diff --git a/internal/controller/plan/pitr.go b/internal/controller/plan/pitr.go index d328f229e93..51408351ee4 100644 --- a/internal/controller/plan/pitr.go +++ b/internal/controller/plan/pitr.go @@ -262,7 +262,7 @@ func (p *PointInTimeRecoveryManager) getLatestBaseBackup() (*dpv1alpha1.Backup, // 2. get the latest backup object var latestBackup *dpv1alpha1.Backup for _, item := range backups { - if item.Spec.BackupType != dpv1alpha1.BackupTypeIncremental && + if item.Spec.BackupType != dpv1alpha1.BackupTypeLogFile && item.Status.Manifests.BackupLog.StopTime != nil && !p.restoreTime.Before(item.Status.Manifests.BackupLog.StopTime) { latestBackup = &item break @@ -367,7 +367,7 @@ func (p *PointInTimeRecoveryManager) getIncrementalBackup(componentName string) client.MatchingLabels{ constant.AppInstanceLabelKey: p.sourceCluster, constant.KBAppComponentLabelKey: componentName, - constant.BackupTypeLabelKeyKey: string(dpv1alpha1.BackupTypeIncremental), + constant.BackupTypeLabelKeyKey: string(dpv1alpha1.BackupTypeLogFile), }); err != nil { return nil, err } diff --git a/internal/controller/plan/pitr_test.go b/internal/controller/plan/pitr_test.go index f93722fab9c..f294fac07f9 100644 --- a/internal/controller/plan/pitr_test.go +++ b/internal/controller/plan/pitr_test.go @@ -217,14 +217,14 @@ var _ = Describe("PITR Functions", func() { incrBackupLabels := map[string]string{ constant.AppInstanceLabelKey: sourceCluster, constant.KBAppComponentLabelKey: mysqlCompName, - constant.BackupTypeLabelKeyKey: string(dpv1alpha1.BackupTypeIncremental), + constant.BackupTypeLabelKeyKey: string(dpv1alpha1.BackupTypeLogFile), } incrStartTime := &startTime incrStopTime := &stopTime backupIncr := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). WithRandomName().SetLabels(incrBackupLabels). SetBackupPolicyName("test-fake"). - SetBackupType(dpv1alpha1.BackupTypeIncremental). + SetBackupType(dpv1alpha1.BackupTypeLogFile). Create(&testCtx).GetObject() backupStatus = dpv1alpha1.BackupStatus{ Phase: dpv1alpha1.BackupCompleted, diff --git a/internal/testutil/apps/backuppolicy_factory.go b/internal/testutil/apps/backuppolicy_factory.go index 1ffc081f084..175a302bf1f 100644 --- a/internal/testutil/apps/backuppolicy_factory.go +++ b/internal/testutil/apps/backuppolicy_factory.go @@ -42,10 +42,10 @@ func NewBackupPolicyFactory(namespace, name string) *MockBackupPolicyFactory { func (factory *MockBackupPolicyFactory) setBasePolicyField(setField func(basePolicy *dataprotectionv1alpha1.BasePolicy)) { var basePolicy *dataprotectionv1alpha1.BasePolicy switch factory.backupType { - case dataprotectionv1alpha1.BackupTypeFull: - basePolicy = &factory.get().Spec.Full.BasePolicy - case dataprotectionv1alpha1.BackupTypeIncremental: - basePolicy = &factory.get().Spec.Incremental.BasePolicy + case dataprotectionv1alpha1.BackupTypeDataFile: + basePolicy = &factory.get().Spec.Datafile.BasePolicy + case dataprotectionv1alpha1.BackupTypeLogFile: + basePolicy = &factory.get().Spec.Logfile.BasePolicy case dataprotectionv1alpha1.BackupTypeSnapshot: basePolicy = &factory.get().Spec.Snapshot.BasePolicy } @@ -59,10 +59,10 @@ func (factory *MockBackupPolicyFactory) setBasePolicyField(setField func(basePol func (factory *MockBackupPolicyFactory) setCommonPolicyField(setField func(commonPolicy *dataprotectionv1alpha1.CommonBackupPolicy)) { var commonPolicy *dataprotectionv1alpha1.CommonBackupPolicy switch factory.backupType { - case dataprotectionv1alpha1.BackupTypeFull: - commonPolicy = factory.get().Spec.Full - case dataprotectionv1alpha1.BackupTypeIncremental: - commonPolicy = factory.get().Spec.Incremental + case dataprotectionv1alpha1.BackupTypeDataFile: + commonPolicy = factory.get().Spec.Datafile + case dataprotectionv1alpha1.BackupTypeLogFile: + commonPolicy = factory.get().Spec.Logfile } if commonPolicy == nil { // ignore @@ -74,15 +74,16 @@ func (factory *MockBackupPolicyFactory) setCommonPolicyField(setField func(commo func (factory *MockBackupPolicyFactory) setScheduleField(setField func(schedulePolicy *dataprotectionv1alpha1.SchedulePolicy)) { var schedulePolicy *dataprotectionv1alpha1.SchedulePolicy switch factory.backupType { - case dataprotectionv1alpha1.BackupTypeFull, dataprotectionv1alpha1.BackupTypeSnapshot: - factory.get().Spec.Schedule.BaseBackup = &dataprotectionv1alpha1.BaseBackupSchedulePolicy{ - SchedulePolicy: dataprotectionv1alpha1.SchedulePolicy{}, - Type: dataprotectionv1alpha1.BaseBackupType(factory.backupType), - } - schedulePolicy = &factory.get().Spec.Schedule.BaseBackup.SchedulePolicy - case dataprotectionv1alpha1.BackupTypeIncremental: - schedulePolicy = &dataprotectionv1alpha1.SchedulePolicy{} - factory.get().Spec.Schedule.Incremental = schedulePolicy + case dataprotectionv1alpha1.BackupTypeDataFile: + factory.get().Spec.Schedule.Datafile = &dataprotectionv1alpha1.SchedulePolicy{} + schedulePolicy = factory.get().Spec.Schedule.Datafile + case dataprotectionv1alpha1.BackupTypeSnapshot: + factory.get().Spec.Schedule.Snapshot = &dataprotectionv1alpha1.SchedulePolicy{} + schedulePolicy = factory.get().Spec.Schedule.Snapshot + // todo: set logfile schedule + case dataprotectionv1alpha1.BackupTypeLogFile: + factory.get().Spec.Schedule.Logfile = &dataprotectionv1alpha1.SchedulePolicy{} + schedulePolicy = factory.get().Spec.Schedule.Snapshot } if schedulePolicy == nil { // ignore @@ -100,22 +101,22 @@ func (factory *MockBackupPolicyFactory) AddSnapshotPolicy() *MockBackupPolicyFac } func (factory *MockBackupPolicyFactory) AddFullPolicy() *MockBackupPolicyFactory { - factory.get().Spec.Full = &dataprotectionv1alpha1.CommonBackupPolicy{ + factory.get().Spec.Datafile = &dataprotectionv1alpha1.CommonBackupPolicy{ PersistentVolumeClaim: dataprotectionv1alpha1.PersistentVolumeClaim{ CreatePolicy: dataprotectionv1alpha1.CreatePVCPolicyIfNotPresent, }, } - factory.backupType = dataprotectionv1alpha1.BackupTypeFull + factory.backupType = dataprotectionv1alpha1.BackupTypeDataFile return factory } func (factory *MockBackupPolicyFactory) AddIncrementalPolicy() *MockBackupPolicyFactory { - factory.get().Spec.Incremental = &dataprotectionv1alpha1.CommonBackupPolicy{ + factory.get().Spec.Logfile = &dataprotectionv1alpha1.CommonBackupPolicy{ PersistentVolumeClaim: dataprotectionv1alpha1.PersistentVolumeClaim{ CreatePolicy: dataprotectionv1alpha1.CreatePVCPolicyIfNotPresent, }, } - factory.backupType = dataprotectionv1alpha1.BackupTypeIncremental + factory.backupType = dataprotectionv1alpha1.BackupTypeLogFile return factory } @@ -135,7 +136,9 @@ func (factory *MockBackupPolicyFactory) SetSchedule(schedule string, enable bool } func (factory *MockBackupPolicyFactory) SetTTL(duration string) *MockBackupPolicyFactory { - factory.get().Spec.TTL = &duration + factory.get().Spec.Retention = &dataprotectionv1alpha1.RetentionSpec{ + TTL: &duration, + } return factory } diff --git a/internal/testutil/apps/backuppolicytemplate_factory.go b/internal/testutil/apps/backuppolicytemplate_factory.go index d6acdaf1d75..efaaa2982ba 100644 --- a/internal/testutil/apps/backuppolicytemplate_factory.go +++ b/internal/testutil/apps/backuppolicytemplate_factory.go @@ -59,7 +59,9 @@ func (factory *MockBackupPolicyTemplateFactory) AddBackupPolicy(componentDef str } func (factory *MockBackupPolicyTemplateFactory) SetTTL(duration string) *MockBackupPolicyTemplateFactory { - factory.getLastBackupPolicy().TTL = &duration + factory.getLastBackupPolicy().Retention = &appsv1alpha1.RetentionSpec{ + TTL: &duration, + } return factory } @@ -67,10 +69,10 @@ func (factory *MockBackupPolicyTemplateFactory) setBasePolicyField(setField func backupPolicy := factory.getLastBackupPolicy() var basePolicy *appsv1alpha1.BasePolicy switch factory.backupType { - case dataprotectionv1alpha1.BackupTypeFull: - basePolicy = &backupPolicy.Full.BasePolicy - case dataprotectionv1alpha1.BackupTypeIncremental: - basePolicy = &backupPolicy.Incremental.BasePolicy + case dataprotectionv1alpha1.BackupTypeDataFile: + basePolicy = &backupPolicy.Datafile.BasePolicy + case dataprotectionv1alpha1.BackupTypeLogFile: + basePolicy = &backupPolicy.Logfile.BasePolicy case dataprotectionv1alpha1.BackupTypeSnapshot: basePolicy = &backupPolicy.Snapshot.BasePolicy } @@ -85,10 +87,10 @@ func (factory *MockBackupPolicyTemplateFactory) setCommonPolicyField(setField fu backupPolicy := factory.getLastBackupPolicy() var commonPolicy *appsv1alpha1.CommonBackupPolicy switch factory.backupType { - case dataprotectionv1alpha1.BackupTypeFull: - commonPolicy = backupPolicy.Full - case dataprotectionv1alpha1.BackupTypeIncremental: - commonPolicy = backupPolicy.Incremental + case dataprotectionv1alpha1.BackupTypeDataFile: + commonPolicy = backupPolicy.Datafile + case dataprotectionv1alpha1.BackupTypeLogFile: + commonPolicy = backupPolicy.Logfile } if commonPolicy == nil { // ignore @@ -101,15 +103,15 @@ func (factory *MockBackupPolicyTemplateFactory) setScheduleField(setField func(s backupPolicy := factory.getLastBackupPolicy() var schedulePolicy *appsv1alpha1.SchedulePolicy switch factory.backupType { - case dataprotectionv1alpha1.BackupTypeFull, dataprotectionv1alpha1.BackupTypeSnapshot: - backupPolicy.Schedule.BaseBackup = &appsv1alpha1.BaseBackupSchedulePolicy{ - SchedulePolicy: appsv1alpha1.SchedulePolicy{}, - Type: appsv1alpha1.BaseBackupType(factory.backupType), - } - schedulePolicy = &backupPolicy.Schedule.BaseBackup.SchedulePolicy - case dataprotectionv1alpha1.BackupTypeIncremental: - schedulePolicy = &appsv1alpha1.SchedulePolicy{} - backupPolicy.Schedule.Incremental = schedulePolicy + case dataprotectionv1alpha1.BackupTypeSnapshot: + backupPolicy.Schedule.Snapshot = &appsv1alpha1.SchedulePolicy{} + schedulePolicy = backupPolicy.Schedule.Snapshot + case dataprotectionv1alpha1.BackupTypeDataFile: + backupPolicy.Schedule.Datafile = &appsv1alpha1.SchedulePolicy{} + schedulePolicy = backupPolicy.Schedule.Datafile + case dataprotectionv1alpha1.BackupTypeLogFile: + backupPolicy.Schedule.Logfile = &appsv1alpha1.SchedulePolicy{} + schedulePolicy = backupPolicy.Schedule.Logfile } if schedulePolicy == nil { // ignore @@ -129,15 +131,15 @@ func (factory *MockBackupPolicyTemplateFactory) AddSnapshotPolicy() *MockBackupP func (factory *MockBackupPolicyTemplateFactory) AddFullPolicy() *MockBackupPolicyTemplateFactory { backupPolicy := factory.getLastBackupPolicy() - backupPolicy.Full = &appsv1alpha1.CommonBackupPolicy{} - factory.backupType = dataprotectionv1alpha1.BackupTypeFull + backupPolicy.Datafile = &appsv1alpha1.CommonBackupPolicy{} + factory.backupType = dataprotectionv1alpha1.BackupTypeDataFile return factory } func (factory *MockBackupPolicyTemplateFactory) AddIncrementalPolicy() *MockBackupPolicyTemplateFactory { backupPolicy := factory.getLastBackupPolicy() - backupPolicy.Incremental = &appsv1alpha1.CommonBackupPolicy{} - factory.backupType = dataprotectionv1alpha1.BackupTypeIncremental + backupPolicy.Logfile = &appsv1alpha1.CommonBackupPolicy{} + factory.backupType = dataprotectionv1alpha1.BackupTypeLogFile return factory } diff --git a/test/integration/backup_mysql_test.go b/test/integration/backup_mysql_test.go index 11afeaa3241..c94f7863ad7 100644 --- a/test/integration/backup_mysql_test.go +++ b/test/integration/backup_mysql_test.go @@ -152,7 +152,7 @@ var _ = Describe("MySQL data protection function", func() { backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). WithRandomName(). SetBackupPolicyName(backupPolicyKey.Name). - SetBackupType(dpv1alpha1.BackupTypeFull). + SetBackupType(dpv1alpha1.BackupTypeDataFile). Create(&testCtx).GetObject() backupKey = client.ObjectKeyFromObject(backup) }