feat: opsrequest support more backup params (apecloud#5470)

apis/apps/v1alpha1/opsrequest_types.go

@@ -370,11 +370,36 @@ type BackupSpec struct {
 
 	// Which backupPolicy is applied to perform this backup
 	// +optional
-	BackupPolicyName string `json:"backupPolicyName"`
+	BackupPolicyName string `json:"backupPolicyName,omitempty"`
 
 	// Backup method name that is defined in backupPolicy.
 	// +optional
-	BackupMethod string `json:"backupMethod"`
+	BackupMethod string `json:"backupMethod,omitempty"`
+
+	// deletionPolicy determines whether the backup contents stored in backup repository
+	// should be deleted when the backup custom resource is deleted.
+	// Supported values are "Retain" and "Delete".
+	// "Retain" means that the backup content and its physical snapshot on backup repository are kept.
+	// "Delete" means that the backup content and its physical snapshot on backup repository are deleted.
+	// +kubebuilder:validation:Enum=Delete;Retain
+	// +kubebuilder:validation:Required
+	// +kubebuilder:default=Delete
+	// +optional
+	DeletionPolicy string `json:"deletionPolicy,omitempty"`
+
+	// retentionPeriod determines a duration up to which the backup should be kept.
+	// Controller will remove all backups that are older than the RetentionPeriod.
+	// For example, RetentionPeriod of `30d` will keep only the backups of last 30 days.
+	// Sample duration format:
+	// - years: 	2y
+	// - months: 	6mo
+	// - days: 		30d
+	// - hours: 	12h
+	// - minutes: 	30m
+	// You can also combine the above durations. For example: 30d12h30m.
+	// If not set, the backup will be kept forever.
+	// +optional
+	RetentionPeriod string `json:"retentionPeriod,omitempty"`
 
 	// if backupType is incremental, parentBackupName is required.
 	// +optional

config/crd/bases/apps.kubeblocks.io_opsrequests.yaml

@@ -72,10 +72,33 @@ spec:
                   backupPolicyName:
                     description: Which backupPolicy is applied to perform this backup
                     type: string
+                  deletionPolicy:
+                    default: Delete
+                    description: deletionPolicy determines whether the backup contents
+                      stored in backup repository should be deleted when the backup
+                      custom resource is deleted. Supported values are "Retain" and
+                      "Delete". "Retain" means that the backup content and its physical
+                      snapshot on backup repository are kept. "Delete" means that
+                      the backup content and its physical snapshot on backup repository
+                      are deleted.
+                    enum:
+                    - Delete
+                    - Retain
+                    type: string
                   parentBackupName:
                     description: if backupType is incremental, parentBackupName is
                       required.
                     type: string
+                  retentionPeriod:
+                    description: "retentionPeriod determines a duration up to which
+                      the backup should be kept. Controller will remove all backups
+                      that are older than the RetentionPeriod. For example, RetentionPeriod
+                      of `30d` will keep only the backups of last 30 days. Sample
+                      duration format: - years: \t2y - months: \t6mo - days: \t\t30d
+                      - hours: \t12h - minutes: \t30m You can also combine the above
+                      durations. For example: 30d12h30m. If not set, the backup will
+                      be kept forever."
+                    type: string
                 type: object
               cancel:
                 description: 'cancel defines the action to cancel the Pending/Creating/Running

controllers/apps/operations/backup.go

@@ -32,6 +32,7 @@ import (
 	"github.com/apecloud/kubeblocks/pkg/constant"
 	intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil"
 	dptypes "github.com/apecloud/kubeblocks/pkg/dataprotection/types"
+	"github.com/apecloud/kubeblocks/pkg/dataprotection/utils"
 )
 
 const backupTimeLayout = "20060102150405"
@@ -120,6 +121,24 @@ func buildBackup(reqCtx intctrlutil.RequestCtx, cli client.Client, opsRequest *a
 		return nil, err
 	}
 
+	backupPolicyList := &dpv1alpha1.BackupPolicyList{}
+	if err := cli.List(reqCtx.Ctx, backupPolicyList, client.InNamespace(cluster.Namespace),
+		client.MatchingLabels(map[string]string{
+			constant.AppInstanceLabelKey: cluster.Name,
+		})); err != nil {
+		return nil, err
+	}
+	defaultBackupMethod, backupMethodMap, err := utils.GetBackupMethodsFromBackupPolicy(backupPolicyList, backupSpec.BackupPolicyName)
+	if err != nil {
+		return nil, err
+	}
+	if backupSpec.BackupMethod == "" {
+		backupSpec.BackupMethod = defaultBackupMethod
+	}
+	if _, ok := backupMethodMap[backupSpec.BackupMethod]; !ok {
+		return nil, fmt.Errorf("backup method %s is not supported, please check cluster's backup policy", backupSpec.BackupMethod)
+	}
+
 	backup := &dpv1alpha1.Backup{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      backupSpec.BackupName,
@@ -132,6 +151,32 @@ func buildBackup(reqCtx intctrlutil.RequestCtx, cli client.Client, opsRequest *a
 		},
 	}
 
+	if backupSpec.DeletionPolicy != "" {
+		backup.Spec.DeletionPolicy = dpv1alpha1.BackupDeletionPolicy(backupSpec.DeletionPolicy)
+	}
+	if backupSpec.RetentionPeriod != "" {
+		retentionPeriod := dpv1alpha1.RetentionPeriod(backupSpec.RetentionPeriod)
+		if _, err := retentionPeriod.ToDuration(); err != nil {
+			return nil, err
+		}
+		backup.Spec.RetentionPeriod = retentionPeriod
+	}
+	if backupSpec.ParentBackupName != "" {
+		parentBackup := dpv1alpha1.Backup{}
+		if err := cli.Get(reqCtx.Ctx, client.ObjectKey{Name: backupSpec.ParentBackupName, Namespace: cluster.Namespace}, &parentBackup); err != nil {
+			return nil, err
+		}
+		// check parent backup exists and completed
+		if parentBackup.Status.Phase != dpv1alpha1.BackupPhaseCompleted {
+			return nil, fmt.Errorf("parent backup %s is not completed", backupSpec.ParentBackupName)
+		}
+		// check parent backup belongs to the cluster of the backup
+		if parentBackup.Labels[constant.AppInstanceLabelKey] != cluster.Name {
+			return nil, fmt.Errorf("parent backup %s is not belong to cluster %s", backupSpec.ParentBackupName, cluster.Name)
+		}
+		backup.Spec.ParentBackupName = backupSpec.ParentBackupName
+	}
+
 	return backup, nil
 }
 

deploy/helm/crds/apps.kubeblocks.io_opsrequests.yaml

@@ -72,10 +72,33 @@ spec:
                   backupPolicyName:
                     description: Which backupPolicy is applied to perform this backup
                     type: string
+                  deletionPolicy:
+                    default: Delete
+                    description: deletionPolicy determines whether the backup contents
+                      stored in backup repository should be deleted when the backup
+                      custom resource is deleted. Supported values are "Retain" and
+                      "Delete". "Retain" means that the backup content and its physical
+                      snapshot on backup repository are kept. "Delete" means that
+                      the backup content and its physical snapshot on backup repository
+                      are deleted.
+                    enum:
+                    - Delete
+                    - Retain
+                    type: string
                   parentBackupName:
                     description: if backupType is incremental, parentBackupName is
                       required.
                     type: string
+                  retentionPeriod:
+                    description: "retentionPeriod determines a duration up to which
+                      the backup should be kept. Controller will remove all backups
+                      that are older than the RetentionPeriod. For example, RetentionPeriod
+                      of `30d` will keep only the backups of last 30 days. Sample
+                      duration format: - years: \t2y - months: \t6mo - days: \t\t30d
+                      - hours: \t12h - minutes: \t30m You can also combine the above
+                      durations. For example: 30d12h30m. If not set, the backup will
+                      be kept forever."
+                    type: string
                 type: object
               cancel:
                 description: 'cancel defines the action to cancel the Pending/Creating/Running

pkg/cli/cmd/cluster/update.go

@@ -52,8 +52,7 @@ import (
 	cfgcore "github.com/apecloud/kubeblocks/pkg/configuration/core"
 	"github.com/apecloud/kubeblocks/pkg/constant"
 	"github.com/apecloud/kubeblocks/pkg/controller/configuration"
-	dptypes "github.com/apecloud/kubeblocks/pkg/dataprotection/types"
-	"github.com/apecloud/kubeblocks/pkg/dataprotection/utils/boolptr"
+	"github.com/apecloud/kubeblocks/pkg/dataprotection/utils"
 	"github.com/apecloud/kubeblocks/pkg/gotemplate"
 )
 
@@ -263,7 +262,18 @@ func (o *updateOptions) buildPatch(flags []*pflag.Flag) error {
 	if o.cluster != nil {
 		// if update the backup config, the backup method must have value
 		if o.cluster.Spec.Backup != nil {
-			defaultBackupMethod, backupMethodMap, err := o.getBackupMethodsFromBackupPolicy()
+			backupPolicyListObj, err := o.dynamic.Resource(types.BackupPolicyGVR()).Namespace(o.namespace).List(context.Background(), metav1.ListOptions{
+				LabelSelector: fmt.Sprintf("%s=%s", constant.AppInstanceLabelKey, o.cluster.Name),
+			})
+			if err != nil {
+				return err
+			}
+			backupPolicyList := &dpv1alpha1.BackupPolicyList{}
+			if err := runtime.DefaultUnstructuredConverter.FromUnstructured(backupPolicyListObj.UnstructuredContent(), backupPolicyList); err != nil {
+				return err
+			}
+
+			defaultBackupMethod, backupMethodMap, err := utils.GetBackupMethodsFromBackupPolicy(backupPolicyList, "")
 			if err != nil {
 				return err
 			}
@@ -616,47 +626,3 @@ func (o *updateOptions) updateBackupPitrEnabled(val string) error {
 	o.cluster.Spec.Backup.PITREnabled = &boolVal
 	return nil
 }
-
-// get backup methods from cluster's backup policy
-// if method's snapshotVolumes is true, use the method as the default backup method
-func (o *updateOptions) getBackupMethodsFromBackupPolicy() (string, map[string]struct{}, error) {
-	if o.cluster == nil {
-		return "", nil, fmt.Errorf("cluster is nil")
-	}
-
-	var backupPolicy []dpv1alpha1.BackupPolicy
-	obj, err := o.dynamic.Resource(types.BackupPolicyGVR()).Namespace(o.namespace).List(context.Background(), metav1.ListOptions{
-		LabelSelector: fmt.Sprintf("%s=%s", constant.AppInstanceLabelKey, o.cluster.Name),
-	})
-	if err != nil {
-		return "", nil, err
-	}
-	for _, item := range obj.Items {
-		var bp dpv1alpha1.BackupPolicy
-		if err = runtime.DefaultUnstructuredConverter.FromUnstructured(item.Object, &bp); err != nil {
-			return "", nil, err
-		}
-		backupPolicy = append(backupPolicy, bp)
-	}
-
-	var defaultBackupMethod string
-	var backupMethodsMap = make(map[string]struct{})
-	for _, policy := range backupPolicy {
-		if policy.Annotations[dptypes.DefaultBackupPolicyAnnotationKey] != annotationTrueValue {
-			continue
-		}
-		if policy.Status.Phase != dpv1alpha1.AvailablePhase {
-			continue
-		}
-		for _, method := range policy.Spec.BackupMethods {
-			if boolptr.IsSetToTrue(method.SnapshotVolumes) {
-				defaultBackupMethod = method.Name
-			}
-			backupMethodsMap[method.Name] = struct{}{}
-		}
-	}
-	if defaultBackupMethod == "" {
-		return "", nil, fmt.Errorf("failed to find default backup method which snapshotVolumes is true, please check cluster's backup policy")
-	}
-	return defaultBackupMethod, backupMethodsMap, nil
-}

pkg/dataprotection/utils/backup.go

@@ -0,0 +1,60 @@
+/*
+Copyright (C) 2022-2023 ApeCloud Co., Ltd
+
+This file is part of KubeBlocks project
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package utils
+
+import (
+	"fmt"
+
+	dpv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1"
+	dptypes "github.com/apecloud/kubeblocks/pkg/dataprotection/types"
+	"github.com/apecloud/kubeblocks/pkg/dataprotection/utils/boolptr"
+)
+
+// GetBackupMethodsFromBackupPolicy get backup methods from backup policy
+// if backup policy is specified, search the backup policy with the name
+// if backup policy is not specified, search the default backup policy
+// if method's snapshotVolumes is true, use the method as the default backup method
+func GetBackupMethodsFromBackupPolicy(backupPolicyList *dpv1alpha1.BackupPolicyList, backupPolicyName string) (string, map[string]struct{}, error) {
+	var defaultBackupMethod string
+	var backupMethodsMap = make(map[string]struct{})
+	for _, policy := range backupPolicyList.Items {
+		// if backupPolicyName is not empty, only use the backup policy with the name
+		if backupPolicyName != "" && policy.Name != backupPolicyName {
+			continue
+		}
+		// if backupPolicyName is empty, only use the default backup policy
+		if backupPolicyName == "" && policy.Annotations[dptypes.DefaultBackupPolicyAnnotationKey] != "true" {
+			continue
+		}
+		if policy.Status.Phase != dpv1alpha1.AvailablePhase {
+			continue
+		}
+		for _, method := range policy.Spec.BackupMethods {
+			if boolptr.IsSetToTrue(method.SnapshotVolumes) {
+				defaultBackupMethod = method.Name
+			}
+			backupMethodsMap[method.Name] = struct{}{}
+		}
+	}
+	if defaultBackupMethod == "" {
+		return "", nil, fmt.Errorf("failed to find default backup method which snapshotVolumes is true, please check cluster's backup policy")
+	}
+	return defaultBackupMethod, backupMethodsMap, nil
+}