feat: support pitr (apecloud#1738)

apis/apps/v1alpha1/opsrequest_types.go

@@ -85,6 +85,10 @@ type OpsRequestSpec struct {
 	// +listType=map
 	// +listMapKey=componentName
 	ExposeList []Expose `json:"expose,omitempty" patchStrategy:"merge,retainKeys" patchMergeKey:"componentName"`
+
+	// cluster RestoreFrom backup or point in time
+	// +optional
+	RestoreFrom *RestoreFromSpec `json:"restoreFrom,omitempty"`
 }
 
 // ComponentOps defines the common variables of component scope operations.
@@ -223,6 +227,42 @@ type Expose struct {
 	Services []ClusterComponentService `json:"services"`
 }
 
+type RestoreFromSpec struct {
+	// use the backup name and component name for restore, support for multiple components' recovery.
+	// +optional
+	Backup []BackupRefSpec `json:"backup,omitempty"`
+
+	// specified the point in time to recovery
+	// +optional
+	PointInTime *PointInTimeRefSpec `json:"pointInTime,omitempty"`
+}
+
+type RefNamespaceName struct {
+	// specified the name
+	// +optional
+	Name string `json:"name,omitempty"`
+
+	// specified the namespace
+	// +optional
+	Namespace string `json:"namespace,omitempty"`
+}
+
+type BackupRefSpec struct {
+	// specify a reference backup to restore
+	// +optional
+	Ref RefNamespaceName `json:"ref,omitempty"`
+}
+
+type PointInTimeRefSpec struct {
+	// specify the time point to restore, with UTC as the time zone.
+	// +optional
+	Time *metav1.Time `json:"time,omitempty"`
+
+	// specify a reference source cluster to restore
+	// +optional
+	Ref RefNamespaceName `json:"ref,omitempty"`
+}
+
 // OpsRequestStatus defines the observed state of OpsRequest
 type OpsRequestStatus struct {
 

apis/dataprotection/v1alpha1/backuptool_types.go

@@ -32,6 +32,11 @@ type BackupToolSpec struct {
 	// +kubebuilder:default=job
 	DeployKind string `json:"deployKind,omitempty"`
 
+	// the type of backup tool, file or pitr
+	// +kubebuilder:validation:Enum={file,pitr}
+	// +kubebuilder:default=file
+	Type string `json:"type,omitempty"`
+
 	// Compute Resources required by this container.
 	// Cannot be updated.
 	// +kubebuilder:pruning:PreserveUnknownFields

config/crd/bases/apps.kubeblocks.io_opsrequests.yaml

@@ -223,6 +223,46 @@ spec:
                 x-kubernetes-list-map-keys:
                 - componentName
                 x-kubernetes-list-type: map
+              restoreFrom:
+                description: cluster RestoreFrom backup or point in time
+                properties:
+                  backup:
+                    description: use the backup name and component name for restore,
+                      support for multiple components' recovery.
+                    items:
+                      properties:
+                        ref:
+                          description: specify a reference backup to restore
+                          properties:
+                            name:
+                              description: specified the name
+                              type: string
+                            namespace:
+                              description: specified the namespace
+                              type: string
+                          type: object
+                      type: object
+                    type: array
+                  pointInTime:
+                    description: specified the point in time to recovery
+                    properties:
+                      ref:
+                        description: specify a reference source cluster to restore
+                        properties:
+                          name:
+                            description: specified the name
+                            type: string
+                          namespace:
+                            description: specified the namespace
+                            type: string
+                        type: object
+                      time:
+                        description: specify the time point to restore, with UTC as
+                          the time zone.
+                        format: date-time
+                        type: string
+                    type: object
+                type: object
               ttlSecondsAfterSucceed:
                 description: ttlSecondsAfterSucceed OpsRequest will be deleted after
                   TTLSecondsAfterSucceed second when OpsRequest.status.phase is Succeed.

config/crd/bases/dataprotection.kubeblocks.io_backuptools.yaml

@@ -286,6 +286,13 @@ spec:
                     type: object
                 type: object
                 x-kubernetes-preserve-unknown-fields: true
+              type:
+                default: file
+                description: the type of backup tool, file or pitr
+                enum:
+                - file
+                - pitr
+                type: string
             required:
             - backupCommands
             - image

deploy/helm/crds/apps.kubeblocks.io_opsrequests.yaml

@@ -223,6 +223,46 @@ spec:
                 x-kubernetes-list-map-keys:
                 - componentName
                 x-kubernetes-list-type: map
+              restoreFrom:
+                description: cluster RestoreFrom backup or point in time
+                properties:
+                  backup:
+                    description: use the backup name and component name for restore,
+                      support for multiple components' recovery.
+                    items:
+                      properties:
+                        ref:
+                          description: specify a reference backup to restore
+                          properties:
+                            name:
+                              description: specified the name
+                              type: string
+                            namespace:
+                              description: specified the namespace
+                              type: string
+                          type: object
+                      type: object
+                    type: array
+                  pointInTime:
+                    description: specified the point in time to recovery
+                    properties:
+                      ref:
+                        description: specify a reference source cluster to restore
+                        properties:
+                          name:
+                            description: specified the name
+                            type: string
+                          namespace:
+                            description: specified the namespace
+                            type: string
+                        type: object
+                      time:
+                        description: specify the time point to restore, with UTC as
+                          the time zone.
+                        format: date-time
+                        type: string
+                    type: object
+                type: object
               ttlSecondsAfterSucceed:
                 description: ttlSecondsAfterSucceed OpsRequest will be deleted after
                   TTLSecondsAfterSucceed second when OpsRequest.status.phase is Succeed.

deploy/helm/crds/dataprotection.kubeblocks.io_backuptools.yaml

@@ -286,6 +286,13 @@ spec:
                     type: object
                 type: object
                 x-kubernetes-preserve-unknown-fields: true
+              type:
+                default: file
+                description: the type of backup tool, file or pitr
+                enum:
+                - file
+                - pitr
+                type: string
             required:
             - backupCommands
             - image

deploy/postgresql/templates/backuppolicytemplate.yaml

@@ -20,5 +20,14 @@ spec:
         connectionCredentialKey:
           passwordKey: password
           usernameKey: username
+      hooks:
+        containerName: postgresql
+        preCommands:
+          - psql -c "SELECT txid_current();CHECKPOINT;"
+      backupStatusUpdates:
+        - path: manifests.backupLog
+          containerName: postgresql
+          script: /kb-scripts/backup-log-collector.sh
+          updateStage: pre
     full:
-      backupToolName: postgres-basebackup
+      backupToolName: postgres-basebackup

deploy/postgresql/templates/backuptool-pitr.yaml

@@ -0,0 +1,54 @@
+apiVersion: dataprotection.kubeblocks.io/v1alpha1
+kind: BackupTool
+metadata:
+  labels:
+    clusterdefinition.kubeblocks.io/name: postgresql
+    kubeblocks.io/backup-tool-type: pitr
+    {{- include "postgresql.labels" . | nindent 4 }}
+  name: postgres-pitr
+spec:
+  backupCommands: []
+  deployKind: job
+  env:
+    - name: VOLUME_DATA_DIR
+      value: /home/postgres/pgdata
+    - name: VOLUME_LOG_DIR
+      value: /home/postgres/pgwal
+    - name: RESTORE_DATA_DIR
+      value: "$(VOLUME_DATA_DIR)/kb_restore"
+    - name: PITR_DIR
+      value: "$(VOLUME_DATA_DIR)/pitr"
+    - name: DATA_DIR
+      value: "$(VOLUME_DATA_DIR)/pgroot/data"
+    - name: CONF_DIR
+      value: "$(VOLUME_DATA_DIR)/conf"
+    - name: LOG_DIR
+      value: "$(VOLUME_LOG_DIR)/pgroot/data/pg_wal"
+    - name: RECOVERY_TIME
+      value: $KB_RECOVERY_TIME
+    - name: TIME_FORMAT
+      value: 2006-01-02 15:04:05 MST
+  image: alpine:3.17
+  logical:
+    restoreCommands:
+      - |
+        set -e;
+        rm -f ${CONF_DIR}/recovery.conf;
+        rm -rf ${DATA_DIR}.old;
+        rm -rf ${DATA_DIR}.failed;
+  physical:
+    restoreCommands:
+      - |
+        set -e;
+        mkdir -p ${PITR_DIR};
+        cp -R $LOG_DIR ${PITR_DIR}/;
+        chmod 777 -R ${PITR_DIR};
+        touch ${DATA_DIR}/recovery.signal;
+        mkdir -p ${CONF_DIR};
+        chmod 777 -R ${CONF_DIR};
+        mkdir -p ${RESTORE_DATA_DIR};
+        echo "cp -R ${DATA_DIR}.old ${DATA_DIR}" > ${RESTORE_DATA_DIR}/kb_restore.sh;
+        echo -e "restore_command=mv ${PITR_DIR}/pg_wal/%f %p\nrecovery_target_time=${RECOVERY_TIME}\nrecovery_target_action=promote" > ${CONF_DIR}/recovery.conf;
+        mv ${DATA_DIR} ${DATA_DIR}.old;
+        sync;
+  type: pitr

deploy/postgresql/templates/configmap.yaml

@@ -25,3 +25,9 @@ data:
       recovery_conf:
         restore_command: cp /home/postgres/pgdata/pgroot/arch/%f %p
         recovery_target_timeline: latest
+  kb_pitr.conf: |
+    method: kb_restore_from_time
+    kb_restore_from_time:
+      command: sh /home/postgres/pgdata/kb_restore/kb_restore.sh
+      keep_existing_recovery_conf: false
+      recovery_conf: {}

deploy/postgresql/templates/scripts.yaml

@@ -32,6 +32,19 @@ data:
             if line and not line.startswith('#'):
                 ret.append(line)
         return ret
+    def postgresql_conf_to_dict(file_path):
+        with open(file_path, 'r') as f:
+            content = f.read()
+        lines = content.splitlines()
+        result = {}
+        for line in lines:
+            if line.startswith('#'):
+                continue
+            if '=' not in line:
+                continue
+            key, value = line.split('=', 1)
+            result[key.strip()] = value.strip()
+        return result
     def main(filename):
         restore_dir = os.environ.get('RESTORE_DATA_DIR', '')
         local_config = yaml.safe_load(
@@ -52,6 +65,14 @@ data:
                 local_config['bootstrap'] = {}
             with open('/home/postgres/conf/kb_restore.conf', 'r') as f:
                 local_config['bootstrap'].update(yaml.safe_load(f))
+        # point in time recovery(PITR)
+        data_dir = os.environ.get('PGDATA', '')
+        if os.path.isfile("/home/postgres/pgdata/conf/recovery.conf"):
+            with open('/home/postgres/conf/kb_pitr.conf', 'r') as f:
+                pitr_config = yaml.safe_load(f)
+                re_config = postgresql_conf_to_dict("/home/postgres/pgdata/conf/recovery.conf")
+                pitr_config[pitr_config['method']]['recovery_conf'].update(re_config)
+                local_config['bootstrap'].update(pitr_config)
         write_file(yaml.dump(local_config, default_flow_style=False), filename, True)
     if __name__ == '__main__':
         main(sys.argv[1])
@@ -69,4 +90,13 @@ data:
     python3 /kb-scripts/generate_patroni_yaml.py tmp_patroni.yaml
     export SPILO_CONFIGURATION=$(cat tmp_patroni.yaml)
     # export SCOPE="$KB_CLUSTER_NAME-$KB_CLUSTER_NAME"
-    exec /launch.sh init
+    exec /launch.sh init
+  backup-log-collector.sh: |
+    #!/bin/bash
+    set -o errexit
+    set -o nounset
+    LOG_START_TIME=$(pg_waldump $(ls -tr $PGDATA/pg_wal/ | grep '[[:digit:]]$'|head -n 1) --rmgr=Transaction 2>/dev/null |head -n 1|awk -F ' COMMIT ' '{print $2}'|awk -F ';' '{print $1}')
+    LOG_STOP_TIME=$(pg_waldump $(ls -t $PGDATA/pg_wal/ | grep '[[:digit:]]$'|head -n 1) --rmgr=Transaction 2>/dev/null |tail -n 1|awk -F ' COMMIT ' '{print $2}'|awk -F ';' '{print $1}')
+    LOG_START_TIME=$(date -d "$LOG_START_TIME" -u '+%Y-%m-%dT%H:%M:%SZ')
+    LOG_STOP_TIME=$(date -d "$LOG_STOP_TIME" -u '+%Y-%m-%dT%H:%M:%SZ')
+    printf "{\"startTime\": \"$LOG_START_TIME\" ,\"stopTime\": \"$LOG_STOP_TIME\"}"

internal/constant/const.go

@@ -73,17 +73,20 @@ const (
 	VolumeTypeLabelKey              = "kubeblocks.io/volume-type"
 	KBManagedByKey                  = "apps.kubeblocks.io/managed-by" // KBManagedByKey marks resources that auto created during operation
 	ClassProviderLabelKey           = "class.kubeblocks.io/provider"
+	BackupToolTypeLabelKey          = "kubeblocks.io/backup-tool-type"
 
 	// kubeblocks.io annotations
-	OpsRequestAnnotationKey           = "kubeblocks.io/ops-request" // OpsRequestAnnotationKey OpsRequest annotation key in Cluster
-	ReconcileAnnotationKey            = "kubeblocks.io/reconcile"   // ReconcileAnnotationKey Notify k8s object to reconcile
-	RestartAnnotationKey              = "kubeblocks.io/restart"     // RestartAnnotationKey the annotation which notices the StatefulSet/DeploySet to restart
-	SnapShotForStartAnnotationKey     = "kubeblocks.io/snapshot-for-start"
-	RestoreFromBackUpAnnotationKey    = "kubeblocks.io/restore-from-backup" // RestoreFromBackUpAnnotationKey specifies the component to recover from the backup.
-	ClusterSnapshotAnnotationKey      = "kubeblocks.io/cluster-snapshot"    // ClusterSnapshotAnnotationKey saves the snapshot of cluster.
-	LeaderAnnotationKey               = "cs.apps.kubeblocks.io/leader"
-	DefaultBackupPolicyAnnotationKey  = "dataprotection.kubeblocks.io/is-default-policy"
-	BackupPolicyTemplateAnnotationKey = "apps.kubeblocks.io/backup-policy-template"
+	OpsRequestAnnotationKey            = "kubeblocks.io/ops-request" // OpsRequestAnnotationKey OpsRequest annotation key in Cluster
+	ReconcileAnnotationKey             = "kubeblocks.io/reconcile"   // ReconcileAnnotationKey Notify k8s object to reconcile
+	RestartAnnotationKey               = "kubeblocks.io/restart"     // RestartAnnotationKey the annotation which notices the StatefulSet/DeploySet to restart
+	SnapShotForStartAnnotationKey      = "kubeblocks.io/snapshot-for-start"
+	RestoreFromBackUpAnnotationKey     = "kubeblocks.io/restore-from-backup" // RestoreFromBackUpAnnotationKey specifies the component to recover from the backup.
+	ClusterSnapshotAnnotationKey       = "kubeblocks.io/cluster-snapshot"    // ClusterSnapshotAnnotationKey saves the snapshot of cluster.
+	LeaderAnnotationKey                = "cs.apps.kubeblocks.io/leader"
+	DefaultBackupPolicyAnnotationKey   = "dataprotection.kubeblocks.io/is-default-policy"
+	BackupPolicyTemplateAnnotationKey  = "apps.kubeblocks.io/backup-policy-template"
+	RestoreFromTimeAnnotationKey       = "kubeblocks.io/restore-from-time"           // RestoreFromTimeAnnotationKey specifies the time to recover from the backup.
+	RestoreFromSrcClusterAnnotationKey = "kubeblocks.io/restore-from-source-cluster" // RestoreFromSrcClusterAnnotationKey specifies the source cluster to recover from the backup.
 
 	// ConfigurationTplLabelPrefixKey clusterVersion or clusterdefinition using tpl
 	ConfigurationTplLabelPrefixKey         = "config.kubeblocks.io/tpl"

internal/controller/builder/builder.go

@@ -707,3 +707,23 @@ func BuildBackupManifestsJob(key types.NamespacedName, backup *dataprotectionv1a
 	}
 	return job, nil
 }
+
+func BuildPITRJob(name string, cluster *appsv1alpha1.Cluster, image string, command []string, args []string,
+	volumes []corev1.Volume, volumeMounts []corev1.VolumeMount, env []corev1.EnvVar) (*batchv1.Job, error) {
+	const tplFile = "pitr_job_template.cue"
+	job := &batchv1.Job{}
+	if err := buildFromCUE(tplFile, map[string]any{
+		"job.metadata.name":              name,
+		"job.metadata.namespace":         cluster.Namespace,
+		"job.spec.template.spec.volumes": volumes,
+		"container.image":                image,
+		"container.command":              command,
+		"container.args":                 args,
+		"container.volumeMounts":         volumeMounts,
+		"container.env":                  env,
+	}, "job", job); err != nil {
+		return nil, err
+	}
+
+	return job, nil
+}