add the crd of databackup to backup the metadata file and metadata in…

…fo file (fluid-cloudnative#533)

* add the crd of databackup, which can backup the metadata file and metadata info file of dadatset

* change backup logic to a helm chart

* fix ci bug

* test

* add doc of backup

* change the backup pod and configmap name

* doc-fix

* add the function of specifing host directory to backup

* improve document and naming

* change databackup charts name

Co-authored-by: 仇伶玮 <[email protected]>

PROJECT

@@ -13,4 +13,7 @@ resources:
 - group: data
   kind: JindoRuntime
   version: v1alpha1
+- group: data
+  kind: DataBackup
+  version: v1alpha1
 version: "2"

api/v1alpha1/databackup_types.go

@@ -0,0 +1,98 @@
+/*
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	"github.com/fluid-cloudnative/fluid/pkg/databackup"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
+// NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
+
+// DataBackupCondition describes conditions that explains transitions on phase
+type DataBackupCondition struct {
+	// Type of condition, either `Complete` or `Failed`
+	Type databackup.ConditionType `json:"type"`
+	// Status of the condition, one of `True`, `False` or `Unknown`
+	Status v1.ConditionStatus `json:"status"`
+	// Reason for the condition's last transition
+	Reason string `json:"reason,omitempty"`
+	// Message is a human-readable message indicating details about the transition
+	Message string `json:"message,omitempty"`
+	// LastProbeTime describes last time this condition was updated.
+	LastProbeTime metav1.Time `json:"lastProbeTime,omitempty"`
+	// LastTransitionTime describes last time the condition transitioned from one status to another.
+	LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"`
+}
+
+// BackupLocation describes the final backup location of DataBackup
+type BackupLocation struct {
+	// Path describes the path of backup, in the form of local:///absolutePath or pvc://<pvcName>/subpath
+	Path string `json:"path,omitempty"`
+	// NodeName describes the nodeName of backup if Path is in the form of local://subpath
+	NodeName string `json:"nodeName,omitempty"`
+}
+
+// DataBackupSpec defines the desired state of DataBackup
+type DataBackupSpec struct {
+	// Dataset defines the target dataset of the DataBackup
+	Dataset string `json:"dataset,omitempty"`
+	// BackupPath defines the target path to save data of the DataBackup
+	BackupPath string `json:"backupPath,omitempty"`
+}
+
+// DataBackupStatus defines the observed state of DataBackup
+type DataBackupStatus struct {
+	// Phase describes current phase of DataBackup
+	Phase databackup.Phase `json:"phase"`
+	// BackupLocation tell user the location to save data of the DataBackup
+	BackupLocation BackupLocation `json:"backupLocation,omitempty"`
+	// Conditions consists of transition information on DataBackup's Phase
+	Conditions []DataBackupCondition `json:"conditions"`
+}
+
+// +kubebuilder:printcolumn:name="Dataset",type="string",JSONPath=`.spec.dataset`
+// +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=`.status.phase`
+// +kubebuilder:printcolumn:name="BackupPath",type="string",JSONPath=`.status.backupLocation.path`
+// +kubebuilder:printcolumn:name="BackupNodeName",type="string",JSONPath=`.status.backupLocation.nodeName`
+// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=`.metadata.creationTimestamp`
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +genclient
+
+// DataBackup is the Schema for the backup API
+type DataBackup struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   DataBackupSpec   `json:"spec,omitempty"`
+	Status DataBackupStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// DataBackupList contains a list of DataBackup
+type DataBackupList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []DataBackup `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&DataBackup{}, &DataBackupList{})
+}

api/v1alpha1/dataset_types.go

@@ -174,6 +174,10 @@ type DatasetStatus struct {
 	// DataLoadRef specifies the running DataLoad job that targets this Dataset.
 	// This is mainly used as a lock to prevent concurrent DataLoad jobs.
 	DataLoadRef string `json:"dataLoadRef,omitempty"`
+
+	// DataBackupRef specifies the running Backup job that targets this Dataset.
+	// This is mainly used as a lock to prevent concurrent DataBackup jobs.
+	DataBackupRef string `json:"dataBackupRef,omitempty"`
 }
 
 // DatasetConditionType defines all kinds of types of cacheStatus.<br>

charts/alluxio/templates/master/statefulset.yaml

@@ -154,6 +154,10 @@ spec:
             name: embedded
           {{- end }}
           volumeMounts:
+          {{- if .Values.master.backupPath }}
+          - name: backup
+            mountPath: /alluxio_backups
+          {{- end }}
           {{ if .Values.initUsers.enabled -}}
           - name: user
             mountPath: /etc/passwd
@@ -270,7 +274,13 @@ spec:
           {{- end }}
       restartPolicy: Always
       volumes:
-        {{ if .Values.initUsers.enabled -}}
+        {{- if .Values.master.backupPath }}
+        - name: backup
+          hostPath:
+            path: {{ .Values.master.backupPath }}
+            type: DirectoryOrCreate
+        {{- end }}
+  {{ if .Values.initUsers.enabled -}}
         - name: dir
           hostPath:
             path: {{ .Values.initUsers.dir }}

charts/alluxio/values.yaml

@@ -95,6 +95,7 @@ master:
   # JVM options specific to the master container
   jvmOptions:
   nodeSelector: {}
+  backupPath:
 
 jobMaster:
   args:

charts/fluid-databackup/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/fluid-databackup/CHANGELOG.md

@@ -0,0 +1,4 @@
+### 0.1.0
+
+- Support parallel prefetch pod
+- Support configurations by setting values

charts/fluid-databackup/Chart.yaml

@@ -0,0 +1,23 @@
+apiVersion: v2
+name: fluid-databackup
+description: A Helm chart for Fluid to backup data
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+appVersion: 0.1.0

charts/fluid-databackup/README.md

@@ -0,0 +1,41 @@
+# fluid-databackup
+
+## Prerequisite
+- Dataset deployed
+- Alluxio Runtime deployed
+- Dataset mountPoint mounted
+- Dataset-related PV, PVC created
+
+## Install
+1. Install fluid-databackup
+
+```shell script
+helm install charts/fluid-databackup
+```
+
+You will see something like this:
+```
+helm install charts/fluid-databackup
+NAME: test
+LAST DEPLOYED: Fri Jan 15 09:18:02 2021
+NAMESPACE: default
+STATUS: deployed
+REVISION: 1
+TEST SUITE: None
+```
+
+one datbackup pod will be launched. You will see one pod running on the node:
+```shell script
+kubectl get pods <datasetname>-databackup-pod -o wide
+```
+
+Once the pod completes, you can check filed backuped:
+```shell script
+$ ls
+hbase-default.yaml  metadata-backup-hbase-default.gz
+```
+
+## Uninstall
+```
+helm del test
+```