Make alluxio image configurable (fluid-cloudnative#380)

* Make alluxio image configurable

* Make alluxio image configurable, to #31235919

* Make alluxio image configurable, to #31235919

* Make alluxio image configurable, to #31235919

charts/fluid/fluid/VERSION

@@ -26,3 +26,8 @@ Branch: https://github.com/Alluxio/alluxio/commits/branch-2.3-fuse
 1109:
 Version: https://github.com/Alluxio/alluxio/commit/238b7eb3ec58270aabf7004ac01cc89e6034b93e
 Branch: https://github.com/Alluxio/alluxio/commits/branch-2.3-fuse
+
+
+1117:
+Version: https://github.com/Alluxio/alluxio/commit/2c412267e0e749ced63262d53043cf753cc11927
+Branch: https://github.com/Alluxio/alluxio/commits/branch-2.3-fuse

charts/fluid/fluid/templates/controller/alluxioruntime_controller.yaml

@@ -32,6 +32,14 @@ spec:
           - name: ALLUXIO_INIT_IMAGE_ENV
             value: {{ .Values.runtime.alluxio.init.image | quote }}
           {{- end }}
+          {{- if .Values.runtime.alluxio.runtime.image }}
+          - name: ALLUXIO_RUNTIME_IMAGE_ENV
+            value: {{ .Values.runtime.alluxio.runtime.image | quote }}
+          {{- end }}
+          {{- if .Values.runtime.alluxio.fuse.image }}
+          - name: ALLUXIO_FUSE_IMAGE_ENV
+            value: {{ .Values.runtime.alluxio.fuse.image | quote }}
+          {{- end }}
           {{- if .Values.runtime.mountRoot }}
           - name: MOUNT_ROOT
             value: {{ .Values.runtime.mountRoot | quote }}

charts/fluid/fluid/templates/controller/dataset_controller.yaml

@@ -33,9 +33,9 @@ spec:
         args:
           - --development=false
         env:
-          {{- if .Values.runtime.alluxio.init.image }}
-          - name: ALLUXIO_INIT_IMAGE_ENV
-            value: {{ .Values.runtime.alluxio.init.image | quote }}
+          {{- if .Values.runtime.alluxio.dataload.image }}
+          - name: ALLUXIO_DATALOAD_IMAGE_ENV
+            value: {{ .Values.runtime.alluxio.dataload.image | quote }}
           {{- end }}
         ports:
         - containerPort: 8080

charts/fluid/fluid/values.yaml

@@ -4,18 +4,24 @@
 
 dataset:
   controller:
-    image: registry.cn-hangzhou.aliyuncs.com/fluid/dataset-controller:v0.5.0-07b9b1d
+    image: registry.cn-hangzhou.aliyuncs.com/fluid/dataset-controller:v0.5.0-a5ca4ff
 
 csi:
   registrar:
     image: registry.cn-hangzhou.aliyuncs.com/acs/csi-node-driver-registrar:v1.2.0
   plugins:
-    image: registry.cn-hangzhou.aliyuncs.com/fluid/fluid-csi:v0.5.0-07b9b1d
+    image: registry.cn-hangzhou.aliyuncs.com/fluid/fluid-csi:v0.5.0-a5ca4ff
 
 runtime:
   mountRoot: /alluxio-mnt
   alluxio:
     init:
-      image: registry.cn-hangzhou.aliyuncs.com/fluid/init-users:v0.4.0-a8ba7c9
+      image: registry.cn-hangzhou.aliyuncs.com/fluid/init-users:v0.5.0-a5ca4ff
     controller:
-      image: registry.cn-hangzhou.aliyuncs.com/fluid/alluxioruntime-controller:v0.5.0-07b9b1d
+      image: registry.cn-hangzhou.aliyuncs.com/fluid/alluxioruntime-controller:v0.5.0-a5ca4ff
+    runtime:
+      image: registry.cn-huhehaote.aliyuncs.com/alluxio/alluxio:2.3.0-SNAPSHOT-2c41226
+    fuse:
+      image: registry.cn-huhehaote.aliyuncs.com/alluxio/alluxio-fuse:2.3.0-SNAPSHOT-2c41226
+    dataload:
+      image: registry.cn-huhehaote.aliyuncs.com/alluxio/alluxio:2.3.0-SNAPSHOT-2c41226

docs/zh/dev/how_to_develop.md

@@ -19,6 +19,13 @@ Fluid需要使用`make`命令进行项目构建，使用以下命令安装`make`
 
 ## 编译、运行和调试
 
+### 安装`controller-gen`
+
+```shell
+$ go get sigs.k8s.io/controller-tools/cmd/[email protected]
+$ cp $GOPATH/bin/controller-gen /usr/local/bin/
+```
+
 ### 获取Fluid源码
 
 ```shell

pkg/common/alluxio.go

@@ -0,0 +1,24 @@
+package common
+
+// Runtime for Alluxio
+const (
+	ALLUXIO_RUNTIME = "alluxio"
+
+	ALLUXIO_NAMESPACE = "alluxio-system"
+
+	ALLUXIO_CHART = ALLUXIO_RUNTIME
+
+	ALLUXIO_DATA_LOADER_IMAGE_ENV = "AlluxioDataLoaderImage"
+
+	// DEFAULT_ALLUXIO_DATA_LOADER_IMAGE = "registry.cn-huhehaote.aliyuncs.com/alluxio/alluxio-data-loader:v2.2.0"
+
+	ALLUXIO_INIT_IMAGE_ENV = "ALLUXIO_INIT_IMAGE_ENV"
+
+	ALLUXIO_RUNTIME_IMAGE_ENV = "ALLUXIO_RUNTIME_IMAGE_ENV"
+
+	ALLUXIO_FUSE_IMAGE_ENV = "ALLUXIO_FUSE_IMAGE_ENV"
+
+	ALLUXIO_DATALOAD_IMAGE_ENV = "ALLUXIO_DATALOAD_IMAGE_ENV"
+
+	DEFAULT_ALLUXIO_INIT_IMAGE = "registry.cn-hangzhou.aliyuncs.com/fluid/init-users:v0.3.0-1467caa"
+)

pkg/common/constants.go

@@ -47,23 +47,6 @@ const (
 	DataLoadJobComplete = "DataLoadJobComplete"
 )
 
-// Runtime for Alluxio
-const (
-	ALLUXIO_RUNTIME = "alluxio"
-
-	ALLUXIO_NAMESPACE = "alluxio-system"
-
-	ALLUXIO_CHART = ALLUXIO_RUNTIME
-
-	ALLUXIO_DATA_LOADER_IMAGE_ENV = "AlluxioDataLoaderImage"
-
-	DEFAULT_ALLUXIO_DATA_LOADER_IMAGE = "registry.cn-huhehaote.aliyuncs.com/alluxio/alluxio-data-loader:v2.2.0"
-
-	ALLUXIO_INIT_IMAGE_ENV = "ALLUXIO_INIT_IMAGE_ENV"
-
-	DEFAULT_ALLUXIO_INIT_IMAGE = "registry.cn-hangzhou.aliyuncs.com/fluid/init-users:v0.3.0-1467caa"
-)
-
 type CacheStoreType string
 
 const (

pkg/controllers/v1alpha1/dataload/implement.go

@@ -14,6 +14,7 @@ import (
 	cdataload "github.com/fluid-cloudnative/fluid/pkg/dataload"
 	"github.com/fluid-cloudnative/fluid/pkg/ddc/alluxio/operations"
 	"github.com/fluid-cloudnative/fluid/pkg/utils"
+	"github.com/fluid-cloudnative/fluid/pkg/utils/docker"
 	"github.com/fluid-cloudnative/fluid/pkg/utils/helm"
 	"github.com/go-logr/logr"
 	"gopkg.in/yaml.v2"
@@ -356,11 +357,16 @@ func (r *DataLoadReconcilerImplement) generateDataLoadValueFile(dataload v1alpha
 		return "", err
 	}
 
+	imageName := "registry.cn-huhehaote.aliyuncs.com/alluxio/alluxio"
+	imageTag := "2.3.0-SNAPSHOT-238b7eb"
+	imageName, imageTag = docker.GetImageRepoTagFromEnv(common.ALLUXIO_DATALOAD_IMAGE_ENV, imageName, imageTag)
+	image := fmt.Sprintf("%s:%s", imageName, imageTag)
+
 	dataloadInfo := cdataload.DataLoadInfo{
 		BackoffLimit:  3,
 		TargetDataset: dataload.Spec.Dataset.Name,
 		LoadMetadata:  dataload.Spec.LoadMetadata,
-		Image:         "registry.cn-huhehaote.aliyuncs.com/alluxio/alluxio:2.3.0-SNAPSHOT-238b7eb",
+		Image:         image,
 	}
 
 	targetPaths := []cdataload.TargetPath{}

pkg/ddc/alluxio/transform.go

@@ -82,12 +82,13 @@ func (e *AlluxioEngine) transform(runtime *datav1alpha1.AlluxioRuntime) (value *
 // 2. Transform the common part
 func (e *AlluxioEngine) transformCommonPart(runtime *datav1alpha1.AlluxioRuntime, value *Alluxio) (err error) {
 
-	value.Image = "registry.cn-huhehaote.aliyuncs.com/alluxio/alluxio"
+	value.Image, value.ImageTag = e.parseRuntimeImage()
+	// value.Image = "registry.cn-huhehaote.aliyuncs.com/alluxio/alluxio"
 	if runtime.Spec.AlluxioVersion.Image != "" {
 		value.Image = runtime.Spec.AlluxioVersion.Image
 	}
 
-	value.ImageTag = "2.3.0-SNAPSHOT-238b7eb"
+	// value.ImageTag = "2.3.0-SNAPSHOT-238b7eb"
 	if runtime.Spec.AlluxioVersion.ImageTag != "" {
 		value.ImageTag = runtime.Spec.AlluxioVersion.ImageTag
 	}

pkg/ddc/alluxio/transform_fuse.go

@@ -26,12 +26,11 @@ import (
 func (e *AlluxioEngine) transformFuse(runtime *datav1alpha1.AlluxioRuntime, dataset *datav1alpha1.Dataset, value *Alluxio) (err error) {
 	value.Fuse = Fuse{}
 
-	value.Fuse.Image = "registry.cn-huhehaote.aliyuncs.com/alluxio/alluxio-fuse"
+	value.Fuse.Image, value.Fuse.ImageTag = e.parseFuseImage()
 	if runtime.Spec.Fuse.Image != "" {
 		value.Fuse.Image = runtime.Spec.Fuse.Image
 	}
 
-	value.Fuse.ImageTag = "2.3.0-SNAPSHOT-238b7eb"
 	if runtime.Spec.Fuse.ImageTag != "" {
 		value.Fuse.ImageTag = runtime.Spec.Fuse.ImageTag
 	}

pkg/ddc/alluxio/transform_optimization.go

@@ -112,7 +112,7 @@ func (e *AlluxioEngine) optimizeDefaultForMaster(runtime *datav1alpha1.AlluxioRu
 
 	if len(value.Master.JvmOptions) == 0 {
 		value.Master.JvmOptions = []string{
-			"-Xmx6G",
+			"-Xmx16G",
 			"-XX:+UnlockExperimentalVMOptions",
 		}
 	}

pkg/ddc/alluxio/utils.go

@@ -27,7 +27,9 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	datav1alpha1 "github.com/fluid-cloudnative/fluid/api/v1alpha1"
+	"github.com/fluid-cloudnative/fluid/pkg/common"
 	"github.com/fluid-cloudnative/fluid/pkg/utils"
+	"github.com/fluid-cloudnative/fluid/pkg/utils/docker"
 	"github.com/fluid-cloudnative/fluid/pkg/utils/kubeclient"
 )
 
@@ -271,3 +273,53 @@ func (e *AlluxioEngine) getAvaliablePort() (allocatedPorts []int, err error) {
 
 	return
 }
+
+func (e *AlluxioEngine) parseRuntimeImage() (image, tag string) {
+	var (
+		defaultImage = "registry.cn-huhehaote.aliyuncs.com/alluxio/alluxio"
+		defaultTag   = "2.3.0-SNAPSHOT-2c41226"
+	)
+
+	image, tag = docker.GetImageRepoTagFromEnv(common.ALLUXIO_RUNTIME_IMAGE_ENV, defaultImage, defaultTag)
+	e.Log.Info("Set image", "image", image, "tag", tag)
+
+	// if value, existed := os.LookupEnv(common.ALLUXIO_RUNTIME_IMAGE_ENV); existed {
+	// 	if matched, err := regexp.MatchString("^\\S+:\\S+$", value); err == nil && matched {
+	// 		k, v := docker.ParseDockerImage(repos)
+	// 		if len(k) ==
+
+	// 	} else {
+	// 		e.Log.Info("Failed to parse the ALLUXIO_RUNTIME_IMAGE_ENV", "ALLUXIO_RUNTIME_IMAGE_ENV", value, "error", err)
+	// 	}
+	// 	e.Log.Info("Get INIT_IMAGE from Env", common.ALLUXIO_RUNTIME_IMAGE_ENV, value)
+	// } else {
+	// 	e.Log.Info("Use Default ALLUXIO_RUNTIME_IMAGE_ENV", "ALLUXIO_RUNTIME_IMAGE_ENV", common.DEFAULT_ALLUXIO_INIT_IMAGE)
+	// }
+
+	return
+}
+
+func (e *AlluxioEngine) parseFuseImage() (image, tag string) {
+	var (
+		defaultImage = "registry.cn-huhehaote.aliyuncs.com/alluxio/alluxio-fuse"
+		defaultTag   = "2.3.0-SNAPSHOT-2c41226"
+	)
+
+	image, tag = docker.GetImageRepoTagFromEnv(common.ALLUXIO_FUSE_IMAGE_ENV, defaultImage, defaultTag)
+	e.Log.Info("Set image", "image", image, "tag", tag)
+
+	// if value, existed := os.LookupEnv(common.ALLUXIO_RUNTIME_IMAGE_ENV); existed {
+	// 	if matched, err := regexp.MatchString("^\\S+:\\S+$", value); err == nil && matched {
+	// 		k, v := docker.ParseDockerImage(repos)
+	// 		if len(k) ==
+
+	// 	} else {
+	// 		e.Log.Info("Failed to parse the ALLUXIO_RUNTIME_IMAGE_ENV", "ALLUXIO_RUNTIME_IMAGE_ENV", value, "error", err)
+	// 	}
+	// 	e.Log.Info("Get INIT_IMAGE from Env", common.ALLUXIO_RUNTIME_IMAGE_ENV, value)
+	// } else {
+	// 	e.Log.Info("Use Default ALLUXIO_RUNTIME_IMAGE_ENV", "ALLUXIO_RUNTIME_IMAGE_ENV", common.DEFAULT_ALLUXIO_INIT_IMAGE)
+	// }
+
+	return
+}

pkg/utils/docker/image.go

@@ -0,0 +1,44 @@
+package docker
+
+import (
+	"os"
+	"regexp"
+	"strings"
+)
+
+// ParseDockerImage extracts repo and tag from image. An empty string is returned if no tag is discovered.
+func ParseDockerImage(image string) (name string, tag string) {
+	matches := strings.Split(image, ":")
+	if len(matches) >= 2 {
+		name = matches[0]
+		tag = matches[1]
+	} else if len(matches) == 1 {
+		name = matches[0]
+		tag = "latest"
+		// return matches[0], "latest"
+	}
+	return
+}
+
+// GetImageRepoTagFromEnv parse the image and tag from environment varaibles, if it's not existed or
+func GetImageRepoTagFromEnv(envName, defaultImage string, defaultTag string) (image, tag string) {
+
+	image = defaultImage
+	tag = defaultTag
+
+	if value, existed := os.LookupEnv(envName); existed {
+		if matched, err := regexp.MatchString("^\\S+:\\S+$", value); err == nil && matched {
+			k, v := ParseDockerImage(value)
+			if len(k) > 0 {
+				image = k
+
+			}
+			if len(v) > 0 {
+				tag = v
+
+			}
+		}
+	}
+
+	return
+}

pkg/utils/docker/image_test.go

@@ -0,0 +1,28 @@
+package docker
+
+import (
+	"testing"
+)
+
+func TestParseDockerImage(t *testing.T) {
+	var testCases = []struct {
+		input string
+		image string
+		tag   string
+	}{
+		{"test:abc", "test", "abc"},
+		{"test", "test", "latest"},
+	}
+	for _, tc := range testCases {
+		image, tag := ParseDockerImage(tc.input)
+		if tc.image != image {
+			t.Errorf("expected image %#v, got %#v",
+				tc.image, image)
+		}
+
+		if tc.tag != tag {
+			t.Errorf("expected tag %#v, got %#v",
+				tc.tag, tag)
+		}
+	}
+}

pkg/utils/helm/utils.go

@@ -135,7 +135,6 @@ func DeleteRelease(name, namespace string) error {
 	return err
 }
 
-
 // ListReleases return an array with all releases' names in a given namespace
 func ListReleases(namespace string) (releases []string, err error) {
 	releases = []string{}