Add dynamic manifest generation

Adds go templated controller manifest

Adds make targets for template generation

Removes hard coded version values

Enables template generation and publishing in CI

.gitignore

@@ -1,3 +1,4 @@
 .idea
 _out/*
 bin/*
+manifests/generated/*

.travis.yml

@@ -12,7 +12,7 @@ services:
 
 env:
   global:
-  - CHANGE_MINIKUBE_NONE_USER=true K8S_VER=1.9.0 MK_VER=0.25.2 K6T_VER=0.4.0 SRC="http://www.tinycorelinux.net/9.x/x86/release/Core-current.iso"
+  - DOCKER_TAG=$TRAVIS_TAG CHANGE_MINIKUBE_NONE_USER=true K8S_VER=1.9.0 MK_VER=0.25.2 K6T_VER=0.4.0 SRC="http://www.tinycorelinux.net/9.x/x86/release/Core-current.iso"
   - secure: RWxtzwVEpMvaJlPUuRo+m6Kv9kJJFID7E0tdbE57d1XJN5nEq9ryxsTu8GIVHDPKIoUs0zqt4MtWi+4Wwb+21B2IoXJiHtc1YUex3L/6h64ztgH0iBjd/2Kv2piN4XVY9eYOUcu0PsSU0+Fv785rNfy2/LOi+3Bgs5d20AU9SZs2QDwKBlgWBXeJx7/aqLeytTUUX4tshBfQsxZkhYfVJgNcA1oUGtGS+zjeYVbmveLYQriOdQsfoWcfSZGumRSVH+jH7YCC2MXvIOfL131aOYm6CS0DlJTR3LDuBtI+9tP5C7eF+4nPO/kkIZzpz9RwkOZK2Ejn6D//unHgbyHqzoU4n4YZxw2DUonwXxCYieQ2nEZI5CQeoKXwV26ufdTvFGUYfYMUUnyk8phlP13eZEIoHNFJR2yeCSVSC6p82kpO0Drl+HJpnpo86IePwlcAp6tOekcGiDuM8i8bRmgPtWQzNxsE34OZ7v1j3S2nCrOgO8S9AE262nM3FsIGXCl2KTtKUDOSx0/1xx27tZhsqGDMOksdfdE6msFTJ1cEU53KoJfyyr1UUAtHfIpGLYu67C3tveBB4pcgvZ8SyiP50O8SQkQWX1ovJPj4kZYEx3PKAvTiwvDcCXU21uJH60Qa3lugiNfabmoX4mxjkRgiUswkW4P/Tc9avVa6g6EKnqA=
 
 notifications:
@@ -24,6 +24,7 @@ notifications:
 
 install:
   - make docker
+  - make manifests
 
 before_script:
 - curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/v$K8S_VER/bin/linux/amd64/kubectl
@@ -41,7 +42,7 @@ before_script:
 - k_wait_all_running pods
 
 script:
-- kubectl apply -f manifests/controller/cdi-controller-deployment.yaml
+- kubectl apply -f manifests/generated/cdi-controller.yaml
 - kubectl apply -f manifests/example/endpoint-secret.yaml
 - "sed \"s#cdi.kubevirt.io/storage.import.endpoint:.*#cdi.kubevirt.io/storage.import.endpoint: \"$SRC#\"\" manifests/example/golden-pvc.yaml | kubectl apply -f -"
 - k_wait_all_running pods
@@ -54,16 +55,15 @@ before_deploy:
 
 deploy:
   # Release :latest images
-  # TODO disabled until manifest templator is merged
-#- provider: script
-#  script: make publish
-#  skip_cleanup: true
-#  on:
-#    branch: master
-#    repo: kubevirt/containerized-data-importer
-  # Release versioned images
 - provider: script
-  script: make publish DOCKER_TAG=$TRAVIS_TAG
+  script: make publish
+  skip_cleanup: true
+  on:
+    branch: master
+    repo: kubevirt/containerized-data-importer
+   # Release versioned images
+- provider: script
+  script: make publish
   branches:
       only: /v[0-9]+\.[0-9]+\.[0-9]+(-alpha\.[0-9]+){0,1}/
   on:
@@ -82,5 +82,5 @@ deploy:
   file_glob: true
   file:
   - bin/*
-  - manifests/controller/cdi-controller-deployment.yaml
+  - manifests/generated/cdi-controller-deployment.yaml
   - manifests/example/*

Makefile

@@ -19,22 +19,23 @@ DOCKER=1
 		test test-functional test-unit \
 		publish \
 		vet \
-		format
+		format \
+		manifests
 
 all: docker
 
 clean:
 ifeq (${DOCKER}, 1)
-	./hack/build/in-docker "./hack/build/build-go.sh clean && rm -rf bin/* _out/*"
+	./hack/build/in-docker "./hack/build/build-go.sh clean; rm -rf bin/* _out/* manifests/generated/*"
 else
-	"./hack/build/build-go.sh clean && rm -rf bin/* _out/*"
+	./hack/build/build-go.sh clean; rm -rf bin/* _out/* manifests/generated/*
 endif
 
 build:
 ifeq (${DOCKER}, 1)
-	./hack/build/in-docker "./hack/build/build-go.sh clean && ./hack/build/build-go.sh build ${WHAT} && ./hack/build/build-copy-artifacts.sh ${WHAT}"
+	./hack/build/in-docker "./hack/build/build-go.sh clean && ./hack/build/build-go.sh build ${WHAT} && DOCKER_REPO=${DOCKER_REPO} DOCKER_TAG=${DOCKER_TAG} VERBOSITY=${VERBOSITY} PULL_POLICY=${PULL_POLICY} ./hack/build/build-manifests.sh ${WHAT} && ./hack/build/build-copy-artifacts.sh ${WHAT}"
 else
-	./hack/build/build-go.sh clean && ./hack/build/build-go.sh build ${WHAT} && ./hack/build/build-copy-artifacts.sh ${WHAT}
+	./hack/build/build-go.sh clean && ./hack/build/build-go.sh build ${WHAT} && ./hack/build/build-manifests.sh ${WHAT} && ./hack/build/build-copy-artifacts.sh ${WHAT}
 endif
 
 build-controller: WHAT = cmd/cdi-controller
@@ -92,4 +93,9 @@ else
 	./hack/build/format.sh
 endif
 
-
+manifests:
+ifeq (${DOCKER}, 1)
+	./hack/build/in-docker "DOCKER_REPO=${DOCKER_REPO} DOCKER_TAG=${DOCKER_TAG} VERBOSITY=${VERBOSITY} PULL_POLICY=${PULL_POLICY} ./hack/build/build-manifests.sh"
+else
+	./hack/build/build-manifests.sh
+endif

README.md

@@ -59,23 +59,34 @@ Supported file formats are:
 - An HTTP or S3 file server hosting VM images
 - An optional "golden" namespace acting as the image registry. The `default` namespace is fine for tire kicking.
 
-### Either clone this repo or download the necessary manifests directly:
+### Download CDI
 
-`$ git clone https://kubevirt.io/containerized-data-importer.git`
+`$ go get kubevirt.io/containerized-data-importer`
 
 *Or*
 
+`$ git clone https://kubevirt.io/containerized-data-importer.git`
+
+*Or download only the yamls:*
+
 ```shell
 $ mkdir cdi-manifests && cd cdi-manifests
 $ wget https://raw.githubusercontent.com/kubevirt/containerized-data-importer/kubevirt-centric-readme/manifests/example/golden-pvc.yaml
 $ wget https://raw.githubusercontent.com/kubevirt/containerized-data-importer/kubevirt-centric-readme/manifests/example/endpoint-secret.yaml
-$ wget https://raw.githubusercontent.com/kubevirt/containerized-data-importer/kubevirt-centric-readme/manifests/controller/controller/cdi-controller-deployment.yaml
 ```
 
 ### Run the CDI Controller
 
-Deploying the CDI controller is straight forward. Choose the namespace where the controller will run and ensure that this namespace has cluster-wide permission to watch all PVCs and pods.
-In this document the _default_ namespace is used, but in a production setup a namespace that is inaccessible to regular users should be used instead. See [Protecting the Golden Image Namespace](#protecting-the-golden-image-namespace) on creating a secure CDI controller namespace.
+Deploying the CDI controller is straight forward. In this document the _default_ namespace is used, but in a production setup a [protected namespace](#protecting-the-golden-image-namespace) that is inaccessible to regular users should be used instead.
+
+1. Deploy the controller from the release manifest:
+
+```
+$ VERSION=<cdi version>
+$ kubectl create -f https://github.com/kubevirt/containerized-data-importer/releases/download/$VERSION/cdi-controller-deployment.yaml
+```
+
+**DEPRECATION NOTICE:** The below method will not be supported in releases following v1.1.0 as `manifests/controller/cdi-controller-deployment.yaml` will be removed.  See [Make Targets (manifests)](./hack/README.md#make-targets) for generating manifests locally.
 
 `$ kubectl -n default create -f https://raw.githubusercontent.com/kubevirt/containerized-data-importer/master/manifests/cdi-controller-deployment.yaml`
 

cmd/cdi-controller/controller.go

@@ -29,11 +29,11 @@ var (
 	verbose       string
 )
 
-// The optional importer and colner images are obtained here along with the supported flags.
+// The importer and cloner images are obtained here along with the supported flags. IMPORTER_IMAGE and CLONER_IMAGE
+// are required by the controller and will cause it to fail if not defined.
 // Note: kubeconfig hierarchy is 1) -kubeconfig flag, 2) $KUBECONFIG exported var. If neither is
 //   specified we do an in-cluster config. For testing it's easiest to export KUBECONFIG.
 func init() {
-	// optional, importer image.  If not provided, uses IMPORTER_DEFAULT_IMAGE
 	const IMPORTER_IMAGE = "IMPORTER_IMAGE"
 	const CLONER_IMAGE = "CLONER_IMAGE"
 
@@ -45,12 +45,12 @@ func init() {
 	// env variables
 	importerImage = os.Getenv(IMPORTER_IMAGE)
 	if importerImage == "" {
-		importerImage = IMPORTER_DEFAULT_IMAGE
+		glog.Fatalf("Environment Variable %q undefined\n", IMPORTER_IMAGE)
 	}
 
 	clonerImage = os.Getenv(CLONER_IMAGE)
 	if clonerImage == "" {
-		clonerImage = CLONER_DEFAULT_IMAGE
+		glog.Fatalf("Environment Variable %q undefined\n", CLONER_IMAGE)
 	}
 
 	pullPolicy = DEFAULT_PULL_POLICY

hack/README.md

@@ -19,35 +19,41 @@ The standard workflow is performed inside a helper container to normalize the bu
 
 `$ make all` executes the full workflow.  For granular control of the workflow, several Make targets are defined:
 
-**Make Targets**
+#### Make Targets
 
 - `all`: cleans up previous build artifacts, compiles all CDI packages and builds containers
 - `clean`: cleans up previous build artifacts
-- `build`: compile all CDI binary artifacts
+- `build`: compile all CDI binary artifacts and generate controller manifest
     - `build-controller`: compile cdi-controller binary
     - `build-importer`: compile cdi-importer binary
+    - No `build-cloner` target exists as the code is written in bash
 - `test`: execute all tests
     - `test-unit`: execute all tests under `./pkg`
     - `test-functional`: execute all tests under `./test`
 - `docker`: compile all binaries and build all containerized
     - `docker-controller`: compile cdi-controller and build cdi-controller image
     - `docker-importer`: compile cdi-importer and build cdi-importer image
     - `docker-cloner`: build the cdi-cloner image (cloner is driven by a shell script, not a binary)
+- `manifests`: Generate a cdi-controller manifest in `manifests/generated/`.  Accepts [make variables](#make-variables) DOCKER_TAG, DOCKER_REPO, VERBOSITY, and PULL_POLICY
 - `push`: compiles, builds, and pushes to the repo passed in `DOCKER_REPO=<my repo>`
-    - push-controller: compile, build, and push cdi-controller
-    - push-importer: compile, build, and push cdi-importer
-    - push-cloner: compile, build, and push cdi-cloner
+    - `push-controller`: compile, build, and push cdi-controller
+    - `push-importer`: compile, build, and push cdi-importer
+    - `push-cloner`: compile, build, and push cdi-cloner
 - `vet`: lint all CDI packages
 - `format`: Execute `shfmt`, `goimports`, and `go vet` on all CDI packages.  Writes back to the source files.
 - `publish`: CI ONLY - this recipe is not intended for use by developers
 
-**Make Variables**
+#### Make Variables
 
 Several variables are provided to alter the targets of the above `Makefile` recipes.
 
-- `WHAT`:  The path from the repo root to a target directory (e.g. `make test WHAT=pkg/importer`)
-- `DOCKER_REPO`: (default: kubevirt) The docker repo to which images are pushed.  Used with `make push` to upload images to non-kubevirt repos.
-- `DOCKER_TAG`: (default: latest) The value with which new images are tagged.
+These may be passed to a target as `$ make VARIABLE=value target`
+
+- `WHAT`:  The path from the repository root to a target directory (e.g. `make test WHAT=pkg/importer`)
+- `DOCKER_REPO`: (default: kubevirt) Set repo globally for image and manifest creation
+- `DOCKER_TAG`: (default: latest) Set global version tags for image and manifest creation
+- `VERBOSITY`: (default: 1) Set global log level verbosity
+- `PULL_POLICY`: (default: IfNotPresent) Set global CDI pull policy
 
 ### Submit PRs
 

hack/build/build-copy-artifacts.sh

@@ -22,18 +22,14 @@ source "${script_dir}"/config.sh
 
 targets="${@:-${DOCKER_IMAGES}}"
 
-if [ -z "${targets}" ]; then
-    targets=${DOCKER_IMAGES}
-fi
-
 for tgt in ${targets}; do
-    BIN_NAME="$(basename ${tgt})"
+    bin_name="$(basename ${tgt})"
+    dest_dir="${CMD_OUT_DIR}/${bin_name}/"
     # Cloner has no build artifact, copy script.sh as well
-    if [[ "${BIN_NAME}" == "${CLONER}" ]]; then
-        mkdir -p "${CMD_OUT_DIR}/${BIN_NAME}/"
-        cp -f "${CDI_DIR}/cmd/${BIN_NAME}/script.sh" "${CMD_OUT_DIR}/${BIN_NAME}/"
+    if [[ "${bin_name}" == "${CLONER}" ]]; then
+        mkdir -p "${CMD_OUT_DIR}/${bin_name}/"
+        cp -f "${CDI_DIR}/cmd/${bin_name}/script.sh" "${dest_dir}"
     fi
     # Copy respective docker files to the directory of the build artifact
-    cp -f "${BUILD_DIR}/docker/${BIN_NAME}/Dockerfile" "${CMD_OUT_DIR}/${BIN_NAME}/"
-
+    cp -f "${BUILD_DIR}/docker/${bin_name}/Dockerfile" "${dest_dir}"
 done

hack/build/build-docker.sh

@@ -43,7 +43,7 @@ for tgt in ${targets}; do
         docker "${docker_opt}" "${IMAGE}"
     elif [ "${docker_opt}" == "publish" ]; then
         if [ -z "${TRAVIS}" ]; then
-            echo "Publishing releases should only be performed by the CI."
+            echo "Publishing releases should only be performed by the CI. "
             exit 1
         fi
         docker "${docker_opt}" ${IMAGE}

hack/build/build-manifests.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
+set -euo pipefail
+
+script_dir="$(readlink -f $(dirname $0))"
+source "${script_dir}"/common.sh
+source "${script_dir}"/config.sh
+
+templates="$@"
+if [ -z "${templates}" ]; then
+    templates="$(find "${MANIFEST_TEMPLATE_DIR}" -name *.in -type f)"
+fi
+
+generator="${BIN_DIR}/manifest-generator"
+
+(cd "${CDI_DIR}/tools/manifest-generator/" && go build -o "${generator}" ./...)
+
+mkdir -p "${MANIFEST_GENERATED_DIR}/"
+
+for tmpl in ${templates}; do
+    tmpl=$(readlink -f "${tmpl}")
+    outFile=$(basename -s .in "${tmpl}")
+    rm -rf "${MANIFEST_GENERATED_DIR}/${outFile}"
+    (${generator} -template="${tmpl}" \
+        -docker-repo="${DOCKER_REPO}" \
+        -docker-tag="${DOCKER_TAG}" \
+        -verbosity="${VERBOSITY}" \
+        -pull-policy="${PULL_POLICY}"
+    ) 1>"${MANIFEST_GENERATED_DIR}/${outFile}"
+done

hack/build/common.sh

@@ -17,3 +17,5 @@ BIN_DIR=${CDI_DIR}/bin
 OUT_DIR=${CDI_DIR}/_out
 CMD_OUT_DIR=${OUT_DIR}/cmd
 BUILD_DIR=${CDI_DIR}/hack/build
+MANIFEST_TEMPLATE_DIR=${CDI_DIR}/manifests/templates
+MANIFEST_GENERATED_DIR=${CDI_DIR}/manifests/generated

hack/build/config.sh

@@ -26,6 +26,8 @@ CLONER_MAIN="cmd/${CLONER}"
 DOCKER_IMAGES="cmd/${CONTROLLER} cmd/${IMPORTER} cmd/${CLONER}"
 DOCKER_REPO=${DOCKER_REPO:-kubevirt}
 DOCKER_TAG=${DOCKER_TAG:-latest}
+VERBOSITY=${VERBOSITY:-1}
+PULL_POLICY=${PULL_POLICY:-IfNotPresent}
 
 function allPkgs {
     ret=$(sed "s,kubevirt.io/containerized-data-importer,${CDI_DIR},g" <(go list ./... | grep -v -E "vendor|pkg/client" | sort -u ))

manifests/controller/cdi-controller-deployment.yaml

@@ -1,3 +1,5 @@
+# DEPRECATED - this file will be removed in the next release and officially replaced by the generated manifest
+---
 kind: ServiceAccount
 apiVersion: v1
 metadata:
@@ -68,6 +70,13 @@ spec:
         image: kubevirt/cdi-controller:v1.0.0
         imagePullPolicy: IfNotPresent
         args: ["-v=1"] # default verbosity; change to 2 or 3 for more detailed logging
+        env:
+        - name: IMPORTER_IMAGE
+          value: kubevirt/cdi-importer:v1.0.0
+        - name: CLONER_IMAGE
+          value: kubevirt/cdi-cloner:v1.0.0
+        - name: PULL_POLICY
+          value: IfNotPresent
 ---
 apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition