diff --git a/.travis.yml b/.travis.yml index 49e05127497f..64e145e35368 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,6 +24,7 @@ script: - make apidocs - make client-python - make manifests DOCKER_PREFIX="docker.io/kubevirt" DOCKER_TAG=$TRAVIS_TAG # falls back to latest if not on a tag +- make olm-verify deploy: - provider: script diff --git a/Makefile b/Makefile index 6aee53189b88..28845547732b 100644 --- a/Makefile +++ b/Makefile @@ -90,7 +90,7 @@ verify-build: hack/verify-build.sh manifests: - hack/dockerized "DOCKER_PREFIX=${DOCKER_PREFIX} DOCKER_TAG=${DOCKER_TAG} IMAGE_PULL_POLICY=${IMAGE_PULL_POLICY} VERBOSITY=${VERBOSITY} ./hack/build-manifests.sh" + hack/dockerized "CSV_VERSION=${CSV_VERSION} DOCKER_PREFIX=${DOCKER_PREFIX} DOCKER_TAG=${DOCKER_TAG} IMAGE_PULL_POLICY=${IMAGE_PULL_POLICY} VERBOSITY=${VERBOSITY} ./hack/build-manifests.sh" .release-functest: make functest > .release-functest 2>&1 @@ -121,6 +121,12 @@ builder-build: builder-publish: ./hack/builder/publish.sh +olm-verify: + hack/dockerized "./hack/olm.sh verify" + +olm-push: + hack/dockerized "CSV_VERSION=${CSV_VERSION} QUAY_USERNAME=${QUAY_USERNAME} QUAY_PASSWORD=${QUAY_PASSWORD} ./hack/olm.sh push" + .PHONY: \ go-build \ go-test \ @@ -145,4 +151,6 @@ builder-publish: cluster-down \ cluster-clean \ cluster-deploy \ - cluster-sync + cluster-sync \ + olm-verify \ + olm-push diff --git a/assets/kubevirt_logo.png b/assets/kubevirt_logo.png new file mode 100644 index 000000000000..9e15e7ffdd18 Binary files /dev/null and b/assets/kubevirt_logo.png differ diff --git a/docs/devel/olm-integration.md b/docs/devel/olm-integration.md new file mode 100644 index 000000000000..18dc09017fb9 --- /dev/null +++ b/docs/devel/olm-integration.md @@ -0,0 +1,151 @@ +# KubeVirt's OLM and Operator Marketplace Integration + +## Introduction + +### Operator Lifecycle Manager (OLM) + +https://github.com/operator-framework/operator-lifecycle-manager + +OLM is the Operator Lifecycle Manager, which consists of 2 operators: + +#### OLM Operator + +Installs application operators based in information in ClusterServiceVersions + +CRDs: + +- ClusterServiceVersion (CSV): + contains application metadata: name, version, icon, required resources, installation, etc... + provided by developer, together with CRD declarations and package description. The latter declares channels and their CSV version. + installed by Catalog Operator + +- OperatorGroup: + declares on which namespaces OLM should operate + provided and installed by developer, or in UI + +#### Catalog Operator + +Prepares installations of operators by installing the application's CRDs and CSVs + +CRDs: + +- CatalogSource: + declares available packages + provided and installed by Marketplace Operator based on CatalogSourceConfig + +- Subscription: + declares which version of an operator to install (which channel from which source) + provided and installed by developer, or in UI + +- InstallPlan: + calculated list of resources to be created in order to automatically install/upgrade a CSV + created and insalled by the Catalog Operator, needs manual or automatic approval + +### Operator Marketplace + +https://github.com/operator-framework/operator-marketplace + +The Operator Marketplace has another operator + +CRDs: + +- OperatorSource: + declares where to find applications bundles (CSV + CRD + package) + provided and installed by developer, and/or already installed pointing to official repositories (community operators) + +- CatalogSourceConfig: + declares which packages to enable in the marketplace + created and deployed by marketplace operator + +## KubeVirt Manifests + +Our OLM / Marketplace manifest templates live in /manifests/release/olm. As for all manifests, you need to run +`make generate && make manifests` for getting their final version in the `_out/` directory. + +The bundle subdirectory contains: + - the ClusterServiceVersion manifest + - the CRD manifest + - the Package manifest: this contains the available distribution channels and their corresponding CSV name + These files are pushed to Quay (after they are processed with) + +Then we have: + - the OperatorSource manifest: this will be deployed to your cluster. + - a Subscription manifest: only needed when not created using the OKD console. + - a OperatorGroup manifest: can be created in the console, too?? + +Last but not least there is a preconditions manifest: if want to test the CSV manifest manually, without +OperatorSource and Subscription, you can deploy this manifest in order to satisfy all conditions, which are declared +in the CSV manifest, so that the OLM operator can deploy the KubeVirt operator. + +## Test a new version + +Note 1: We use a k8s cluster >= v1.11 for this. You might want to use a OKD cluster with OLM and Marketplace already installed. +Note 2: You need a Quay.io account + +- create manifests with your repository and version info, e.g.: + + TODO: actually use CSV_VERSION!!! + + `CSV_VERSION= DOCKER_PREFIX="docker.io/" DOCKER_TAG="" sh -c 'make generate && make manifests'` +- verify manifests: + `make olm-verify` +- push images: + `DOCKER_PREFIX="index.docker.io/" DOCKER_TAG="" make bazel-push-images` +- push the operator bundle: + `CSV_VERSION= QUAY_USER= QUAY_PASSWORD= make olm-push` + Note: you need to update the CSV version (and so run `make manifests`) on every push! (or maybe delete an old version before pushing again?) + +- install OLM and Marketplace (see below) + +- install KubeVirt OperatorSource: + `cd _out/manifests/release/olm` + `kubectl apply -f kubevirt-operatorsource.yaml` +- check that a CatalogSourceConfig and a CatalogSource were created in the marketplace namespace +- WORKAROUND: the OKD console only shows operators from CatalogSources in the `olm` namespace. In order to get it there, + you need to edit the CatalogSourceConfig and change the targetNamespace from `marketplace` to `olm`. The new + CatalogSource should be created automatically. +- create the kubevirt namespace: + `kubectl create ns kubevirt` +- install the OperatorGroup for the new namespace: + `kubectl apply -f operatorgroup.yaml` +- create a Subscription: + `kubectl apply -f kubevirt-subsription.yaml` +- check that a InstallPlan was created +- check that the KubeVirt operator was installed +- install a KubeVirt CR + +Bonus: install the OKD Console: + +- we need cluster-admin permissions for the kube-system:default account: + `kubectl create clusterrolebinding defaultadmin --clusterrole cluster-admin --serviceaccount kube-system:default` +- in the OLM repository, run `./scripts/run_console_local.sh` +- open `localhost:9000` in a browser + +## Release a new version + +Travis cares for this on every release. + +## Installing OLM on Kubernetes + +- clone github.com/operator-framework/operator-lifecycle-manager +- `cd deploy/upstream/quickstart` +- `kubectl apply -f olm.yaml` +- if you get an error, try again, CRDs might have been too slow + +## Installing Marketplace on Kubernetes + +- clone github.com/operator-framework/operator-marketplace +- `cd deploy/upstream/manifests` +- `kubectl apply -f upstream/` +- if you get an error about rolebinding, repeat with `--validate=false` + +## Sources + +CSV description: https://github.com/operator-framework/operator-lifecycle-manager/blob/master/Documentation/design/building-your-csv.md +Publish bundles: https://github.com/operator-framework/community-operators/blob/master/docs/testing-operators.md +Install OLM: https://github.com/operator-framework/operator-lifecycle-manager/blob/master/Documentation/install/install.md +Install and use Marketplace: https://github.com/operator-framework/operator-marketplace + +## Important + +- the Quay repo name needs to match the package name (https://github.com/operator-framework/operator-marketplace/issues/122#issuecomment-470820491) \ No newline at end of file diff --git a/hack/build-manifests.sh b/hack/build-manifests.sh index 3d2650a3f4b7..6a361b49352f 100755 --- a/hack/build-manifests.sh +++ b/hack/build-manifests.sh @@ -23,6 +23,7 @@ source hack/common.sh source hack/config.sh manifest_docker_prefix=${manifest_docker_prefix-${docker_prefix}} +kubevirt_logo_path="assets/kubevirt_logo.png" rm -rf ${MANIFESTS_OUT_DIR} rm -rf ${MANIFEST_TEMPLATES_OUT_DIR} @@ -44,12 +45,18 @@ done # then process variables args=$(cd ${KUBEVIRT_DIR}/manifests && find . -type f -name "*.yaml.in.tmp") for arg in $args; do + + infile=${KUBEVIRT_DIR}/manifests/${arg} + final_out_dir=$(dirname ${MANIFESTS_OUT_DIR}/${arg}) - final_templates_out_dir=$(dirname ${MANIFEST_TEMPLATES_OUT_DIR}/${arg}) mkdir -p ${final_out_dir} + + final_templates_out_dir=$(dirname ${MANIFEST_TEMPLATES_OUT_DIR}/${arg}) mkdir -p ${final_templates_out_dir} + manifest=$(basename -s .in.tmp ${arg}) - infile=${KUBEVIRT_DIR}/manifests/${arg} + manifest="${manifest/VERSION/${csv_version}}" + outfile=${final_out_dir}/${manifest} template_outfile=${final_templates_out_dir}/${manifest}.j2 @@ -61,6 +68,8 @@ for arg in $args; do --container-tag=${docker_tag} \ --image-pull-policy=${image_pull_policy} \ --verbosity=${verbosity} \ + --csv-version=${csv_version} \ + --kubevirt-logo-path=${kubevirt_logo_path} \ --input-file=${infile} >${outfile} ${KUBEVIRT_DIR}/tools/manifest-templator/manifest-templator \ @@ -71,6 +80,8 @@ for arg in $args; do --container-tag="{{ docker_tag }}" \ --image-pull-policy="{{ image_pull_policy }}" \ --verbosity=${verbosity} \ + --csv-version=${csv_version} \ + --kubevirt-logo-path=${kubevirt_logo_path} \ --input-file=${infile} >${template_outfile} done diff --git a/hack/builder/Dockerfile b/hack/builder/Dockerfile index 3f7bc2e87ac5..4d53741e4fd3 100644 --- a/hack/builder/Dockerfile +++ b/hack/builder/Dockerfile @@ -1,6 +1,6 @@ FROM fedora:28 -ENV LIBVIRT_VERSION 5.0.0 +ENV LIBVIRT_VERSION 5.1.0 # Install packages RUN dnf install -y dnf-plugins-core && \ @@ -23,7 +23,9 @@ RUN dnf install -y dnf-plugins-core && \ rsync-daemon \ rsync \ qemu-img \ - protobuf-compiler && \ + protobuf-compiler \ + python3-devel \ + redhat-rpm-config && \ dnf -y clean all ENV GIMME_GO_VERSION=1.11.5 diff --git a/hack/builder/build.sh b/hack/builder/build.sh index ac541f356df9..02ff5fde01e6 100755 --- a/hack/builder/build.sh +++ b/hack/builder/build.sh @@ -5,4 +5,6 @@ SCRIPT_DIR="$( pwd )" -docker build -t kubevirt/builder:28-5.0.0 -f ${SCRIPT_DIR}/Dockerfile ${SCRIPT_DIR} +. ${SCRIPT_DIR}/version.sh + +docker build -t kubevirt/builder:${VERSION} -f ${SCRIPT_DIR}/Dockerfile ${SCRIPT_DIR} diff --git a/hack/builder/publish.sh b/hack/builder/publish.sh index d91e1f20b153..4a1b39628753 100755 --- a/hack/builder/publish.sh +++ b/hack/builder/publish.sh @@ -1,4 +1,11 @@ #!/usr/bin/env bash -docker tag kubevirt/builder:28-5.0.0 docker.io/kubevirt/builder:28-5.0.0 -docker push docker.io/kubevirt/builder:28-5.0.0 +SCRIPT_DIR="$( + cd "$(dirname "$BASH_SOURCE[0]")" + pwd +)" + +. ${SCRIPT_DIR}/version.sh + +docker tag kubevirt/builder:${VERSION} docker.io/kubevirt/builder:${VERSION} +docker push docker.io/kubevirt/builder:${VERSION} diff --git a/hack/builder/version.sh b/hack/builder/version.sh new file mode 100644 index 000000000000..52f0305cec18 --- /dev/null +++ b/hack/builder/version.sh @@ -0,0 +1 @@ +VERSION=28-5.1.0 diff --git a/hack/config-default.sh b/hack/config-default.sh index 0762eaa090aa..a27b29c1c01f 100644 --- a/hack/config-default.sh +++ b/hack/config-default.sh @@ -9,3 +9,9 @@ namespace=kubevirt cdi_namespace=cdi image_pull_policy=${IMAGE_PULL_POLICY:-IfNotPresent} verbosity=${VERBOSITY:-2} + +# try to derive csv_version from docker tag. But it must start with x.y.z, without leading v +default_csv_version="${docker_tag/latest/0.0.0}" +default_csv_version="${default_csv_version/devel/0.0.0}" +[[ $default_csv_version == v* ]] && default_csv_version="${default_csv_version/v/}" +csv_version=${CSV_VERSION:-$default_csv_version} diff --git a/hack/config.sh b/hack/config.sh index 3549811f5c79..f6d70ccd6f7e 100644 --- a/hack/config.sh +++ b/hack/config.sh @@ -1,5 +1,5 @@ unset binaries docker_images docker_prefix docker_tag docker_tag_alt manifest_templates \ - master_ip network_provider kubeconfig manifest_docker_prefix namespace image_pull_policy verbosity + master_ip network_provider kubeconfig manifest_docker_prefix namespace image_pull_policy verbosity csv_version KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER:-${PROVIDER}} @@ -13,4 +13,4 @@ test -f "hack/config-provider-${KUBEVIRT_PROVIDER}.sh" && source hack/config-pro test -f "hack/config-local.sh" && source hack/config-local.sh export binaries docker_images docker_prefix docker_tag docker_tag_alt manifest_templates \ - master_ip network_provider kubeconfig namespace image_pull_policy verbosity + master_ip network_provider kubeconfig namespace image_pull_policy verbosity csv_version diff --git a/hack/generate.sh b/hack/generate.sh index fc1472cd06e0..6945b5d54426 100755 --- a/hack/generate.sh +++ b/hack/generate.sh @@ -23,11 +23,14 @@ ${KUBEVIRT_DIR}/tools/resource-generator/resource-generator --type=vm >${KUBEVIR ${KUBEVIRT_DIR}/tools/resource-generator/resource-generator --type=vmim >${KUBEVIRT_DIR}/manifests/generated/vmim-resource.yaml ${KUBEVIRT_DIR}/tools/resource-generator/resource-generator --type=kv >${KUBEVIRT_DIR}/manifests/generated/kv-resource.yaml ${KUBEVIRT_DIR}/tools/resource-generator/resource-generator --type=kv-cr --namespace={{.Namespace}} --pullPolicy={{.ImagePullPolicy}} >${KUBEVIRT_DIR}/manifests/generated/kubevirt-cr.yaml.in -${KUBEVIRT_DIR}/tools/resource-generator/resource-generator --type=rbac --namespace={{.Namespace}} >${KUBEVIRT_DIR}/manifests/generated/rbac.authorization.k8s.yaml.in +${KUBEVIRT_DIR}/tools/resource-generator/resource-generator --type=kubevirt-rbac --namespace={{.Namespace}} >${KUBEVIRT_DIR}/manifests/generated/rbac-kubevirt.authorization.k8s.yaml.in +${KUBEVIRT_DIR}/tools/resource-generator/resource-generator --type=cluster-rbac --namespace={{.Namespace}} >${KUBEVIRT_DIR}/manifests/generated/rbac-cluster.authorization.k8s.yaml.in +${KUBEVIRT_DIR}/tools/resource-generator/resource-generator --type=operator-rbac --namespace={{.Namespace}} >${KUBEVIRT_DIR}/manifests/generated/rbac-operator.authorization.k8s.yaml.in ${KUBEVIRT_DIR}/tools/resource-generator/resource-generator --type=prometheus --namespace={{.Namespace}} >${KUBEVIRT_DIR}/manifests/generated/prometheus.yaml.in ${KUBEVIRT_DIR}/tools/resource-generator/resource-generator --type=virt-api --namespace={{.Namespace}} --repository={{.DockerPrefix}} --version={{.DockerTag}} --pullPolicy={{.ImagePullPolicy}} --verbosity={{.Verbosity}} >${KUBEVIRT_DIR}/manifests/generated/virt-api.yaml.in ${KUBEVIRT_DIR}/tools/resource-generator/resource-generator --type=virt-controller --namespace={{.Namespace}} --repository={{.DockerPrefix}} --version={{.DockerTag}} --pullPolicy={{.ImagePullPolicy}} --verbosity={{.Verbosity}} >${KUBEVIRT_DIR}/manifests/generated/virt-controller.yaml.in ${KUBEVIRT_DIR}/tools/resource-generator/resource-generator --type=virt-handler --namespace={{.Namespace}} --repository={{.DockerPrefix}} --version={{.DockerTag}} --pullPolicy={{.ImagePullPolicy}} --verbosity={{.Verbosity}} >${KUBEVIRT_DIR}/manifests/generated/virt-handler.yaml.in +${KUBEVIRT_DIR}/tools/resource-generator/resource-generator --type=virt-operator --namespace={{.Namespace}} --repository={{.DockerPrefix}} --version={{.DockerTag}} --pullPolicy={{.ImagePullPolicy}} --verbosity={{.Verbosity}} >${KUBEVIRT_DIR}/manifests/generated/virt-operator.yaml.in (cd ${KUBEVIRT_DIR}/tools/vms-generator/ && go build) vms_docker_prefix=${DOCKER_PREFIX:-registry:5000/kubevirt} diff --git a/hack/kubevirt-builder/Dockerfile b/hack/kubevirt-builder/Dockerfile index 1d5b29f36a77..612bef686c24 100644 --- a/hack/kubevirt-builder/Dockerfile +++ b/hack/kubevirt-builder/Dockerfile @@ -1,4 +1,4 @@ -FROM kubevirt/builder@sha256:cd662847df816a0c5cf1a99dcb905301d305befe93d50bee8d00fdd5c74beb19 +FROM kubevirt/builder:28-5.1.0 ENV GIMME_GO_VERSION=1.11.5 ENV GOPATH="/go" GOBIN="/usr/bin" @@ -13,7 +13,7 @@ RUN \ go get -u github.com/rmohr/go-swagger-utils/swagger-doc && \ go get -u github.com/onsi/ginkgo/ginkgo -RUN pip install j2cli +RUN pip install j2cli && pip3 install operator-courier COPY rsyncd.conf /etc/rsyncd.conf diff --git a/hack/olm.sh b/hack/olm.sh new file mode 100755 index 000000000000..1f83cd042272 --- /dev/null +++ b/hack/olm.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# +# This file is part of the KubeVirt project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright 2019 Red Hat, Inc. +# + +set -e + +source $(dirname "$0")/common.sh + +if [ $# -ne 1 ]; then + echo "usage: ${0} verify | push" +fi + +BUNDLE_DIR=${OUT_DIR}/manifests/release/olm/bundle +echo "using these manifests:" +ls ${BUNDLE_DIR} + +case ${1} in + +verify) + + IFS= + result=$(operator-courier verify ${BUNDLE_DIR} 2>&1) + echo $result + + if [[ $result =~ "ERROR" ]]; then + echo "olm verify failed!" + exit 1 + fi + + echo "olm verify success!" + exit 0 + + ;; + +push) + + if [[ -z "$CSV_VERSION" ]] || [[ -z "$QUAY_USERNAME" ]] || [[ -z "$QUAY_USERNAME" ]]; then + echo "please set CSV_VERSION, QUAY_USERNAME and QUAY_PASSWORD" + exit 1 + fi + + echo "getting auth token from Quay" + AUTH_TOKEN=$(curl -sH "Content-Type: application/json" -XPOST https://quay.io/cnr/api/v1/users/login -d ' + { + "user": { + "username": "'"${QUAY_USERNAME}"'", + "password": "'"${QUAY_PASSWORD}"'" + } + }' | jq -r '.token') + + echo "pushing bundle" + operator-courier push "$BUNDLE_DIR" "$QUAY_USERNAME" kubevirt "$CSV_VERSION" "$AUTH_TOKEN" + + ;; +esac diff --git a/manifests/generated/rbac-cluster.authorization.k8s.yaml.in b/manifests/generated/rbac-cluster.authorization.k8s.yaml.in new file mode 100644 index 000000000000..03fb93f51bff --- /dev/null +++ b/manifests/generated/rbac-cluster.authorization.k8s.yaml.in @@ -0,0 +1,135 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + kubernetes.io/bootstrapping: rbac-defaults + kubevirt.io: "" + name: kubevirt.io:default +rules: +- apiGroups: + - subresources.kubevirt.io + resources: + - version + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + kubevirt.io: "" + name: kubevirt.io:default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubevirt.io:default +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:authenticated +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:unauthenticated +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + kubevirt.io: "" + rbac.authorization.k8s.io/aggregate-to-admin: "true" + name: kubevirt.io:admin +rules: +- apiGroups: + - subresources.kubevirt.io + resources: + - virtualmachineinstances/console + - virtualmachineinstances/vnc + verbs: + - get +- apiGroups: + - subresources.kubevirt.io + resources: + - virtualmachines/restart + verbs: + - put + - update +- apiGroups: + - kubevirt.io + resources: + - virtualmachines + - virtualmachineinstances + - virtualmachineinstancepresets + - virtualmachineinstancereplicasets + verbs: + - get + - delete + - create + - update + - patch + - list + - watch + - deletecollection +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + kubevirt.io: "" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + name: kubevirt.io:edit +rules: +- apiGroups: + - subresources.kubevirt.io + resources: + - virtualmachineinstances/console + - virtualmachineinstances/vnc + verbs: + - get +- apiGroups: + - subresources.kubevirt.io + resources: + - virtualmachines/restart + verbs: + - put + - update +- apiGroups: + - kubevirt.io + resources: + - virtualmachines + - virtualmachineinstances + - virtualmachineinstancepresets + - virtualmachineinstancereplicasets + verbs: + - get + - delete + - create + - update + - patch + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + kubevirt.io: "" + rbac.authorization.k8s.io/aggregate-to-view: "true" + name: kubevirt.io:view +rules: +- apiGroups: + - kubevirt.io + resources: + - virtualmachines + - virtualmachineinstances + - virtualmachineinstancepresets + - virtualmachineinstancereplicasets + verbs: + - get + - list + - watch diff --git a/manifests/generated/rbac.authorization.k8s.yaml.in b/manifests/generated/rbac-kubevirt.authorization.k8s.yaml.in similarity index 67% rename from manifests/generated/rbac.authorization.k8s.yaml.in rename to manifests/generated/rbac-kubevirt.authorization.k8s.yaml.in index 40559d501d96..70b359853293 100644 --- a/manifests/generated/rbac.authorization.k8s.yaml.in +++ b/manifests/generated/rbac-kubevirt.authorization.k8s.yaml.in @@ -1,139 +1,4 @@ --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - labels: - kubernetes.io/bootstrapping: rbac-defaults - kubevirt.io: "" - name: kubevirt.io:default -rules: -- apiGroups: - - subresources.kubevirt.io - resources: - - version - verbs: - - get - - list ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - labels: - kubevirt.io: "" - name: kubevirt.io:default -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kubevirt.io:default -subjects: -- apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:authenticated -- apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:unauthenticated ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - kubevirt.io: "" - rbac.authorization.k8s.io/aggregate-to-admin: "true" - name: kubevirt.io:admin -rules: -- apiGroups: - - subresources.kubevirt.io - resources: - - virtualmachineinstances/console - - virtualmachineinstances/vnc - verbs: - - get -- apiGroups: - - subresources.kubevirt.io - resources: - - virtualmachines/restart - verbs: - - put - - update -- apiGroups: - - kubevirt.io - resources: - - virtualmachines - - virtualmachineinstances - - virtualmachineinstancepresets - - virtualmachineinstancereplicasets - verbs: - - get - - delete - - create - - update - - patch - - list - - watch - - deletecollection ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - kubevirt.io: "" - rbac.authorization.k8s.io/aggregate-to-edit: "true" - name: kubevirt.io:edit -rules: -- apiGroups: - - subresources.kubevirt.io - resources: - - virtualmachineinstances/console - - virtualmachineinstances/vnc - verbs: - - get -- apiGroups: - - subresources.kubevirt.io - resources: - - virtualmachines/restart - verbs: - - put - - update -- apiGroups: - - kubevirt.io - resources: - - virtualmachines - - virtualmachineinstances - - virtualmachineinstancepresets - - virtualmachineinstancereplicasets - verbs: - - get - - delete - - create - - update - - patch - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - kubevirt.io: "" - rbac.authorization.k8s.io/aggregate-to-view: "true" - name: kubevirt.io:view -rules: -- apiGroups: - - kubevirt.io - resources: - - virtualmachines - - virtualmachineinstances - - virtualmachineinstancepresets - - virtualmachineinstancereplicasets - verbs: - - get - - list - - watch ---- apiVersion: v1 kind: ServiceAccount metadata: diff --git a/manifests/generated/rbac-operator.authorization.k8s.yaml.in b/manifests/generated/rbac-operator.authorization.k8s.yaml.in new file mode 100644 index 000000000000..d53fc9659f5a --- /dev/null +++ b/manifests/generated/rbac-operator.authorization.k8s.yaml.in @@ -0,0 +1,388 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + kubevirt.io: "" + name: kubevirt-operator + namespace: {{.Namespace}} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + kubevirt.io: "" + name: kubevirt-operator +rules: +- apiGroups: + - kubevirt.io + resources: + - kubevirts + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - serviceaccounts + - services + verbs: + - get + - list + - watch + - create + - update + - delete +- apiGroups: + - batch + resources: + - jobs + verbs: + - get + - list + - watch + - create + - delete +- apiGroups: + - apps + resources: + - deployments + - daemonsets + verbs: + - get + - list + - watch + - create + - delete +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterroles + - clusterrolebindings + - roles + - rolebindings + verbs: + - get + - list + - watch + - create + - delete +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch + - create + - delete +- apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + verbs: + - get + - list + - watch +- apiGroups: + - security.openshift.io + resourceNames: + - privileged + resources: + - securitycontextconstraints + verbs: + - get + - patch + - update +- apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + - mutatingwebhookconfigurations + verbs: + - get + - create + - update +- apiGroups: + - apiregistration.k8s.io + resources: + - apiservices + verbs: + - get + - create + - update +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list +- apiGroups: + - "" + resources: + - pods/exec + verbs: + - create +- apiGroups: + - kubevirt.io + resources: + - virtualmachines + - virtualmachineinstances + - virtualmachineinstancemigrations + verbs: + - get + - list + - watch + - delete +- apiGroups: + - kubevirt.io + resources: + - virtualmachineinstancepresets + verbs: + - watch + - list +- apiGroups: + - "" + resourceNames: + - extension-apiserver-authentication + resources: + - configmaps + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - limitranges + verbs: + - watch + - list +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - delete + - update + - create +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - pods + - configmaps + - endpoints + verbs: + - get + - list + - watch + - delete + - update + - create +- apiGroups: + - "" + resources: + - events + verbs: + - update + - create + - patch +- apiGroups: + - "" + resources: + - pods/finalizers + verbs: + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch +- apiGroups: + - kubevirt.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - cdi.kubevirt.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - k8s.cni.cncf.io + resources: + - network-attachment-definitions + verbs: + - get + - list + - watch +- apiGroups: + - kubevirt.io + resources: + - virtualmachineinstances + verbs: + - update + - list + - watch +- apiGroups: + - "" + resources: + - secrets + - persistentvolumeclaims + verbs: + - get +- apiGroups: + - "" + resources: + - nodes + verbs: + - patch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch +- apiGroups: + - subresources.kubevirt.io + resources: + - version + verbs: + - get + - list +- apiGroups: + - subresources.kubevirt.io + resources: + - virtualmachineinstances/console + - virtualmachineinstances/vnc + verbs: + - get +- apiGroups: + - subresources.kubevirt.io + resources: + - virtualmachines/restart + verbs: + - put + - update +- apiGroups: + - kubevirt.io + resources: + - virtualmachines + - virtualmachineinstances + - virtualmachineinstancepresets + - virtualmachineinstancereplicasets + verbs: + - get + - delete + - create + - update + - patch + - list + - watch + - deletecollection +- apiGroups: + - subresources.kubevirt.io + resources: + - virtualmachineinstances/console + - virtualmachineinstances/vnc + verbs: + - get +- apiGroups: + - subresources.kubevirt.io + resources: + - virtualmachines/restart + verbs: + - put + - update +- apiGroups: + - kubevirt.io + resources: + - virtualmachines + - virtualmachineinstances + - virtualmachineinstancepresets + - virtualmachineinstancereplicasets + verbs: + - get + - delete + - create + - update + - patch + - list + - watch +- apiGroups: + - kubevirt.io + resources: + - virtualmachines + - virtualmachineinstances + - virtualmachineinstancepresets + - virtualmachineinstancereplicasets + verbs: + - get + - list + - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + kubevirt.io: "" + name: kubevirt-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubevirt-operator +subjects: +- kind: ServiceAccount + name: kubevirt-operator + namespace: {{.Namespace}} diff --git a/manifests/generated/virt-operator.yaml.in b/manifests/generated/virt-operator.yaml.in new file mode 100644 index 000000000000..fa18554b5c12 --- /dev/null +++ b/manifests/generated/virt-operator.yaml.in @@ -0,0 +1,56 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + kubevirt.io: virt-operator + name: virt-operator + namespace: {{.Namespace}} +spec: + replicas: 1 + selector: + matchLabels: + kubevirt.io: virt-operator + strategy: {} + template: + metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly","operator":"Exists"}]' + labels: + kubevirt.io: virt-operator + prometheus.kubevirt.io: "" + name: virt-operator + spec: + containers: + - command: + - virt-operator + - --port + - "8443" + - -v + - {{.Verbosity}} + env: + - name: OPERATOR_IMAGE + value: {{.DockerPrefix}}/virt-operator:{{.DockerTag}} + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.annotations['olm.targetNamespaces'] + image: {{.DockerPrefix}}/virt-operator:{{.DockerTag}} + imagePullPolicy: {{.ImagePullPolicy}} + name: virt-operator + ports: + - containerPort: 8443 + name: metrics + protocol: TCP + readinessProbe: + httpGet: + path: /metrics + port: 8443 + scheme: HTTPS + initialDelaySeconds: 5 + timeoutSeconds: 10 + resources: {} + securityContext: + runAsNonRoot: true + serviceAccountName: kubevirt-operator diff --git a/manifests/release/kubevirt-operator.yaml.in b/manifests/release/kubevirt-operator.yaml.in index a5df3c9ccf65..9b2a177b03f7 100644 --- a/manifests/release/kubevirt-operator.yaml.in +++ b/manifests/release/kubevirt-operator.yaml.in @@ -28,82 +28,5 @@ rules: - list - watch - deletecollection ---- -kind: ServiceAccount -apiVersion: v1 -metadata: - name: kubevirt-operator - namespace: {{.Namespace}} - labels: - operator.kubevirt.io: "" ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: kubevirt-operator - namespace: {{.Namespace}} - labels: - operator.kubevirt.io: "" -roleRef: - kind: ClusterRole - name: cluster-admin - apiGroup: rbac.authorization.k8s.io -subjects: - - kind: ServiceAccount - name: kubevirt-operator - namespace: {{.Namespace}} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: virt-operator - namespace: {{.Namespace}} - labels: - operator.kubevirt.io: "virt-operator" -spec: - replicas: 1 - selector: - matchLabels: - operator.kubevirt.io: virt-operator - template: - metadata: - annotations: - scheduler.alpha.kubernetes.io/critical-pod: "" - scheduler.alpha.kubernetes.io/tolerations: | - [ - { - "key": "CriticalAddonsOnly", - "operator": "Exists" - } - ] - labels: - operator.kubevirt.io: virt-operator - prometheus.kubevirt.io: "" - spec: - serviceAccountName: kubevirt-operator - containers: - - name: virt-operator - image: &image {{.DockerPrefix}}/virt-operator:{{.DockerTag}} - imagePullPolicy: {{.ImagePullPolicy}} - command: - - virt-operator - - --port - - "8443" - - -v - - {{.Verbosity}} - ports: - - containerPort: 8443 - name: "metrics" - protocol: "TCP" - readinessProbe: - httpGet: - scheme: HTTPS - port: "metrics" - path: "/metrics" - initialDelaySeconds: 5 - periodSeconds: 10 - env: - - name: OPERATOR_IMAGE - value: *image - securityContext: - runAsNonRoot: true +{{index .GeneratedManifests "rbac-operator.authorization.k8s.yaml.in"}} +{{index .GeneratedManifests "virt-operator.yaml.in"}} diff --git a/manifests/release/kubevirt.yaml.in b/manifests/release/kubevirt.yaml.in index 7e2fd4f67157..c97227b1b78e 100644 --- a/manifests/release/kubevirt.yaml.in +++ b/manifests/release/kubevirt.yaml.in @@ -6,7 +6,8 @@ metadata: kubevirt.io: "" name: {{.Namespace}} {{index .GeneratedManifests "prometheus.yaml.in"}} -{{index .GeneratedManifests "rbac.authorization.k8s.yaml.in"}} +{{index .GeneratedManifests "rbac-cluster.authorization.k8s.yaml.in"}} +{{index .GeneratedManifests "rbac-kubevirt.authorization.k8s.yaml.in"}} {{index .GeneratedManifests "virt-api.yaml.in"}} {{index .GeneratedManifests "virt-controller.yaml.in"}} {{index .GeneratedManifests "virt-handler.yaml.in"}} diff --git a/manifests/release/olm/bundle/kubevirt-crds.yaml.in b/manifests/release/olm/bundle/kubevirt-crds.yaml.in new file mode 100644 index 000000000000..5e4ea66bd04a --- /dev/null +++ b/manifests/release/olm/bundle/kubevirt-crds.yaml.in @@ -0,0 +1 @@ +{{index .GeneratedManifests "kv-resource.yaml"}} diff --git a/manifests/release/olm/bundle/kubevirt-package.yaml.in b/manifests/release/olm/bundle/kubevirt-package.yaml.in new file mode 100644 index 000000000000..c83f7d95e158 --- /dev/null +++ b/manifests/release/olm/bundle/kubevirt-package.yaml.in @@ -0,0 +1,4 @@ +packageName: kubevirt +channels: + - name: alpha + currentCSV: kubevirtoperator.v{{.CsvVersion}} diff --git a/manifests/release/olm/bundle/kubevirt.vVERSION.csv.yaml.in b/manifests/release/olm/bundle/kubevirt.vVERSION.csv.yaml.in new file mode 100644 index 000000000000..073e449792d0 --- /dev/null +++ b/manifests/release/olm/bundle/kubevirt.vVERSION.csv.yaml.in @@ -0,0 +1,111 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: ClusterServiceVersion +metadata: + name: kubevirtoperator.v{{.CsvVersion}} + namespace: placeholder + annotations: + capabilities: "Full Lifecycle" + categories: "Virtualization" + alm-examples: | + [ + { + "apiVersion":"kubevirt.io/v1alpha3", + "kind":"KubeVirt", + "metadata": { + "name":"kubevirt", + "namespace":"kubevirt" + }, + "spec": { + "imagePullPolicy":"Always" + } + } + ] + description: Creates and maintains KubeVirt deployments +spec: + displayName: KubeVirt + description: | + KubeVirt is a cluster add-on that provides a way to run virtualized workloads alongside containerized workload in a Kubernetes / OpenShift native way. + + _The KubeVirt Operator does not support updates yet._ + keywords: + - KubeVirt + - Virtualization + version: {{.CsvVersion}} + maturity: alpha + #replaces: kubevirtoperator.v?.?.? + maintainers: + - name: Red Hat + email: kubevirt-dev@googlegroups.com + provider: + name: Red Hat + links: + - name: KubeVirt + url: https://kubevirt.io + - name: Source Code + url: https://github.com/kubevirt/kubevirt + icon: + - base64data: {{.KubeVirtLogo}} + mediatype: image/png + labels: + alm-owner-kubevirt: kubevirtoperator + operated-by: kubevirtoperator + selector: + matchLabels: + alm-owner-kubevirt: kubevirtoperator + operated-by: kubevirtoperator + installModes: + - type: OwnNamespace + supported: true + - type: SingleNamespace + supported: true + - type: MultiNamespace + supported: true + - type: AllNamespaces + supported: true + install: + strategy: deployment + spec: + clusterPermissions: + - serviceAccountName: kubevirt-operator + rules: +{{.OperatorRules}} + deployments: + - name: virt-operator + spec: +{{.OperatorDeploymentSpec}} + customresourcedefinitions: + owned: + - name: kubevirts.kubevirt.io + version: v1alpha3 + kind: KubeVirt + displayName: KubeVirt deployment + description: Represents a KubeVirt deployment. + specDescriptors: + - description: The ImagePullPolicy to use for the KubeVirt components. + displayName: ImagePullPolicy + path: imagePullPolicy + x-descriptors: + - 'urn:alm:descriptor:io.kubernetes:imagePullPolicy' + statusDescriptors: + - description: The deployment phase. + displayName: Phase + path: phase + x-descriptors: + - 'urn:alm:descriptor:io.kubernetes.phase' + - description: Explanation for the current status of the cluster. + displayName: Conditions + path: conditions + x-descriptors: + - 'urn:alm:descriptor:io.kubernetes.conditions' + - description: The observed version of the KubeVirt deployment. + displayName: Observed KubeVirt Version + path: observedKubeVirtVersion + - 'urn:alm:descriptor:text' + - description: The targeted version of the KubeVirt deployment. + displayName: Target KubeVirt Version + path: targetKubeVirtVersion + - 'urn:alm:descriptor:text' + - description: The version of the KubeVirt Operator. + displayName: KubeVirt Operator Version + path: operatorVersion + - 'urn:alm:descriptor:text' diff --git a/manifests/release/olm/kubevirt-csv-preconditions.yaml.in b/manifests/release/olm/kubevirt-csv-preconditions.yaml.in new file mode 100644 index 000000000000..b64a5ee21109 --- /dev/null +++ b/manifests/release/olm/kubevirt-csv-preconditions.yaml.in @@ -0,0 +1,9 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + labels: + kubevirt.io: "" + name: {{.Namespace}} +{{index .GeneratedManifests "kv-resource.yaml"}} +{{index .GeneratedManifests "rbac-operator.authorization.k8s.yaml.in"}} diff --git a/manifests/release/olm/kubevirt-operatorsource.yaml.in b/manifests/release/olm/kubevirt-operatorsource.yaml.in new file mode 100644 index 000000000000..6daaf7513571 --- /dev/null +++ b/manifests/release/olm/kubevirt-operatorsource.yaml.in @@ -0,0 +1,11 @@ +apiVersion: marketplace.redhat.com/v1alpha1 +kind: OperatorSource +metadata: + name: kubevirt + namespace: marketplace +spec: + type: appregistry + endpoint: https://quay.io/cnr + registryNamespace: kubevirt + displayName: "KubeVirt" + publisher: "Red Hat" diff --git a/manifests/release/olm/kubevirt-subscription.yaml.in b/manifests/release/olm/kubevirt-subscription.yaml.in new file mode 100644 index 000000000000..b67f61f68991 --- /dev/null +++ b/manifests/release/olm/kubevirt-subscription.yaml.in @@ -0,0 +1,10 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: kubevirt + namespace: {{.Namespace}} +spec: + channel: alpha + name: kubevirt + source: kubevirt + sourceNamespace: olm diff --git a/manifests/release/olm/operatorgroup.yaml.in b/manifests/release/olm/operatorgroup.yaml.in new file mode 100644 index 000000000000..489c28fa7b7d --- /dev/null +++ b/manifests/release/olm/operatorgroup.yaml.in @@ -0,0 +1,5 @@ +apiVersion: operators.coreos.com/v1alpha2 +kind: OperatorGroup +metadata: + name: {{.Namespace}} + namespace: {{.Namespace}} diff --git a/pkg/virt-operator/creation/components/crds.go b/pkg/virt-operator/creation/components/crds.go index 2d2dff93b772..bbe82ceaa924 100644 --- a/pkg/virt-operator/creation/components/crds.go +++ b/pkg/virt-operator/creation/components/crds.go @@ -247,7 +247,9 @@ func NewKubeVirtCrd() *extv1beta1.CustomResourceDefinition { return crd } -// used by manifest generation +// Used by manifest generation +// If you change something, you probably need to change the CSV manifest too, +// see /manifests/release/kubevirt.vVERSION.csv.yaml.in func NewKubeVirtCR(namespace string, pullPolicy corev1.PullPolicy) *virtv1.KubeVirt { return &virtv1.KubeVirt{ TypeMeta: metav1.TypeMeta{ diff --git a/pkg/virt-operator/creation/components/deployments.go b/pkg/virt-operator/creation/components/deployments.go index 426819a165c2..40a4c3b39b34 100644 --- a/pkg/virt-operator/creation/components/deployments.go +++ b/pkg/virt-operator/creation/components/deployments.go @@ -180,17 +180,11 @@ func NewApiServerService(namespace string) *corev1.Service { func newPodTemplateSpec(name string, repository string, version string, pullPolicy corev1.PullPolicy) (*corev1.PodTemplateSpec, error) { - tolerations := []corev1.Toleration{ - { - Key: "CriticalAddonsOnly", - Operator: corev1.TolerationOpExists, - }, - } - tolerationsStr, err := json.Marshal(tolerations) - + tolerations, err := criticalAddonsToleration() if err != nil { - return nil, fmt.Errorf("unable to create service: %v", err) + return nil, fmt.Errorf("unable to create toleration: %v", err) } + return &corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ @@ -199,7 +193,7 @@ func newPodTemplateSpec(name string, repository string, version string, pullPoli }, Annotations: map[string]string{ "scheduler.alpha.kubernetes.io/critical-pod": "", - "scheduler.alpha.kubernetes.io/tolerations": string(tolerationsStr), + "scheduler.alpha.kubernetes.io/tolerations": string(tolerations), }, Name: name, }, @@ -468,9 +462,123 @@ func NewHandlerDaemonSet(namespace string, repository string, version string, pu } +// Used for manifest generation only +func NewOperatorDeployment(namespace string, repository string, version string, pullPolicy corev1.PullPolicy, verbosity string) (*appsv1.Deployment, error) { + + name := "virt-operator" + image := fmt.Sprintf("%s/%s:%s", repository, name, version) + + tolerations, err := criticalAddonsToleration() + if err != nil { + return nil, fmt.Errorf("unable to create toleration: %v", err) + } + + deployment := &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "Deployment", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + Labels: map[string]string{ + virtv1.AppLabel: name, + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: int32Ptr(1), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + virtv1.AppLabel: name, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + virtv1.AppLabel: name, + "prometheus.kubevirt.io": "", + }, + Annotations: map[string]string{ + "scheduler.alpha.kubernetes.io/critical-pod": "", + "scheduler.alpha.kubernetes.io/tolerations": string(tolerations), + }, + Name: name, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: "kubevirt-operator", + Containers: []corev1.Container{ + { + Name: name, + Image: image, + ImagePullPolicy: pullPolicy, + Command: []string{ + "virt-operator", + "--port", + "8443", + "-v", + verbosity, + }, + Ports: []corev1.ContainerPort{ + { + Name: "metrics", + Protocol: corev1.ProtocolTCP, + ContainerPort: 8443, + }, + }, + ReadinessProbe: &corev1.Probe{ + Handler: corev1.Handler{ + HTTPGet: &corev1.HTTPGetAction{ + Scheme: corev1.URISchemeHTTPS, + Port: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 8443, + }, + Path: "/metrics", + }, + }, + InitialDelaySeconds: 5, + TimeoutSeconds: 10, + }, + Env: []corev1.EnvVar{ + { + Name: "OPERATOR_IMAGE", + Value: image, + }, + { + Name: "WATCH_NAMESPACE", // not used yet + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.annotations['olm.targetNamespaces']", // filled by OLM + }, + }, + }, + }, + }, + }, + SecurityContext: &corev1.PodSecurityContext{ + RunAsNonRoot: boolPtr(true), + }, + }, + }, + }, + } + + return deployment, nil +} + func int32Ptr(i int32) *int32 { return &i } func boolPtr(b bool) *bool { return &b } +func criticalAddonsToleration() ([]byte, error) { + tolerations := []corev1.Toleration{ + { + Key: "CriticalAddonsOnly", + Operator: corev1.TolerationOpExists, + }, + } + tolerationsStr, err := json.Marshal(tolerations) + return tolerationsStr, err +} diff --git a/pkg/virt-operator/creation/rbac/BUILD.bazel b/pkg/virt-operator/creation/rbac/BUILD.bazel index ede197345703..b6a17264cfa9 100644 --- a/pkg/virt-operator/creation/rbac/BUILD.bazel +++ b/pkg/virt-operator/creation/rbac/BUILD.bazel @@ -7,6 +7,7 @@ go_library( "cluster.go", "controller.go", "handler.go", + "operator.go", ], importpath = "kubevirt.io/kubevirt/pkg/virt-operator/creation/rbac", visibility = ["//visibility:public"], diff --git a/pkg/virt-operator/creation/rbac/operator.go b/pkg/virt-operator/creation/rbac/operator.go new file mode 100644 index 000000000000..e532147db993 --- /dev/null +++ b/pkg/virt-operator/creation/rbac/operator.go @@ -0,0 +1,283 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright 2019 Red Hat, Inc. + * + */ +package rbac + +import ( + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + virtv1 "kubevirt.io/kubevirt/pkg/api/v1" +) + +// Used for manifest generation only, not by the operator itself +func GetAllOperator(namespace string) []interface{} { + return []interface{}{ + newOperatorServiceAccount(namespace), + NewOperatorClusterRole(), + newOperatorClusterRoleBinding(namespace), + } +} + +func newOperatorServiceAccount(namespace string) *corev1.ServiceAccount { + return &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ServiceAccount", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "kubevirt-operator", + Labels: map[string]string{ + virtv1.AppLabel: "", + }, + }, + } +} + +// pulic, because it's used in manifest-templator +func NewOperatorClusterRole() *rbacv1.ClusterRole { + // These are permissions needed by the operator itself. + // For successfully deploying KubeVirt with the operator, you need to add everything + // that the KubeVirt components' rules use, see below + // (you can't create rules with permissions you don't have yourself) + operatorRole := &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "ClusterRole", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "kubevirt-operator", + Labels: map[string]string{ + virtv1.AppLabel: "", + }, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{ + "kubevirt.io", + }, + Resources: []string{ + "kubevirts", + }, + Verbs: []string{ + "get", + "list", + "watch", + "patch", + "update", + }, + }, + { + APIGroups: []string{ + "", + }, + Resources: []string{ + "serviceaccounts", + "services", + }, + Verbs: []string{ + "get", + "list", + "watch", + "create", + "update", + "delete", + }, + }, + { + APIGroups: []string{ + "batch", + }, + Resources: []string{ + "jobs", + }, + Verbs: []string{ + "get", + "list", + "watch", + "create", + "delete", + }, + }, + { + APIGroups: []string{ + "apps", + }, + Resources: []string{ + "deployments", + "daemonsets", + }, + Verbs: []string{ + "get", + "list", + "watch", + "create", + "delete", + }, + }, + { + APIGroups: []string{ + "rbac.authorization.k8s.io", + }, + Resources: []string{ + "clusterroles", + "clusterrolebindings", + "roles", + "rolebindings", + }, + Verbs: []string{ + "get", + "list", + "watch", + "create", + "delete", + }, + }, + { + APIGroups: []string{ + "apiextensions.k8s.io", + }, + Resources: []string{ + "customresourcedefinitions", + }, + Verbs: []string{ + "get", + "list", + "watch", + "create", + "delete", + }, + }, + { + APIGroups: []string{ + "security.openshift.io", + }, + Resources: []string{ + "securitycontextconstraints", + }, + Verbs: []string{ + "get", + "list", + "watch", + }, + }, + { + APIGroups: []string{ + "security.openshift.io", + }, + Resources: []string{ + "securitycontextconstraints", + }, + ResourceNames: []string{ + "privileged", + }, + Verbs: []string{ + "get", + "patch", + "update", + }, + }, + }, + } + + // now append all rules needed by KubeVirt's components + operatorRole.Rules = append(operatorRole.Rules, getKubeVirtComponentsRules()...) + return operatorRole +} + +func getKubeVirtComponentsRules() []rbacv1.PolicyRule { + + var rules []rbacv1.PolicyRule + + // namespace doesn't matter, we are only interested in the rules of both Roles and ClusterRoles + all := GetAllApiServer("") + all = append(all, GetAllController("")...) + all = append(all, GetAllHandler("")...) + all = append(all, GetAllCluster("")...) + + for _, resource := range all { + switch resource.(type) { + case *rbacv1.ClusterRole: + role, _ := resource.(*rbacv1.ClusterRole) + rules = append(rules, role.Rules...) + case *rbacv1.Role: + role, _ := resource.(*rbacv1.Role) + rules = append(rules, role.Rules...) + } + } + + // OLM doesn't support role refs + // so we need special handling for auth delegation for the apiserver, + // by adding the rules of the system:auth-delegator role manually + authDelegationRules := []rbacv1.PolicyRule{ + { + APIGroups: []string{ + "authentication.k8s.io", + }, + Resources: []string{ + "tokenreviews", + }, + Verbs: []string{ + "create", + }, + }, + { + APIGroups: []string{ + "authorization.k8s.io", + }, + Resources: []string{ + "subjectaccessreviews", + }, + Verbs: []string{ + "create", + }, + }, + } + rules = append(rules, authDelegationRules...) + + return rules +} + +func newOperatorClusterRoleBinding(namespace string) *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "ClusterRoleBinding", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "kubevirt-operator", + Labels: map[string]string{ + virtv1.AppLabel: "", + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: "kubevirt-operator", + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Namespace: namespace, + Name: "kubevirt-operator", + }, + }, + } +} diff --git a/tools/manifest-templator/BUILD.bazel b/tools/manifest-templator/BUILD.bazel index ac2fad63ecd0..c780458cccd4 100644 --- a/tools/manifest-templator/BUILD.bazel +++ b/tools/manifest-templator/BUILD.bazel @@ -5,7 +5,13 @@ go_library( srcs = ["manifest-templator.go"], importpath = "kubevirt.io/kubevirt/tools/manifest-templator", visibility = ["//visibility:private"], - deps = ["//vendor/github.com/spf13/pflag:go_default_library"], + deps = [ + "//pkg/virt-operator/creation/components:go_default_library", + "//pkg/virt-operator/creation/rbac:go_default_library", + "//tools/util:go_default_library", + "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/api/core/v1:go_default_library", + ], ) go_binary( diff --git a/tools/manifest-templator/manifest-templator.go b/tools/manifest-templator/manifest-templator.go index c1e0420c2a00..0b109c50be96 100644 --- a/tools/manifest-templator/manifest-templator.go +++ b/tools/manifest-templator/manifest-templator.go @@ -20,24 +20,37 @@ package main import ( + "bufio" + "encoding/base64" "flag" "fmt" "io/ioutil" "os" "path/filepath" + "strings" "text/template" + "k8s.io/api/core/v1" + + "kubevirt.io/kubevirt/pkg/virt-operator/creation/components" + "kubevirt.io/kubevirt/pkg/virt-operator/creation/rbac" + "kubevirt.io/kubevirt/tools/util" + "github.com/spf13/pflag" ) type templateData struct { - Namespace string - CDINamespace string - DockerTag string - DockerPrefix string - ImagePullPolicy string - Verbosity string - GeneratedManifests map[string]string + Namespace string + CDINamespace string + DockerTag string + DockerPrefix string + ImagePullPolicy string + Verbosity string + CsvVersion string + OperatorDeploymentSpec string + OperatorRules string + KubeVirtLogo string + GeneratedManifests map[string]string } func main() { @@ -45,12 +58,15 @@ func main() { cdiNamespace := flag.String("cdi-namespace", "", "") dockerPrefix := flag.String("container-prefix", "", "") dockerTag := flag.String("container-tag", "", "") + csvVersion := flag.String("csv-version", "", "") imagePullPolicy := flag.String("image-pull-policy", "IfNotPresent", "") verbosity := flag.String("verbosity", "2", "") genDir := flag.String("generated-manifests-dir", "", "") inputFile := flag.String("input-file", "", "") processFiles := flag.Bool("process-files", false, "") processVars := flag.Bool("process-vars", false, "") + kubeVirtLogoPath := flag.String("kubevirt-logo-path", "", "") + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) pflag.CommandLine.ParseErrorsWhitelist.UnknownFlags = true pflag.Parse() @@ -70,6 +86,10 @@ func main() { data.DockerPrefix = *dockerPrefix data.ImagePullPolicy = *imagePullPolicy data.Verbosity = fmt.Sprintf("\"%s\"", *verbosity) + data.CsvVersion = *csvVersion + data.OperatorDeploymentSpec = getOperatorDeploymentSpec(data) + data.OperatorRules = getOperatorRules() + data.KubeVirtLogo = getKubeVirtLogo(*kubeVirtLogoPath) } else { // keep templates data.Namespace = "{{.Namespace}}" @@ -78,6 +98,10 @@ func main() { data.DockerPrefix = "{{.DockerPrefix}}" data.ImagePullPolicy = "{{.ImagePullPolicy}}" data.Verbosity = "{{.Verbosity}}" + data.CsvVersion = "{{.CsvVersion}}" + data.OperatorDeploymentSpec = "{{.OperatorDeploymentSpec}}" + data.OperatorRules = "{{.OperatorRules}}" + data.KubeVirtLogo = "{{.KubeVirtLogo}}" } if *processFiles { @@ -104,3 +128,68 @@ func main() { panic(err) } } + +func getOperatorRules() string { + rules := rbac.NewOperatorClusterRole().Rules + writer := strings.Builder{} + for _, rule := range rules { + err := util.MarshallObject(rule, &writer) + if err != nil { + panic(err) + } + } + return fixResourceString(writer.String(), 14) +} + +func getOperatorDeploymentSpec(data templateData) string { + deployment, err := components.NewOperatorDeployment(data.Namespace, data.DockerPrefix, data.DockerTag, v1.PullPolicy(data.ImagePullPolicy), data.Verbosity) + if err != nil { + panic(err) + } + writer := strings.Builder{} + err = util.MarshallObject(deployment.Spec, &writer) + if err != nil { + panic(err) + } + return fixResourceString(writer.String(), 12) +} + +func fixResourceString(in string, indention int) string { + out := strings.Builder{} + scanner := bufio.NewScanner(strings.NewReader(in)) + for scanner.Scan() { + line := scanner.Text() + // remove separator lines + if !strings.HasPrefix(line, "---") { + // indent so that it fits into the manifest + // spaces is is indention - 2, because we want to have 2 spaces less for being able to start an array + spaces := strings.Repeat(" ", indention-2) + if strings.HasPrefix(line, "apiGroups") { + // spaces + array start + out.WriteString(spaces + "- " + line + "\n") + } else { + // 2 more spaces + out.WriteString(spaces + " " + line + "\n") + } + } + } + return out.String() +} + +func getKubeVirtLogo(path string) string { + file, err := os.Open(path) + if err != nil { + panic(err) + } + + // Read entire file into byte slice. + reader := bufio.NewReader(file) + content, err := ioutil.ReadAll(reader) + if err != nil { + panic(err) + } + + // Encode as base64. + encoded := base64.StdEncoding.EncodeToString(content) + return encoded +} diff --git a/tools/resource-generator/resource-generator.go b/tools/resource-generator/resource-generator.go index 14294fc47fdc..8f0ad12f5ced 100644 --- a/tools/resource-generator/resource-generator.go +++ b/tools/resource-generator/resource-generator.go @@ -58,15 +58,24 @@ func main() { util.MarshallObject(components.NewKubeVirtCrd(), os.Stdout) case "kv-cr": util.MarshallObject(components.NewKubeVirtCR(*namespace, imagePullPolicy), os.Stdout) - case "rbac": + case "kubevirt-rbac": all := make([]interface{}, 0) - all = append(all, rbac.GetAllCluster(*namespace)...) all = append(all, rbac.GetAllApiServer(*namespace)...) all = append(all, rbac.GetAllController(*namespace)...) all = append(all, rbac.GetAllHandler(*namespace)...) for _, r := range all { util.MarshallObject(r, os.Stdout) } + case "cluster-rbac": + all := rbac.GetAllCluster(*namespace) + for _, r := range all { + util.MarshallObject(r, os.Stdout) + } + case "operator-rbac": + all := rbac.GetAllOperator(*namespace) + for _, r := range all { + util.MarshallObject(r, os.Stdout) + } case "prometheus": util.MarshallObject(components.NewPrometheusService(*namespace), os.Stdout) case "virt-api": @@ -93,6 +102,13 @@ func main() { panic(fmt.Errorf("error generating virt-handler deployment %v", err)) } util.MarshallObject(handler, os.Stdout) + case "virt-operator": + operator, err := components.NewOperatorDeployment(*namespace, *repository, *version, imagePullPolicy, *verbosity) + if err != nil { + panic(fmt.Errorf("error generating virt-operator deployment %v", err)) + + } + util.MarshallObject(operator, os.Stdout) default: panic(fmt.Errorf("unknown resource type %s", *resourceType)) } diff --git a/tools/util/marshaller.go b/tools/util/marshaller.go index afe78c2b6fbe..750602a3b48b 100644 --- a/tools/util/marshaller.go +++ b/tools/util/marshaller.go @@ -89,10 +89,15 @@ func MarshallObject(obj interface{}, writer io.Writer) error { return err } - // fix templates by removing quotes... + // fix templates by removing unneeded single quotes... s := string(yamlBytes) s = strings.Replace(s, "'{{", "{{", -1) s = strings.Replace(s, "}}'", "}}", -1) + + // fix double quoted strings by removing unneeded single quotes... + s = strings.Replace(s, " '\"", " \"", -1) + s = strings.Replace(s, "\"'\n", "\"\n", -1) + yamlBytes = []byte(s) _, err = writer.Write([]byte("---\n")) diff --git a/tools/vms-generator/utils/utils.go b/tools/vms-generator/utils/utils.go index e06afb08786a..6ab5a0f2dbaa 100644 --- a/tools/vms-generator/utils/utils.go +++ b/tools/vms-generator/utils/utils.go @@ -72,8 +72,6 @@ const VmiPresetSmall = "vmi-preset-small" const VmiMigration = "migration-job" -const KubeVirt = "kubevirt-cr" - const ( busVirtio = "virtio" busSata = "sata" @@ -87,8 +85,6 @@ const ( const windowsFirmware = "5d307ca9-b3ef-428c-8861-06e72d69f223" -const apiVersion = "kubevirt.io/v1alpha3" - var DockerPrefix = "registry:5000/kubevirt" var DockerTag = "devel"