beating on kubevirt

lanefu · Sep 7, 2024 · 96f37ae · 96f37ae
1 parent dee8609
commit 96f37ae
Show file tree

Hide file tree

Showing 15 changed files with 344 additions and 3 deletions.
diff --git a/files/manifests/kubevirt/armbian/armbian_vm2_pvc_x86_64.yaml b/files/manifests/kubevirt/armbian/armbian_vm2_pvc_x86_64.yaml
@@ -0,0 +1,79 @@
+---
+apiVersion: kubevirt.io/v1
+kind: VirtualMachine
+metadata:
+  creationTimestamp: 2018-07-04T15:03:08Z
+  generation: 1
+  labels:
+    kubevirt.io/os: linux
+  name: armbian-vm2
+spec:
+  running: true
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        kubevirt.io/domain: vm2
+    spec:
+      domain:
+        cpu:
+          cores: 4
+        devices:
+          interfaces:
+            - name: podnet
+              masquerade: {}
+            - name: hostnetwork
+              bridge: {}
+            #            - name: hostnetwork
+            #              binding:
+            #                name: macvtap
+          disks:
+            - disk:
+                bus: virtio
+              name: disk0
+            - disk:
+                bus: virtio
+                readonly: true
+              name: cloudinitdisk
+        features:
+          acpi: {}
+          smm:
+            enabled: false
+        firmware:
+          bootloader:
+            efi:
+              secureBoot: false
+        resources:
+          requests:
+            memory: 11G
+      networks:
+        - name: podnet
+          pod: {}
+        - name: hostnetwork
+          multus:
+            networkName: bridge-test
+        #        - name: hostnetwork
+        #          multus:
+        #            networkName: macvtapnetwork
+      volumes:
+        - name: disk0
+          persistentVolumeClaim:
+            claimName: armbian
+        - cloudInitNoCloud:
+            networkData: |
+              version: 2
+              ethernets:
+                enp1s0:
+                  addresses: [ fd10:0:2::2/120 ]
+                  dhcp4: true
+                  gateway6: fd10:0:2::1
+            userData: |
+              #cloud-config
+              hostname: vm2
+              password: root
+              chpasswd: { expire: False }
+              ssh_pwauth: True
+              disable_root: false
+              ssh_authorized_keys:
+              - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGAhF7pobuxW0ulzW5nEU2CiC9miWV350/F/+JfeDMXF Its 2021 people have died and its time for a new key
+          name: cloudinitdisk
diff --git a/files/manifests/kubevirt/armbian/dv_armbian_x86_64.yaml b/files/manifests/kubevirt/armbian/dv_armbian_x86_64.yaml
@@ -0,0 +1,17 @@
+apiVersion: cdi.kubevirt.io/v1beta1
+kind: DataVolume
+metadata:
+  name: "armbian"
+spec:
+  storage:
+    accessModes:
+      - ReadWriteOnce
+    resources:
+      requests:
+        storage: 40Gi
+  source:
+    http:
+      #      url: "https://github.com/armsurvivors/armbian-release/releases/download/24.08.23-armsurvivors-423/Armbian-unofficial_24.08.23-armsurvivors-423_Uefi-x86_bookworm_current_6.6.47-metadata-cloud-k8s-1.30.img.qcow2.xz"
+      # bad # url: "https://github.com/armsurvivors/armbian-release/releases/download/24.08.23-armsurvivors-423/Armbian-unofficial_24.08.23-armsurvivors-423_Uefi-x86_bookworm_edge_6.10.6-metadata-cloud.img.qcow2.xz"
+      #url: "https://github.com/armsurvivors/armbian-release/releases/download/24.08.21-armsurvivors-417/Armbian-unofficial_24.08.21-armsurvivors-417_Uefi-x86_bookworm_edge_6.10.6-metadata-cloud-k8s-1.30.img.qcow2.xz"
+      url: "https://github.com/armsurvivors/armbian-release/releases/download/24.08.23-armsurvivors-423/Armbian-unofficial_24.08.23-armsurvivors-423_Uefi-x86_bookworm_edge_6.10.6-metadata-cloud-k8s-1.30.img.qcow2.xz"
diff --git a/files/manifests/kubevirt/bridge/bridge_attachment_definition.yaml b/files/manifests/kubevirt/bridge/bridge_attachment_definition.yaml
@@ -0,0 +1,12 @@
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: bridge-test
+spec:
+  config: '{
+    "cniVersion": "0.3.1",
+    "name": "bridge-test",
+    "type": "bridge",
+    "bridge": "bridge0",
+    "disableContainerInterface": false
+    }'
diff --git a/files/manifests/kubevirt/config/feature_gates.yaml b/files/manifests/kubevirt/config/feature_gates.yaml
@@ -0,0 +1,11 @@
+apiVersion: kubevirt.io/v1
+kind: KubeVirt
+metadata:
+  name: kubevirt
+  namespace: kubevirt
+spec:
+  configuration:
+    developerConfiguration:
+      featureGates:
+        #        - DataVolumes
+        - NetworkBindingPlugins
diff --git a/files/manifests/kubevirt/debian/debian_vm1_pvc_x86_64.yaml b/files/manifests/kubevirt/debian/debian_vm1_pvc_x86_64.yaml
@@ -0,0 +1,60 @@
+---
+apiVersion: kubevirt.io/v1
+kind: VirtualMachine
+metadata:
+  creationTimestamp: 2018-07-04T15:03:08Z
+  generation: 1
+  labels:
+    kubevirt.io/os: linux
+  name: debian-vm1
+spec:
+  running: true
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        kubevirt.io/domain: vm1
+    spec:
+      domain:
+        cpu:
+          cores: 2
+        devices:
+          interfaces:
+            - name: default
+              masquerade: {}
+          disks:
+            - disk:
+                bus: virtio
+              name: disk0
+            - disk:
+                bus: virtio
+                readonly: true
+              name: cloudinitdisk
+        resources:
+          requests:
+            memory: 1024M
+      networks:
+        - name: default
+          pod: {}
+      volumes:
+        - name: disk0
+          persistentVolumeClaim:
+            claimName: debian
+        - cloudInitNoCloud:
+            networkData: |
+              version: 2
+              ethernets:
+                enp1s0:
+                  addresses: [ fd10:0:2::2/120 ]
+                  dhcp4: true
+                  gateway6: fd10:0:2::1
+            userData: |
+              #cloud-config
+              hostname: vm1
+              password: root
+              chpasswd: { expire: False }
+              ssh_pwauth: True
+              disable_root: false
+              ssh_authorized_keys:
+              - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGAhF7pobuxW0ulzW5nEU2CiC9miWV350/F/+JfeDMXF Its 2021 people have died and its time for a new key
+          name: cloudinitdisk
diff --git a/files/manifests/kubevirt/debian/dv_debian_x86_64.yaml b/files/manifests/kubevirt/debian/dv_debian_x86_64.yaml
@@ -0,0 +1,14 @@
+apiVersion: cdi.kubevirt.io/v1beta1
+kind: DataVolume
+metadata:
+  name: "debian"
+spec:
+  storage:
+    accessModes:
+      - ReadWriteOnce
+    resources:
+      requests:
+        storage: 5Gi
+  source:
+    http:
+      url: "https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.raw"
diff --git a/files/manifests/kubevirt/fedora/dv_fedora_x86_64.yaml b/files/manifests/kubevirt/fedora/dv_fedora_x86_64.yaml
@@ -0,0 +1,15 @@
+apiVersion: cdi.kubevirt.io/v1beta1
+kind: DataVolume
+metadata:
+  name: "fedora"
+spec:
+  storage:
+    accessModes:
+      - ReadWriteOnce
+    resources:
+      requests:
+        storage: 10Gi
+  source:
+    http:
+      #url: "https://download.fedoraproject.org/pub/fedora/linux/releases/40/Cloud/x86_64/images/Fedora-Cloud-Base-AmazonEC2.x86_64-40-1.14.raw.xz"
+      url: "https://download.fedoraproject.org/pub/fedora/linux/releases/40/Cloud/x86_64/images/Fedora-Cloud-Base-Generic.x86_64-40-1.14.qcow2"
diff --git a/files/manifests/kubevirt/fedora/fedora_vm3_pvc_x86_64.yaml b/files/manifests/kubevirt/fedora/fedora_vm3_pvc_x86_64.yaml
@@ -0,0 +1,60 @@
+---
+apiVersion: kubevirt.io/v1
+kind: VirtualMachine
+metadata:
+  creationTimestamp: 2018-07-04T15:03:08Z
+  generation: 1
+  labels:
+    kubevirt.io/os: linux
+  name: fedora-vm3
+spec:
+  running: true
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        kubevirt.io/domain: vm3
+    spec:
+      domain:
+        cpu:
+          cores: 2
+        devices:
+          interfaces:
+            - name: default
+              masquerade: {}
+          disks:
+            - disk:
+                bus: virtio
+              name: disk0
+            - disk:
+                bus: virtio
+                readonly: true
+              name: cloudinitdisk
+        resources:
+          requests:
+            memory: 1024M
+      networks:
+        - name: default
+          pod: {}
+      volumes:
+        - name: disk0
+          persistentVolumeClaim:
+            claimName: fedora
+        - cloudInitNoCloud:
+            networkData: |
+              version: 2
+              ethernets:
+                enp1s0:
+                  addresses: [ fd10:0:2::2/120 ]
+                  dhcp4: true
+                  gateway6: fd10:0:2::1
+            userData: |
+              #cloud-config
+              hostname: vm3
+              password: root
+              chpasswd: { expire: False }
+              ssh_pwauth: True
+              disable_root: false
+              ssh_authorized_keys:
+              - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGAhF7pobuxW0ulzW5nEU2CiC9miWV350/F/+JfeDMXF Its 2021 people have died and its time for a new key
+          name: cloudinitdisk
diff --git a/files/manifests/kubevirt/mactap/attachment_definition_bridge0.yaml b/files/manifests/kubevirt/mactap/attachment_definition_bridge0.yaml
@@ -0,0 +1,13 @@
+kind: NetworkAttachmentDefinition
+apiVersion: k8s.cni.cncf.io/v1
+metadata:
+  name: macvtapnetwork
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: macvtap.network.kubevirt.io/eno1
+spec:
+  config: '{
+    "cniVersion": "0.3.1",
+    "name": "macvtapnetwork",
+    "type": "macvtap",
+    "mtu": 1500
+    }'
diff --git a/files/manifests/kubevirt/mactap/mactap_minimal_configmap.yaml b/files/manifests/kubevirt/mactap/mactap_minimal_configmap.yaml
@@ -0,0 +1,6 @@
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: macvtap-deviceplugin-config
+data:
+  DP_MACVTAP_CONF: "[]"
diff --git a/files/scripts/install_cni_reference_plugins.sh b/files/scripts/install_cni_reference_plugins.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+export ARCH_CNI=$([ $(uname -m) = aarch64 ] && echo arm64 || echo amd64)
+export CNI_PLUGIN_VERSION=v1.5.1
+mkdir -p /opt/cni/bin
+curl -L -o cni-plugins.tgz "https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGIN_VERSION}/cni-plugins-linux-${ARCH_CNI}-${CNI_PLUGIN_VERSION}".tgz &&
+  tar -C /opt/cni/bin -xzf cni-plugins.tgz --wildcards --no-anchored 'bridge'
diff --git a/playbooks/kubevirt/provision_kubevirt_test.yml b/playbooks/kubevirt/provision_kubevirt_test.yml
@@ -1,5 +1,6 @@
 ---
-- hosts:
+- name: provision kubevirt test instance
+  hosts:
     - milddragon
 
   vars:
@@ -15,7 +16,7 @@
       flannel-backend: "none"
 
   tasks:
-    - name: provison whole cluster
+    - name: Provison whole cluster
       import_role:
         name: xanmanning.k3s
       tags:

diff --git a/scripts/kubevirt/create_test.sh b/scripts/kubevirt/create_test.sh
@@ -5,6 +5,10 @@ source armlab_kubevirt_test.conf
 
 ansible-playbook playbooks/kubevirt/provision_kubevirt_test.yml
 
+#install reference plugins
+##FIXME move to playbook
+ansible ${PRIMARY_HOST} -m script -a 'files/scripts/install_cni_reference_plugins.sh' --become
+
 ./scripts/fetch_kubeconfig.sh ${PRIMARY_HOST_IP}
 export KUBECONFIG=$(pwd)/kubeconfig
 
@@ -13,3 +17,5 @@ sleep 30
 cilium install
 
 kubectl get pods --all-namespaces
+
+./scripts/kubevirt/install_kubevirt.sh
diff --git a/scripts/kubevirt/destroy_test.sh b/scripts/kubevirt/destroy_test.sh
@@ -4,3 +4,5 @@
 ansible-playbook playbooks/kubevirt/provision_kubevirt_test.yml -i inventory/test.yml -e k3s_state=stopped
 ansible-playbook playbooks/kubevirt/provision_kubevirt_test.yml -i inventory/test.yml -e k3s_state=uninstalled
 ansible milddragon -i inventory/test.yml -m command -a 'nft flush ruleset' --become
+##fixme delete /opt/cni/bin contents
+ansible milddragon -i inventory/test.yml -m file -a 'path=/opt/cni/bin state=absent' --become
diff --git a/scripts/kubevirt/install_kubevirt.sh b/scripts/kubevirt/install_kubevirt.sh
@@ -1,5 +1,19 @@
 #!/bin/bash
 
+##check for crew
+kubectl krew version
+if [ $? -eq 1 ]; then
+  echo "need to install krew see:"
+  echo "https://krew.sigs.k8s.io/docs/user-guide/setup/install/"
+  exit 1
+fi
+
+kubectl virt version
+if [ $? -eq 1 ]; then
+  echo "install virt plugin"
+  kubectl krew install virt
+fi
+
 export VERSION=$(curl -s https://storage.googleapis.com/kubevirt-prow/release/kubevirt/kubevirt/stable.txt)
 echo "install kubevirt operator version ${VERSION}"
 kubectl create -f https://github.com/kubevirt/kubevirt/releases/download/${VERSION}/kubevirt-operator.yaml
@@ -32,4 +46,29 @@ if [ $? -eq 1 ]; then
   exit 1
 fi
 
-kubectl krew install virt
+#set feature gates
+
+kubectl apply -f files/manifests/kubevirt/config/feature_gates.yaml --server-side
+
+#install multus
+
+kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset-thick.yml
+
+function install_mactap_cni() {
+  #install mactap-cni in default
+  kubectl apply -f https://raw.githubusercontent.com/kubevirt/macvtap-cni/main/manifests/macvtap.yaml
+
+  #setup basic bridge requiremnets
+  kubectl apply -f files/manifests/kubevirt/mactap
+
+  #create mactap binding
+  kubectl patch kubevirts -n kubevirt kubevirt --type=json -p='[{"op": "add", "path": "/spec/configuration/network",   "value": {
+            "binding": {
+                "macvtap": {
+                    "domainAttachmentType": "tap"
+                }
+            }
+        }}]'
+}
+
+echo done for now