Merge pull request kubevirt#144 from rmohr/migration

Finalize migration flow

cluster/migration.yaml

@@ -1,6 +1,7 @@
 apiVersion: kubevirt.io/v1alpha1
 kind: Migration
 metadata:
-  name: testvm-migration
+  generateName: testvm-migration
 spec:
-  migratingVMName: testvm
+  selector:
+    name: testvm

cluster/vagrant/setup_kubernetes_common.sh

@@ -50,7 +50,7 @@ repo_gpgcheck=1
 gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg
        https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
 EOF
-yum install --nogpgcheck -y docker kubelet kubeadm kubectl kubernetes-cni
+yum install -y docker kubelet kubeadm kubectl kubernetes-cni
 
 # To get the qemu user and libvirt
 yum install -y qemu-common qemu-kvm qemu-system-x86 libcgroup-tools libvirt || :

cluster/vm.json

@@ -5,17 +5,16 @@
   "apiVersion": "kubevirt.io/v1alpha1",
   "kind": "VM",
   "spec": {
-    "nodeSelector": {
-      "kubernetes.io/hostname": "master"
-    },
     "domain": {
       "devices": {
         "disks": [
           {
             "device": "disk",
             "driver": {
               "name": "qemu",
-              "type": "raw"
+              "type": "raw",
+              "cache": "none"
+
             },
             "snapshot": "external",
             "source": {

cluster/vm.yaml

@@ -53,5 +53,3 @@ spec:
       type:
         os: hvm
     type: qemu
-  nodeSelector:
-    kubernetes.io/hostname: master

cmd/virt-controller/virt-controller.go

@@ -86,7 +86,7 @@ func main() {
 	go migrationController.Run(1, stop)
 	migrationController.WaitForSync(stop)
 
-	_, jobController := watch.NewJobController(vmService, nil, restClient)
+	_, jobController := watch.NewJobController(vmService, nil, clientSet, restClient)
 	jobController.StartInformer(stop)
 	go jobController.Run(1, stop)
 	jobController.WaitForSync(stop)

pkg/api/v1/types.go

@@ -202,10 +202,12 @@ const (
 )
 
 const (
-	AppLabel      string = "kubevirt.io/app"
-	DomainLabel   string = "kubevirt.io/domain"
-	UIDLabel      string = "kubevirt.io/vmUID"
-	NodeNameLabel string = "kubevirt.io/nodeName"
+	AppLabel          string = "kubevirt.io/app"
+	DomainLabel       string = "kubevirt.io/domain"
+	VMUIDLabel        string = "kubevirt.io/vmUID"
+	NodeNameLabel     string = "kubevirt.io/nodeName"
+	MigrationUIDLabel string = "kubevirt.io/migrationUID"
+	MigrationLabel    string = "kubevirt.io/migration"
 )
 
 func NewVM(name string, uid types.UID) *VM {
@@ -294,7 +296,7 @@ func NewSpice(vmName string) *Spice {
 func NewMinimalMigration(name string, vmName string) *Migration {
 	migration := NewMigrationReferenceFromName(name)
 	migration.Spec = MigrationSpec{
-		MigratingVMName: vmName,
+		Selector: VMSelector{vmName},
 	}
 	return migration
 }
@@ -326,10 +328,14 @@ type Migration struct {
 }
 
 // MigrationSpec is a description of a VM Migration
+// For example "destinationNodeName": "testvm" will migrate a VM called "testvm" in the namespace "default"
 type MigrationSpec struct {
-	// The Kubernetes name of the Virtual Machine object to select for one migration.
-	// For example "destinationNodeName": "testvm" will migrate a VM called "testvm" in the namespace "default"
-	MigratingVMName string `json:"migratingVMName,omitempty"`
+	// Criterias for selecting the VM to migrate.
+	// For example
+	// selector:
+	//   name: testvm
+	// will select the VM `testvm` for migration
+	Selector VMSelector `json:"selector"`
 	// Criteria to use when selecting the destination for the migration
 	// for example, to select by the hostname, specify `kubernetes.io/hostname: master`
 	// other possible choices include the hardware required to run the vm or
@@ -340,7 +346,14 @@ type MigrationSpec struct {
 	// randomGenerator: superfastdevice,
 	// app: mysql,
 	// licensedForServiceX: true
-	DestinationNodeSelector map[string]string `json:"destinationNodeSelector,omitempty"`
+	// Note that these selectors are additions to the node selectors on the VM itself and they must not exist on the VM.
+	// If they are conflicting with the VM, no migration will be started.
+	NodeSelector map[string]string `json:"nodeSelector,omitempty"`
+}
+
+type VMSelector struct {
+	// Name of the VM to migrate
+	Name string `json:"name" valid:"required"`
 }
 
 type MigrationPhase string
@@ -353,7 +366,7 @@ const (
 	MigrationPending MigrationPhase = "Pending"
 
 	// Migration is actively progressing
-	MigrationInProgress MigrationPhase = "In Progress"
+	MigrationInProgress MigrationPhase = "InProgress"
 
 	// Migration has completed successfully
 	MigrationSucceeded MigrationPhase = "Succeeded"

pkg/api/v1/types_swagger_generated.go

@@ -45,9 +45,15 @@ func (Migration) SwaggerDoc() map[string]string {
 
 func (MigrationSpec) SwaggerDoc() map[string]string {
 	return map[string]string{
-		"":                        "MigrationSpec is a description of a VM Migration",
-		"migratingVMName":         "The Kubernetes name of the Virtual Machine object to select for one migration.\nFor example \"destinationNodeName\": \"testvm\" will migrate a VM called \"testvm\" in the namespace \"default\"",
-		"destinationNodeSelector": "Criteria to use when selecting the destination for the migration\nfor example, to select by the hostname, specify `kubernetes.io/hostname: master`\nother possible choices include the hardware required to run the vm or\nor lableing of the nodes to indicate their roles in larger applications.\nexamples:\ndisktype: ssd,\nrandomGenerator: /dev/random,\nrandomGenerator: superfastdevice,\napp: mysql,\nlicensedForServiceX: true",
+		"":             "MigrationSpec is a description of a VM Migration\nFor example \"destinationNodeName\": \"testvm\" will migrate a VM called \"testvm\" in the namespace \"default\"",
+		"selector":     "Criterias for selecting the VM to migrate.\nFor example\nselector:\n  name: testvm\nwill select the VM `testvm` for migration",
+		"nodeSelector": "Criteria to use when selecting the destination for the migration\nfor example, to select by the hostname, specify `kubernetes.io/hostname: master`\nother possible choices include the hardware required to run the vm or\nor lableing of the nodes to indicate their roles in larger applications.\nexamples:\ndisktype: ssd,\nrandomGenerator: /dev/random,\nrandomGenerator: superfastdevice,\napp: mysql,\nlicensedForServiceX: true\nNote that these selectors are additions to the node selectors on the VM itself and they must not exist on the VM.\nIf they are conflicting with the VM, no migration will be started.",
+	}
+}
+
+func (VMSelector) SwaggerDoc() map[string]string {
+	return map[string]string{
+		"name": "Name of the VM to migrate",
 	}
 }
 

pkg/virt-controller/services/generated_mock_vm.go

@@ -6,8 +6,7 @@ package services
 import (
 	gomock "github.com/golang/mock/gomock"
 	v1 "k8s.io/client-go/pkg/api/v1"
-	v10 "k8s.io/client-go/pkg/apis/batch/v1"
-	v11 "kubevirt.io/kubevirt/pkg/api/v1"
+	v10 "kubevirt.io/kubevirt/pkg/api/v1"
 )
 
 // Mock of VMService interface
@@ -31,7 +30,7 @@ func (_m *MockVMService) EXPECT() *_MockVMServiceRecorder {
 	return _m.recorder
 }
 
-func (_m *MockVMService) StartVMPod(_param0 *v11.VM) error {
+func (_m *MockVMService) StartVMPod(_param0 *v10.VM) error {
 	ret := _m.ctrl.Call(_m, "StartVMPod", _param0)
 	ret0, _ := ret[0].(error)
 	return ret0
@@ -41,7 +40,7 @@ func (_mr *_MockVMServiceRecorder) StartVMPod(arg0 interface{}) *gomock.Call {
 	return _mr.mock.ctrl.RecordCall(_mr.mock, "StartVMPod", arg0)
 }
 
-func (_m *MockVMService) DeleteVMPod(_param0 *v11.VM) error {
+func (_m *MockVMService) DeleteVMPod(_param0 *v10.VM) error {
 	ret := _m.ctrl.Call(_m, "DeleteVMPod", _param0)
 	ret0, _ := ret[0].(error)
 	return ret0
@@ -51,7 +50,7 @@ func (_mr *_MockVMServiceRecorder) DeleteVMPod(arg0 interface{}) *gomock.Call {
 	return _mr.mock.ctrl.RecordCall(_mr.mock, "DeleteVMPod", arg0)
 }
 
-func (_m *MockVMService) GetRunningVMPods(_param0 *v11.VM) (*v1.PodList, error) {
+func (_m *MockVMService) GetRunningVMPods(_param0 *v10.VM) (*v1.PodList, error) {
 	ret := _m.ctrl.Call(_m, "GetRunningVMPods", _param0)
 	ret0, _ := ret[0].(*v1.PodList)
 	ret1, _ := ret[1].(error)
@@ -62,17 +61,17 @@ func (_mr *_MockVMServiceRecorder) GetRunningVMPods(arg0 interface{}) *gomock.Ca
 	return _mr.mock.ctrl.RecordCall(_mr.mock, "GetRunningVMPods", arg0)
 }
 
-func (_m *MockVMService) DeleteMigration(_param0 *v11.Migration) error {
-	ret := _m.ctrl.Call(_m, "DeleteMigration", _param0)
+func (_m *MockVMService) DeleteMigrationTargetPods(_param0 *v10.Migration) error {
+	ret := _m.ctrl.Call(_m, "DeleteMigrationTargetPods", _param0)
 	ret0, _ := ret[0].(error)
 	return ret0
 }
 
-func (_mr *_MockVMServiceRecorder) DeleteMigration(arg0 interface{}) *gomock.Call {
-	return _mr.mock.ctrl.RecordCall(_mr.mock, "DeleteMigration", arg0)
+func (_mr *_MockVMServiceRecorder) DeleteMigrationTargetPods(arg0 interface{}) *gomock.Call {
+	return _mr.mock.ctrl.RecordCall(_mr.mock, "DeleteMigrationTargetPods", arg0)
 }
 
-func (_m *MockVMService) GetRunningMigrationPods(_param0 *v11.Migration) (*v1.PodList, error) {
+func (_m *MockVMService) GetRunningMigrationPods(_param0 *v10.Migration) (*v1.PodList, error) {
 	ret := _m.ctrl.Call(_m, "GetRunningMigrationPods", _param0)
 	ret0, _ := ret[0].(*v1.PodList)
 	ret1, _ := ret[1].(error)
@@ -83,7 +82,7 @@ func (_mr *_MockVMServiceRecorder) GetRunningMigrationPods(arg0 interface{}) *go
 	return _mr.mock.ctrl.RecordCall(_mr.mock, "GetRunningMigrationPods", arg0)
 }
 
-func (_m *MockVMService) SetupMigration(migration *v11.Migration, vm *v11.VM) error {
+func (_m *MockVMService) SetupMigration(migration *v10.Migration, vm *v10.VM) error {
 	ret := _m.ctrl.Call(_m, "SetupMigration", migration, vm)
 	ret0, _ := ret[0].(error)
 	return ret0
@@ -93,7 +92,7 @@ func (_mr *_MockVMServiceRecorder) SetupMigration(arg0, arg1 interface{}) *gomoc
 	return _mr.mock.ctrl.RecordCall(_mr.mock, "SetupMigration", arg0, arg1)
 }
 
-func (_m *MockVMService) UpdateMigration(migration *v11.Migration) error {
+func (_m *MockVMService) UpdateMigration(migration *v10.Migration) error {
 	ret := _m.ctrl.Call(_m, "UpdateMigration", migration)
 	ret0, _ := ret[0].(error)
 	return ret0
@@ -103,30 +102,43 @@ func (_mr *_MockVMServiceRecorder) UpdateMigration(arg0 interface{}) *gomock.Cal
 	return _mr.mock.ctrl.RecordCall(_mr.mock, "UpdateMigration", arg0)
 }
 
-func (_m *MockVMService) FetchVM(vmName string) (*v11.VM, error) {
+func (_m *MockVMService) FetchVM(vmName string) (*v10.VM, bool, error) {
 	ret := _m.ctrl.Call(_m, "FetchVM", vmName)
-	ret0, _ := ret[0].(*v11.VM)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
+	ret0, _ := ret[0].(*v10.VM)
+	ret1, _ := ret[1].(bool)
+	ret2, _ := ret[2].(error)
+	return ret0, ret1, ret2
 }
 
 func (_mr *_MockVMServiceRecorder) FetchVM(arg0 interface{}) *gomock.Call {
 	return _mr.mock.ctrl.RecordCall(_mr.mock, "FetchVM", arg0)
 }
 
-func (_m *MockVMService) StartMigration(vm *v11.VM, sourceNode *v1.Node, targetNode *v1.Node) error {
-	ret := _m.ctrl.Call(_m, "StartMigration", vm, sourceNode, targetNode)
+func (_m *MockVMService) FetchMigration(migrationName string) (*v10.Migration, bool, error) {
+	ret := _m.ctrl.Call(_m, "FetchMigration", migrationName)
+	ret0, _ := ret[0].(*v10.Migration)
+	ret1, _ := ret[1].(bool)
+	ret2, _ := ret[2].(error)
+	return ret0, ret1, ret2
+}
+
+func (_mr *_MockVMServiceRecorder) FetchMigration(arg0 interface{}) *gomock.Call {
+	return _mr.mock.ctrl.RecordCall(_mr.mock, "FetchMigration", arg0)
+}
+
+func (_m *MockVMService) StartMigration(migration *v10.Migration, vm *v10.VM, sourceNode *v1.Node, targetNode *v1.Node) error {
+	ret := _m.ctrl.Call(_m, "StartMigration", migration, vm, sourceNode, targetNode)
 	ret0, _ := ret[0].(error)
 	return ret0
 }
 
-func (_mr *_MockVMServiceRecorder) StartMigration(arg0, arg1, arg2 interface{}) *gomock.Call {
-	return _mr.mock.ctrl.RecordCall(_mr.mock, "StartMigration", arg0, arg1, arg2)
+func (_mr *_MockVMServiceRecorder) StartMigration(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
+	return _mr.mock.ctrl.RecordCall(_mr.mock, "StartMigration", arg0, arg1, arg2, arg3)
 }
 
-func (_m *MockVMService) GetMigrationJob(vm *v11.VM) (*v10.Job, bool, error) {
-	ret := _m.ctrl.Call(_m, "GetMigrationJob", vm)
-	ret0, _ := ret[0].(*v10.Job)
+func (_m *MockVMService) GetMigrationJob(migration *v10.Migration) (*v1.Pod, bool, error) {
+	ret := _m.ctrl.Call(_m, "GetMigrationJob", migration)
+	ret0, _ := ret[0].(*v1.Pod)
 	ret1, _ := ret[1].(bool)
 	ret2, _ := ret[2].(error)
 	return ret0, ret1, ret2

pkg/virt-controller/services/template.go

@@ -5,18 +5,15 @@ import (
 	"strconv"
 	"strings"
 
-	metav1 "k8s.io/client-go/pkg/apis/meta/v1"
-
 	kubev1 "k8s.io/client-go/pkg/api/v1"
-	batchv1 "k8s.io/client-go/pkg/apis/batch/v1"
 	"kubevirt.io/kubevirt/pkg/api/v1"
 	"kubevirt.io/kubevirt/pkg/logging"
 	"kubevirt.io/kubevirt/pkg/precond"
 )
 
 type TemplateService interface {
 	RenderLaunchManifest(*v1.VM) (*kubev1.Pod, error)
-	RenderMigrationJob(*v1.VM, *kubev1.Node, *kubev1.Node) (*batchv1.Job, error)
+	RenderMigrationJob(*v1.VM, *kubev1.Node, *kubev1.Node) (*kubev1.Pod, error)
 }
 
 type templateService struct {
@@ -56,7 +53,7 @@ func (t *templateService) RenderLaunchManifest(vm *v1.VM) (*kubev1.Pod, error) {
 			Labels: map[string]string{
 				v1.AppLabel:    "virt-launcher",
 				v1.DomainLabel: domain,
-				v1.UIDLabel:    uid,
+				v1.VMUIDLabel:  uid,
 			},
 		},
 		Spec: kubev1.PodSpec{
@@ -69,7 +66,7 @@ func (t *templateService) RenderLaunchManifest(vm *v1.VM) (*kubev1.Pod, error) {
 	return &pod, nil
 }
 
-func (t *templateService) RenderMigrationJob(vm *v1.VM, sourceNode *kubev1.Node, targetNode *kubev1.Node) (*batchv1.Job, error) {
+func (t *templateService) RenderMigrationJob(vm *v1.VM, sourceNode *kubev1.Node, targetNode *kubev1.Node) (*kubev1.Pod, error) {
 	srcAddr := ""
 	dstAddr := ""
 	for _, addr := range sourceNode.Status.Addresses {
@@ -98,29 +95,22 @@ func (t *templateService) RenderMigrationJob(vm *v1.VM, sourceNode *kubev1.Node,
 	}
 	destUri := fmt.Sprintf("qemu+tcp://%s/system", dstAddr)
 
-	job := batchv1.Job{
+	job := kubev1.Pod{
 		ObjectMeta: kubev1.ObjectMeta{
 			GenerateName: "virt-migration",
 			Labels: map[string]string{
 				v1.DomainLabel: vm.GetObjectMeta().GetName(),
+				v1.AppLabel:    "migration",
 			},
 		},
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Job",
-			APIVersion: "batch/v1",
-		},
-		Spec: batchv1.JobSpec{
-			Template: kubev1.PodTemplateSpec{
-				Spec: kubev1.PodSpec{
-					RestartPolicy: kubev1.RestartPolicyNever,
-					Containers: []kubev1.Container{
-						{
-							Name:  "virt-migration",
-							Image: "kubevirt/virt-handler:devel",
-							Command: []string{
-								"virsh", "-c", srcUri, "migrate", "--tunnelled", "--p2p", vm.Spec.Domain.Name, destUri,
-							},
-						},
+		Spec: kubev1.PodSpec{
+			RestartPolicy: kubev1.RestartPolicyNever,
+			Containers: []kubev1.Container{
+				{
+					Name:  "virt-migration",
+					Image: "kubevirt/virt-handler:devel",
+					Command: []string{
+						"virsh", "-c", srcUri, "migrate", "--tunnelled", "--p2p", vm.Spec.Domain.Name, destUri,
 					},
 				},
 			},

pkg/virt-controller/services/template_test.go

@@ -27,7 +27,7 @@ var _ = Describe("Template", func() {
 				Expect(pod.ObjectMeta.Labels).To(Equal(map[string]string{
 					v1.AppLabel:    "virt-launcher",
 					v1.DomainLabel: "testvm",
-					v1.UIDLabel:    "1234",
+					v1.VMUIDLabel:  "1234",
 				}))
 				Expect(pod.ObjectMeta.GenerateName).To(Equal("virt-launcher-testvm-----"))
 				Expect(pod.Spec.NodeSelector).To(BeEmpty())
@@ -49,7 +49,7 @@ var _ = Describe("Template", func() {
 				Expect(pod.ObjectMeta.Labels).To(Equal(map[string]string{
 					v1.AppLabel:    "virt-launcher",
 					v1.DomainLabel: "testvm",
-					v1.UIDLabel:    "1234",
+					v1.VMUIDLabel:  "1234",
 				}))
 				Expect(pod.ObjectMeta.GenerateName).To(Equal("virt-launcher-testvm-----"))
 				Expect(pod.Spec.NodeSelector).To(Equal(map[string]string{
@@ -105,7 +105,7 @@ var _ = Describe("Template", func() {
 
 					job, err := svc.RenderMigrationJob(vm, &srcNodeIp, &destNodeIp)
 					Expect(err).ToNot(HaveOccurred())
-					Expect(job.Spec.Template.Spec.RestartPolicy).To(Equal(kubev1.RestartPolicyNever))
+					Expect(job.Spec.RestartPolicy).To(Equal(kubev1.RestartPolicyNever))
 				})
 				It("should use the first ip it finds", func() {
 					vm := v1.NewMinimalVM("testvm")
@@ -114,7 +114,7 @@ var _ = Describe("Template", func() {
 					refCommand := []string{
 						"virsh", "-c", "qemu+tcp://127.0.0.2/system", "migrate", "--tunnelled", "--p2p", "testvm",
 						"qemu+tcp://127.0.0.3/system"}
-					Expect(job.Spec.Template.Spec.Containers[0].Command).To(Equal(refCommand))
+					Expect(job.Spec.Containers[0].Command).To(Equal(refCommand))
 				})
 			})
 			Context("migration template with incorrect parameters", func() {