Skip to content

Commit

Permalink
Merge pull request kubevirt#1099 from stu-gott/kvm-monitor
Browse files Browse the repository at this point in the history
Consume KVM devices using device plugin framework
fabiand authored Jun 19, 2018
2 parents a1069b3 + 2f6dc5f commit d265c3c
Showing 510 changed files with 378,521 additions and 45,695 deletions.
109 changes: 103 additions & 6 deletions api/openapi-spec/swagger.json
Original file line number Diff line number Diff line change
@@ -3085,6 +3085,85 @@
}
},
"definitions": {
"big.Int": {
"required": [
"neg",
"abs"
],
"properties": {
"abs": {
"type": "array",
"items": {
"$ref": "#/definitions/big.Word"
}
},
"neg": {
"type": "boolean"
}
}
},
"big.Word": {},
"inf.Dec": {
"required": [
"unscaled",
"scale"
],
"properties": {
"scale": {
"$ref": "#/definitions/inf.Scale"
},
"unscaled": {
"$ref": "#/definitions/big.Int"
}
}
},
"resource.Quantity": {
"required": [
"i",
"d",
"s",
"Format"
],
"properties": {
"Format": {
"type": "string"
},
"d": {
"$ref": "#/definitions/resource.infDecAmount"
},
"i": {
"$ref": "#/definitions/resource.int64Amount"
},
"s": {
"type": "string"
}
}
},
"resource.infDecAmount": {
"required": [
"Dec"
],
"properties": {
"Dec": {
"$ref": "#/definitions/inf.Dec"
}
}
},
"resource.int64Amount": {
"required": [
"value",
"scale"
],
"properties": {
"scale": {
"$ref": "#/definitions/resource.Scale"
},
"value": {
"type": "integer",
"format": "int64"
}
}
},
"types.UID": {},
"v1.APIGroup": {
"description": "APIGroup contains the name, the supported versions, and the preferred version of a group.",
@@ -3760,7 +3839,10 @@
},
"matchLabels": {
"description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.",
"type": "object"
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
},
@@ -3936,7 +4018,10 @@
"properties": {
"annotations": {
"description": "Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations",
"type": "object"
"type": "object",
"additionalProperties": {
"type": "string"
}
},
"clusterName": {
"description": "The name of the cluster which the object belongs to. This is used to distinguish resources with same name and namespace in different clusters. This field is not set anywhere right now and apiserver is going to ignore it if set in create or update request.",
@@ -3977,7 +4062,10 @@
},
"labels": {
"description": "Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels",
"type": "object"
"type": "object",
"additionalProperties": {
"type": "string"
}
},
"name": {
"description": "Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names",
@@ -4201,11 +4289,17 @@
"properties": {
"limits": {
"description": "Limits describes the maximum amount of compute resources allowed.\nValid resource keys are \"memory\" and \"cpu\".\n+optional",
"type": "object"
"type": "object",
"additionalProperties": {
"$ref": "#/definitions/resource.Quantity"
}
},
"requests": {
"description": "Requests is a description of the initial vmi resources.\nValid resource keys are \"memory\" and \"cpu\".\n+optional",
"type": "object"
"type": "object",
"additionalProperties": {
"$ref": "#/definitions/resource.Quantity"
}
}
}
},
@@ -4683,7 +4777,10 @@
},
"nodeSelector": {
"description": "NodeSelector is a selector which must be true for the vmi to fit on a node.\nSelector which must match a node's labels for the vmi to be scheduled on that node.\nMore info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/\n+optional",
"type": "object"
"type": "object",
"additionalProperties": {
"type": "string"
}
},
"subdomain": {
"description": "If specified, the fully qualified vmi hostname will be \"\u003chostname\u003e.\u003csubdomain\u003e.\u003cpod namespace\u003e.svc.\u003ccluster domain\u003e\".\nIf not specified, the vmi will not have a domainname at all. The DNS entry will resolve to the vmi,\nno matter if the vmi itself can pick up a hostname.\n+optional",
13 changes: 12 additions & 1 deletion cmd/virt-handler/virt-handler.go
Original file line number Diff line number Diff line change
@@ -58,13 +58,17 @@ const (
hostOverride = ""

virtShareDir = "/var/run/kubevirt"

// This value is derived from default MaxPods in Kubelet Config
maxDevices = 110
)

type virtHandlerApp struct {
service.ServiceListen
HostOverride string
VirtShareDir string
WatchdogTimeoutDuration time.Duration
MaxDevices int
}

var _ service.Service = &virtHandlerApp{}
@@ -134,6 +138,7 @@ func (app *virtHandlerApp) Run() {
domainSharedInformer,
gracefulShutdownInformer,
int(app.WatchdogTimeoutDuration.Seconds()),
maxDevices,
)

// Bootstrapping. From here on the startup order matters
@@ -152,13 +157,19 @@ func (app *virtHandlerApp) AddFlags() {
app.AddCommonFlags()

flag.StringVar(&app.HostOverride, "hostname-override", hostOverride,
"Name under which the node is registered in kubernetes, where this virt-handler instance is running on")
"Name under which the node is registered in Kubernetes, where this virt-handler instance is running on")

flag.StringVar(&app.VirtShareDir, "kubevirt-share-dir", virtShareDir,
"Shared directory between virt-handler and virt-launcher")

flag.DurationVar(&app.WatchdogTimeoutDuration, "watchdog-timeout", defaultWatchdogTimeout,
"Watchdog file timeout")

// TODO: the Device Plugin API does not allow for infinitely available (shared) devices
// so the current approach is to register an arbitrary number.
// This should be deprecated if the API allows for shared resources in the future
flag.IntVar(&app.MaxDevices, "max-devices", maxDevices,
"Number of devices to register with Kubernetes device plugin framework")
}

func main() {
8 changes: 2 additions & 6 deletions cmd/virt-launcher/entrypoint.sh
Original file line number Diff line number Diff line change
@@ -8,12 +8,8 @@ _term() {

trap _term SIGTERM SIGINT SIGQUIT

# HACK
# Try to create /dev/kvm if not present
if [ ! -e /dev/kvm ]; then
mknod /dev/kvm c 10 $(grep '\<kvm\>' /proc/misc | cut -f 1 -d' ')
fi

# FIXME: The plugin framework doesn't appear to (currently) have a means
# to specify device ownership. This needs to be re-visited if that changes
chown :qemu /dev/kvm
chmod 660 /dev/kvm

4 changes: 2 additions & 2 deletions cmd/virt-launcher/virt-launcher.go
Original file line number Diff line number Diff line change
@@ -228,7 +228,7 @@ func main() {
watchdogInterval := flag.Duration("watchdog-update-interval", defaultWatchdogInterval, "Interval at which watchdog file should be updated")
readinessFile := flag.String("readiness-file", "/tmp/health", "Pod looks for this file to determine when virt-launcher is initialized")
gracePeriodSeconds := flag.Int("grace-period-seconds", 30, "Grace period to observe before sending SIGTERM to vm process")
allowEmulation := flag.Bool("allow-emulation", false, "Allow fallback to emulation if /dev/kvm is not present")
useEmulation := flag.Bool("use-emulation", false, "Use software emulation")

// set new default verbosity, was set to 0 by glog
flag.Set("v", "2")
@@ -261,7 +261,7 @@ func main() {
// Start the virt-launcher command service.
// Clients can use this service to tell virt-launcher
// to start/stop virtual machines
options := cmdserver.NewServerOptions(*allowEmulation)
options := cmdserver.NewServerOptions(*useEmulation)
socketPath := cmdclient.SocketFromNamespaceName(*virtShareDir, *namespace, *name)
startCmdServer(socketPath, domainManager, stopChan, options)

4 changes: 2 additions & 2 deletions docs/env-providers.md
Original file line number Diff line number Diff line change
@@ -26,7 +26,7 @@ Requires:
Usage:

```bash
export KUBEVIRT_PROVIDER=k8s-1.9.3 # choose this provider
export KUBEVIRT_PROVIDER=k8s-1.10.4 # choose this provider
export KUBEVIRT_NUM_NODES=3 # master + two nodes
make cluster-up
```
@@ -52,4 +52,4 @@ make cluster-up
* Create a `cluster/$KUBEVIRT_PROVIDER` directory
* Create a `cluster/$KUBEVIRT_PROVIDER/provider.sh` files
* This file has to contain the functions `up`, `build`, `down` and `_kubectl`
* Have a look at `cluster/k8s-1.9.3/provider.sh` for a reference implementation
* Have a look at `cluster/k8s-1.10.4/provider.sh` for a reference implementation
9 changes: 7 additions & 2 deletions docs/getting-started.md
Original file line number Diff line number Diff line change
@@ -18,14 +18,19 @@ virtualization should be preferred.
Runs master and nodes containers, when each one of them run virtual machine via QEMU.
In additional it runs dnsmasq and docker registry containers.

### Compatibility

The minimum compatible Kubernetes version is 1.10.0. Important features required
for scheduling and memory are missing or incompatible with previous versions.

### Compile and run it

Build all required artifacts and launch the
dockerizied environment:

```bash
# Build and deploy KubeVirt on Kubernetes 1.10.3 in our vms inside containers
export KUBEVIRT_PROVIDER=k8s-1.10.3 # this is also the default if no KUBEVIRT_PROVIDER is set
# Build and deploy KubeVirt on Kubernetes 1.10.4 in our vms inside containers
export KUBEVIRT_PROVIDER=k8s-1.10.4 # this is also the default if no KUBEVIRT_PROVIDER is set
make cluster-up
make cluster-sync
```
18 changes: 8 additions & 10 deletions docs/software-emulation.md
Original file line number Diff line number Diff line change
@@ -5,30 +5,28 @@ This is almost always desirable, however there are a few exceptions where this
approach is problematic. For instance, running KubeVirt on a cluster where the
nodes do not support hardware emulation.

If `allowEmulation` is enabled, hardware emulation via `/dev/kvm` will still be
attempted first. Software emulation will only be used if the device is
unavailable.
If `useEmulation` is enabled, hardware emulation via `/dev/kvm` will not be
attempted. `qemu` will be used for software emulation instead.

# Configuration

Enabling software emulation as a fallback is a cluster-wide setting, and is
activated using a ConfigMap in the `kube-system` namespace. It can be enabled
with the following command:
Enabling software emulation is a cluster-wide setting, and is activated using a
ConfigMap in the `kube-system` namespace. It can be enabled with the following
command:

```bash
cluster/kubectl.sh --namespace kube-system create configmap kubevirt-config \
--from-literal debug.allowEmulation=true
--from-literal debug.useEmulation=true
```

If the `kube-system/kubevirt-config` ConfigMap already exists, the above entry
can be added using:


```bash
cluster/kubectl.sh --namespace kube-system edit configmap kubevirt-config
```

In this case, add the `debug.allowEmulation: "true"` setting to `data`:
In this case, add the `debug.useEmulation: "true"` setting to `data`:

```yaml
apiVersion: v1
@@ -37,7 +35,7 @@ metadata:
name: kubevirt-config
namespace: kube-system
data:
debug.allowEmulation: "true"
debug.useEmulation: "true"

```

Loading

0 comments on commit d265c3c

Please sign in to comment.