do streaming qcow2->raw conversion if we can. also put resource limit…

…s on qemu-img

.gitignore

@@ -7,3 +7,5 @@ release-announcement
 .project
 .vscode
 cluster/.*
+vendor/**
+.history

hack/build/docker/cdi-controller/Dockerfile

@@ -1,4 +1,4 @@
-FROM fedora:27
+FROM fedora:28
 
 COPY ./cdi-controller /usr/bin/cdi-controller
 

hack/build/docker/cdi-importer/Dockerfile

@@ -1,6 +1,6 @@
-FROM fedora:27
+FROM fedora:28
 
-RUN dnf install -y qemu-img
+RUN dnf install -y qemu-img qemu-block-curl
 
 RUN mkdir /data
 

pkg/image/prlimit.go

@@ -0,0 +1,41 @@
+/*
+Copyright 2018 The CDI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package image
+
+import (
+	"syscall"
+	"unsafe"
+
+	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
+)
+
+func setAddressSpaceLimit(pid int, value uint64) error {
+	return prlimit(pid, unix.RLIMIT_AS, &syscall.Rlimit{Cur: value, Max: value})
+}
+
+func setCPUTimeLimit(pid int, value uint64) error {
+	return prlimit(pid, unix.RLIMIT_CPU, &syscall.Rlimit{Cur: value, Max: value})
+}
+
+func prlimit(pid int, limit int, value *syscall.Rlimit) error {
+	_, _, e1 := syscall.RawSyscall6(syscall.SYS_PRLIMIT64, uintptr(pid), uintptr(limit), uintptr(unsafe.Pointer(value)), 0, 0, 0)
+	if e1 != 0 {
+		return errors.Wrapf(e1, "error setting prlimit on %d with value %d on pid %d", limit, value, pid)
+	}
+	return nil
+}

pkg/image/qemu.go

@@ -1,18 +1,81 @@
+/*
+Copyright 2018 The CDI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package image
 
 import (
+	"bytes"
+	"fmt"
+	"net/url"
 	"os/exec"
 
 	"github.com/pkg/errors"
+
+	"github.com/golang/glog"
+)
+
+const (
+	networkTimeoutSecs = 3600    //max is 10000
+	maxMemory          = 1 << 30 //value from OpenStack Nova
+	maxCPUSecs         = 30      //value from OpenStack Nova
 )
 
 // ConvertQcow2ToRaw is a wrapper for qemu-img convert which takes a qcow2 file (specified by src) and converts
 // it to a raw image (written to the provided dest file)
 func ConvertQcow2ToRaw(src, dest string) error {
-	cmd := exec.Command("qemu-img", "convert", "-f", "qcow2", "-O", "raw", src, dest)
-	err := cmd.Run()
+	err := execWithLimits("qemu-img", "convert", "-f", "qcow2", "-O", "raw", src, dest)
+	if err != nil {
+		return errors.Wrap(err, "could not convert local qcow2 image to raw")
+	}
+	return nil
+}
+
+// ConvertQcow2ToRawStream converts an http accessible qcow2 image to raf format without locally caching the qcow2 image
+func ConvertQcow2ToRawStream(url *url.URL, dest string) error {
+	jsonArg := fmt.Sprintf("json: {\"file.driver\": \"%s\", \"file.url\": \"%s\", \"file.timeout\": %d}", url.Scheme, url, networkTimeoutSecs)
+	err := execWithLimits("qemu-img", "convert", "-f", "qcow2", "-O", "raw", jsonArg, dest)
+	if err != nil {
+		return errors.Wrap(err, "could not stream/convert qcow2 image to raw")
+	}
+	return nil
+}
+
+func execWithLimits(command string, args ...string) error {
+	var buf bytes.Buffer
+	cmd := exec.Command(command, args...)
+	cmd.Stdout = &buf
+	cmd.Stderr = &buf
+	err := cmd.Start()
+	if err != nil {
+		return errors.Wrapf(err, "Couldn't start %s", command)
+	}
+	defer cmd.Process.Kill()
+	err = setAddressSpaceLimit(cmd.Process.Pid, maxMemory)
+	if err != nil {
+		return errors.Wrap(err, "Couldn't set address space limit")
+	}
+	err = setCPUTimeLimit(cmd.Process.Pid, maxCPUSecs)
+	if err != nil {
+		return errors.Wrap(err, "Couldn't set CPU time limit")
+	}
+	err = cmd.Wait()
 	if err != nil {
-		return errors.Wrap(err, "could not convert qcow2 image to raw")
+		glog.Errorf("%s %s failed output is:\n", command, args)
+		glog.Errorf("%s\n", buf.String())
+		return errors.Wrapf(err, "%s execution failed", command)
 	}
 	return nil
 }

pkg/importer/dataStream.go

@@ -1,3 +1,19 @@
+/*
+Copyright 2018 The CDI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package importer
 
 // DEPRECATION NOTICE: Support for local (file://) endpoints will be removed from CDI in the next
@@ -500,8 +516,18 @@ func closeReaders(readers []reader) (rtnerr error) {
 	return rtnerr
 }
 
+func (d DataStream) isHTTPQcow2() bool {
+	// assuming len(d.Readers) == 3 is [http, mr, mr]
+	// should prob get rid of the redundant MultiReader
+	return (d.url.Scheme == "http" || d.url.Scheme == "https") && d.qemu && len(d.Readers) == 3
+}
+
 // Copy endpoint to dest based on passed-in reader.
 func (d DataStream) copy(dest string) error {
+	if d.isHTTPQcow2() {
+		glog.V(3).Infoln("Doing streaming qcow2 to raw conversion")
+		return image.ConvertQcow2ToRawStream(d.url, dest)
+	}
 	return copy(d.topReader(), dest, d.qemu)
 }