Merge pull request kubevirt#8304 from oshoval/update_sono

sonobuoy: Collect whole artifacts directory

docs/conformance.md

@@ -56,6 +56,11 @@ Failed: 0
 Skipped: 579
 ```
 
+### Debugging failures
+
+In case of a failure the artifacts will be collected as well.  
+Find them at `plugins/kubevirt-conformance/results/global/k8s-reporter` of the tar file.
+
 ## Executing conformance tests in the development environment
 
 ```bash

hack/builder/Dockerfile

@@ -7,7 +7,7 @@ ENV BAZEL_VERSION=4.2.1
 ENV GIMME_GO_VERSION=1.17.8
 ENV GRADLE_VERSION=6.6
 ENV OPERATOR_COURIER_VERSION=2.1.11
-ENV SONOBUOY_VERSION=0.19.0
+ENV SONOBUOY_VERSION=0.56.9
 ENV GOLANGCI_LINT_VERSION=v1.46.2
 
 ENV KUBEVIRT_CREATE_BAZELRCS=false

hack/conformance.sh

@@ -38,15 +38,17 @@ echo 'Downloading report about the test execution'
 results_archive=${ARTIFACTS}/$(cd ${ARTIFACTS} && sonobuoy retrieve)
 
 echo 'Results:'
-sonobuoy results ${results_archive}
+RES=$(sonobuoy results ${results_archive})
+echo "$RES"
 
 echo "Extracting the full report and keep it under ${ARTIFACTS}"
 tar xf ${results_archive} -C ${ARTIFACTS}
 cp ${ARTIFACTS}/plugins/kubevirt-conformance/results/global/junit.xml ${ARTIFACTS}/junit.conformance.xml
 
 echo 'Evaluating success of the test run'
-sonobuoy results ${results_archive} | grep -q 'Status: passed' || {
+if ! echo "$RES" | grep -q 'Status: passed'; then
     echo 'Conformance suite has failed'
     exit 1
-}
+fi
+
 echo 'Conformance suite has successfully completed'

hack/dockerized

@@ -12,7 +12,7 @@ fi
 
 fail_if_cri_bin_missing
 
-KUBEVIRT_BUILDER_IMAGE="quay.io/kubevirt/builder:2209141252-da37250e4"
+KUBEVIRT_BUILDER_IMAGE="quay.io/kubevirt/builder:2209281119-14a6cb40f"
 
 SYNC_OUT=${SYNC_OUT:-true}
 

manifests/release/conformance.yaml.in

@@ -6,8 +6,5 @@ spec:
   command:
   - /usr/bin/conformance
   image: {{.DockerPrefix}}/conformance:{{.DockerTag}}
+  imagePullPolicy: Always
   name: plugin
-  resources: {}
-  volumeMounts:
-  - mountPath: /tmp/results
-    name: results

tests/conformance/conformance.go

@@ -11,7 +11,7 @@ import (
 var containerTag = ""
 
 func done(files []string) {
-	err := ioutil.WriteFile("/tmp/results/done", []byte(strings.Join(files, "\n")), 0666)
+	err := ioutil.WriteFile("/tmp/sonobuoy/results/done", []byte(strings.Join(files, "\n")), 0666)
 	if err != nil {
 		fmt.Printf("Failed to notify sonobuoy that I am done: %v\n", err)
 	}
@@ -20,7 +20,7 @@ func done(files []string) {
 func main() {
 	err := execute()
 	done([]string{
-		"/tmp/results/junit.xml",
+		"/tmp/sonobuoy/results",
 	})
 	if err != nil {
 		os.Exit(1)
@@ -30,7 +30,7 @@ func main() {
 func execute() error {
 	args := []string{}
 	args = append(args, "--container-tag", containerTag)
-	args = append(args, "--junit-output", "/tmp/results/junit.xml")
+	args = append(args, "--junit-output", "/tmp/sonobuoy/results/junit.xml")
 	// additional conformance test overrides
 	if value, exists := os.LookupEnv("E2E_SKIP"); exists {
 		args = append(args, "--ginkgo.skip", value)
@@ -52,7 +52,7 @@ func execute() error {
 
 	cmd := exec.Command("/usr/bin/go_default_test", args...)
 	cmd.Env = os.Environ()
-	cmd.Env = append(cmd.Env, "ARTIFACTS=/tmp/results/")
+	cmd.Env = append(cmd.Env, "ARTIFACTS=/tmp/sonobuoy/results/")
 	cmd.Stderr = os.Stderr
 	cmd.Stdin = os.Stdin
 	cmd.Stdout = os.Stdout