Skip to content

Commit 6c3efde

Browse files
authored
KO 276 useful integ tests (#171)
* Start translating lifecycle tests into ginkgo tests * Build out more helper libraries for running kubectl and dumping k8s logs
1 parent 7d5e04b commit 6c3efde

File tree

14 files changed

+563
-56
lines changed

14 files changed

+563
-56
lines changed

go.sum

+3
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,10 @@ github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBv
3434
github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
3535
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
3636
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
37+
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
3738
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
3839
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
40+
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
3941
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
4042
github.com/magefile/mage v1.9.0 h1:t3AU2wNwehMCW97vuqQLtw6puppWXHO+O2MHo5a50XE=
4143
github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
@@ -89,6 +91,7 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
8991
golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
9092
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
9193
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
94+
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
9295
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
9396
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
9497
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=

mage/ginkgo/lib.go

+114
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
package ginkgo_util
2+
3+
import (
4+
"fmt"
5+
"regexp"
6+
"time"
7+
8+
ginkgo "github.com/onsi/ginkgo"
9+
. "github.com/onsi/gomega"
10+
11+
"github.com/riptano/dse-operator/mage/kubectl"
12+
mageutil "github.com/riptano/dse-operator/mage/util"
13+
)
14+
15+
// Wrapper type to make it simpler to
16+
// set a namespace one time and execute all of your
17+
// KCmd objects inside of it, and then use Gomega
18+
// assertions on panic
19+
type NsWrapper struct {
20+
Namespace string
21+
TestSuiteName string
22+
LogDir string
23+
stepCounter int
24+
}
25+
26+
func NewWrapper(suiteName string, namespace string) NsWrapper {
27+
return NsWrapper{
28+
Namespace: namespace,
29+
TestSuiteName: suiteName,
30+
LogDir: genSuiteLogDir(suiteName),
31+
stepCounter: 1,
32+
}
33+
34+
}
35+
36+
func (k NsWrapper) ExecV(kcmd kubectl.KCmd) error {
37+
err := kcmd.InNamespace(k.Namespace).ExecV()
38+
return err
39+
}
40+
41+
func (k NsWrapper) ExecVPanic(kcmd kubectl.KCmd) {
42+
err := kcmd.InNamespace(k.Namespace).ExecV()
43+
Expect(err).ToNot(HaveOccurred())
44+
}
45+
46+
func (k NsWrapper) Output(kcmd kubectl.KCmd) (string, error) {
47+
out, err := kcmd.InNamespace(k.Namespace).Output()
48+
return out, err
49+
}
50+
51+
func (k NsWrapper) OutputPanic(kcmd kubectl.KCmd) string {
52+
out, err := kcmd.InNamespace(k.Namespace).Output()
53+
Expect(err).ToNot(HaveOccurred())
54+
return out
55+
}
56+
57+
func (k NsWrapper) WaitForOutput(kcmd kubectl.KCmd, expected string, seconds int) error {
58+
return kubectl.WaitForOutput(kcmd.InNamespace(k.Namespace), expected, seconds)
59+
}
60+
61+
func (k NsWrapper) WaitForOutputPanic(kcmd kubectl.KCmd, expected string, seconds int) {
62+
err := kubectl.WaitForOutput(kcmd.InNamespace(k.Namespace), expected, seconds)
63+
Expect(err).ToNot(HaveOccurred())
64+
}
65+
66+
func (k *NsWrapper) countStep() int {
67+
n := k.stepCounter
68+
k.stepCounter++
69+
return n
70+
}
71+
72+
//===================================
73+
// Logging functions for the NsWrapper
74+
// that execute the Kcmd and then dump
75+
// k8s logs for that namespace
76+
//====================================
77+
func sanitizeForLogDirs(s string) string {
78+
reg, err := regexp.Compile(`[\s\\\/\-\.,]`)
79+
mageutil.PanicOnError(err)
80+
return reg.ReplaceAllLiteralString(s, "_")
81+
}
82+
83+
func genSuiteLogDir(suiteName string) string {
84+
datetime := time.Now().Format("2006.01.02_15:04:05")
85+
return fmt.Sprintf("../../build/kubectl_dump/%s/%s",
86+
sanitizeForLogDirs(suiteName), datetime)
87+
}
88+
89+
func (ns *NsWrapper) genTestLogDir(description string) string {
90+
sanitizedDesc := sanitizeForLogDirs(description)
91+
return fmt.Sprintf("%s/%02d_%s", ns.LogDir, ns.countStep(), sanitizedDesc)
92+
}
93+
94+
func (ns *NsWrapper) ExecAndLog(description string, kcmd kubectl.KCmd) {
95+
ginkgo.By(description)
96+
defer kubectl.DumpLogs(ns.genTestLogDir(description), ns.Namespace).ExecVPanic()
97+
execErr := ns.ExecV(kcmd)
98+
Expect(execErr).ToNot(HaveOccurred())
99+
}
100+
101+
func (ns *NsWrapper) OutputAndLog(description string, kcmd kubectl.KCmd) string {
102+
ginkgo.By(description)
103+
defer kubectl.DumpLogs(ns.genTestLogDir(description), ns.Namespace).ExecVPanic()
104+
output, execErr := ns.Output(kcmd)
105+
Expect(execErr).ToNot(HaveOccurred())
106+
return output
107+
}
108+
109+
func (ns *NsWrapper) WaitForOutputAndLog(description string, kcmd kubectl.KCmd, expected string, seconds int) {
110+
ginkgo.By(description)
111+
defer kubectl.DumpLogs(ns.genTestLogDir(description), ns.Namespace).ExecVPanic()
112+
execErr := ns.WaitForOutput(kcmd, expected, seconds)
113+
Expect(execErr).ToNot(HaveOccurred())
114+
}

mage/go.mod

+2
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ go 1.13
44

55
require (
66
github.com/magefile/mage v1.9.0
7+
github.com/onsi/ginkgo v1.10.1
8+
github.com/onsi/gomega v1.7.0
79
github.com/otiai10/copy v1.0.2
810
github.com/pkg/errors v0.8.1 // indirect
911
github.com/stretchr/testify v1.4.0

mage/go.sum

+6
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
3030
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
3131
github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
3232
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
33+
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
3334
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
3435
github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
3536
github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok=
@@ -56,8 +57,10 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m
5657
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
5758
github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
5859
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
60+
github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
5961
github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
6062
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
63+
github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
6164
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
6265
github.com/otiai10/copy v1.0.2 h1:DDNipYy6RkIkjMwy+AWzgKiNTyj2RUI9yEMeETEpVyc=
6366
github.com/otiai10/copy v1.0.2/go.mod h1:c7RpqBkwMom4bYTSkLSym4VSJz/XtncWRAj/J4PEIMY=
@@ -85,6 +88,7 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ
8588
golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
8689
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
8790
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
91+
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg=
8892
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
8993
golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
9094
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -97,9 +101,11 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+
97101
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
98102
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
99103
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
104+
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
100105
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
101106
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
102107
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
108+
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
103109
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
104110
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
105111
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

mage/integ-tests/lib.go

+66
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
package integutil
2+
3+
import (
4+
"fmt"
5+
"io/ioutil"
6+
"os"
7+
"strings"
8+
9+
shutil "github.com/riptano/dse-operator/mage/sh"
10+
mageutil "github.com/riptano/dse-operator/mage/util"
11+
)
12+
13+
const (
14+
envIntegDir = "M_INTEG_DIR"
15+
envGinkgoNoColor = "M_GINKGO_NOCOLOR"
16+
)
17+
18+
func listTestSuiteDirs() []string {
19+
contents, err := ioutil.ReadDir("./tests")
20+
mageutil.PanicOnError(err)
21+
var testDirs []string
22+
for _, info := range contents {
23+
if info.IsDir() && info.Name() != "testdata" {
24+
testDirs = append(testDirs, info.Name())
25+
}
26+
}
27+
return testDirs
28+
}
29+
30+
/// Prints a comma-delimited list of test suite dirs.
31+
/// This is mainly useful for a CI pipeline.
32+
func List() {
33+
dirs := listTestSuiteDirs()
34+
fmt.Println(strings.Join(dirs, ","))
35+
}
36+
37+
func runGinkgoTests(path string) {
38+
os.Setenv("CGO_ENABLED", "0")
39+
args := []string{"test", "-timeout", "99999s", "-v"}
40+
noColor := os.Getenv(envGinkgoNoColor)
41+
if strings.ToLower(noColor) == "true" {
42+
args = append(args, "--ginkgo.noColor")
43+
}
44+
45+
args = append(args, path)
46+
shutil.RunVPanic("go", args...)
47+
48+
}
49+
50+
/// Run all ginkgo integration tests.
51+
func RunAll() {
52+
runGinkgoTests("./tests/...")
53+
}
54+
55+
/// Run a single ginkgo integration test.
56+
///
57+
/// This target requires that the env var
58+
/// M_INTEG_DIR is set to the desired test
59+
/// suite directory name.
60+
func RunSingle() {
61+
integDir := mageutil.RequireEnv(envIntegDir)
62+
dir := fmt.Sprintf("./tests/%s", integDir)
63+
err := os.Chdir(dir)
64+
mageutil.PanicOnError(err)
65+
runGinkgoTests("./")
66+
}

mage/kind/lib.go

+7-12
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"github.com/magefile/mage/mg"
99
cfgutil "github.com/riptano/dse-operator/mage/config"
1010
dockerutil "github.com/riptano/dse-operator/mage/docker"
11+
integutil "github.com/riptano/dse-operator/mage/integ-tests"
1112
"github.com/riptano/dse-operator/mage/kubectl"
1213
"github.com/riptano/dse-operator/mage/operator"
1314
shutil "github.com/riptano/dse-operator/mage/sh"
@@ -39,8 +40,6 @@ func loadImage(image string) {
3940
shutil.RunVPanic("kind", "load", "docker-image", image)
4041
}
4142

42-
// Install Kind.
43-
//
4443
// Currently there is no concept of "global tool install"
4544
// with the go cli. With the new module system, your project's
4645
// go.mod and go.sum files will be updated with new dependencies
@@ -80,21 +79,17 @@ func SetupEmptyCluster() {
8079
deleteCluster()
8180
createCluster()
8281
kubectl.ClusterInfoForContext("kind-kind").ExecVPanic()
82+
kubectl.ApplyFiles("operator/deploy/kind/rancher-local-path-storage.yaml").
83+
ExecVPanic()
84+
//TODO make this part optional
85+
operator.BuildDocker()
86+
loadImage(operatorImage)
8387
}
8488

8589
/// Run all Ginkgo integration tests.
8690
func RunIntegTests() {
8791
mg.Deps(SetupEmptyCluster)
88-
os.Setenv("CGO_ENABLED", "0")
89-
90-
args := []string{"test", "-v", "./tests/..."}
91-
noColor := os.Getenv("GINKGO_NOCOLOR")
92-
if strings.ToLower(noColor) == "true" {
93-
args = append(args, "-ginkgo.noColor")
94-
}
95-
96-
shutil.RunVPanic("go", args...)
97-
92+
integutil.RunAll()
9893
err := deleteCluster()
9994
mageutil.PanicOnError(err)
10095
}

mage/kubectl/lib.go

+69
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ package kubectl
22

33
import (
44
"fmt"
5+
"os"
6+
"time"
57

68
shutil "github.com/riptano/dse-operator/mage/sh"
79
)
@@ -69,11 +71,21 @@ func (k KCmd) WithFlag(name string, value string) KCmd {
6971
return k
7072
}
7173

74+
func (k KCmd) WithLabel(label string) KCmd {
75+
k.Args = append(k.Args, "-l", label)
76+
return k
77+
}
78+
7279
func ClusterInfoForContext(ctxt string) KCmd {
7380
args := []string{"--context", ctxt}
7481
return KCmd{Command: "cluster-info", Args: args}
7582
}
7683

84+
func CreateNamespace(namespace string) KCmd {
85+
args := []string{"namespace", namespace}
86+
return KCmd{Command: "create", Args: args}
87+
}
88+
7789
func CreateSecretLiteral(name string, user string, pw string) KCmd {
7890
args := []string{"secret", "generic", name}
7991
flags := map[string]string{
@@ -139,3 +151,60 @@ func ApplyFiles(paths ...string) KCmd {
139151
}
140152
return KCmd{Command: "apply", Args: args}
141153
}
154+
155+
func PatchMerge(resource string, data string) KCmd {
156+
args := []string{resource, "--patch", data, "--type", "merge"}
157+
return KCmd{Command: "patch", Args: args}
158+
}
159+
160+
func WaitForOutput(k KCmd, expected string, seconds int) error {
161+
c := make(chan string)
162+
timer := time.NewTimer(time.Duration(seconds) * time.Second)
163+
cquit := make(chan bool)
164+
defer close(cquit)
165+
166+
var actual string
167+
var err error
168+
169+
go func() {
170+
for actual != expected {
171+
select {
172+
case <-cquit:
173+
return
174+
default:
175+
actual, err = k.Output()
176+
// Execute at most once every second
177+
time.Sleep(time.Second)
178+
}
179+
}
180+
c <- actual
181+
}()
182+
183+
select {
184+
case <-timer.C:
185+
msg := fmt.Sprintf("Timed out waiting for value. Expected %s, but got %s.", expected, actual)
186+
if err != nil {
187+
msg = fmt.Sprintf("%s\nThe following error occured while querying k8s: %v", msg, err)
188+
}
189+
e := fmt.Errorf(msg)
190+
return e
191+
case <-c:
192+
return nil
193+
}
194+
}
195+
196+
func DumpAllLogs(path string) KCmd {
197+
//Make dir if doesn't exist
198+
_ = os.MkdirAll(path, os.ModePerm)
199+
args := []string{"dump", "-A"}
200+
flags := map[string]string{"output-directory": path}
201+
return KCmd{Command: "cluster-info", Args: args, Flags: flags}
202+
}
203+
204+
func DumpLogs(path string, namespace string) KCmd {
205+
//Make dir if doesn't exist
206+
_ = os.MkdirAll(path, os.ModePerm)
207+
args := []string{"dump", "-n", namespace}
208+
flags := map[string]string{"output-directory": path}
209+
return KCmd{Command: "cluster-info", Args: args, Flags: flags}
210+
}

magefile.go

+2
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ import (
1313
"github.com/riptano/dse-operator/mage/operator"
1414
// mage:import kind
1515
_ "github.com/riptano/dse-operator/mage/kind"
16+
// mage:import integ
17+
_ "github.com/riptano/dse-operator/mage/integ-tests"
1618
)
1719

1820
// Clean all build artifacts, does not clean up old docker images.

0 commit comments

Comments
 (0)