Skip to content

Commit

Permalink
Pilot e2e test support for local docker registry (istio#3920)
Browse files Browse the repository at this point in the history
- Various changes to the e2e tests
- Re-introducing old mixer docs (with many changes) for
running a local docker registry in k8s.
  • Loading branch information
Nathan Mittler authored Mar 5, 2018
1 parent a5eb925 commit 7ba4568
Show file tree
Hide file tree
Showing 13 changed files with 190 additions and 35 deletions.
16 changes: 11 additions & 5 deletions tests/e2e/tests/pilot/pilot_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ var (
func init() {
flag.StringVar(&config.Hub, "hub", config.Hub, "Docker hub")
flag.StringVar(&config.Tag, "tag", config.Tag, "Docker tag")
flag.StringVar(&config.ImagePullPolicy, "image-pull-policy", config.ImagePullPolicy,
"Pull policy for Docker images")
flag.StringVar(&config.IstioNamespace, "ns", config.IstioNamespace,
"Namespace in which to install Istio components (empty to create/delete temporary one)")
flag.StringVar(&config.Namespace, "n", config.Namespace,
Expand Down Expand Up @@ -170,14 +172,18 @@ func doTest(testName string, config *tutil.Config, t *testing.T) {
}

for _, test := range tests {
// If the user has specified a test, skip all other tests
if len(config.SelectedTest) > 0 && config.SelectedTest != test.String() {
continue
}

// Run the test the configured number of times.
for i := 0; i < config.TestCount; i++ {
testName := test.String()

// User specified test doesn't match this test ... skip it.
if len(config.SelectedTest) > 0 && config.SelectedTest != testName {
t.Run(testName, func(t *testing.T) {
t.Skipf("Skipping test [%v] due to user-specified test: %v", t.Name(), config.SelectedTest)
})
continue
}

if config.TestCount > 1 {
testName = testName + "_attempt_" + strconv.Itoa(i+1)
}
Expand Down
2 changes: 1 addition & 1 deletion tests/e2e/tests/pilot/testdata/app.yaml.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ spec:
containers:
- name: app
image: {{.Hub}}/app:{{.Tag}}
imagePullPolicy: IfNotPresent
imagePullPolicy: {{.ImagePullPolicy}}
args:
- --port
- "{{.port1}}"
Expand Down
2 changes: 1 addition & 1 deletion tests/e2e/tests/pilot/testdata/ca.yaml.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -21,4 +21,4 @@ spec:
containers:
- name: istio-ca-container
image: {{.Hub}}/istio-ca:{{.Tag}}
imagePullPolicy: IfNotPresent
imagePullPolicy: {{.ImagePullPolicy}}
4 changes: 2 additions & 2 deletions tests/e2e/tests/pilot/testdata/eureka.yaml.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,12 @@ spec:
containers:
- name: eureka
image: netflixoss/eureka:1.3.1
imagePullPolicy: IfNotPresent
imagePullPolicy: {{.ImagePullPolicy}}
ports:
- containerPort: 8080
- name: mirror
image: {{.Hub}}/eurekamirror:{{.Tag}}
imagePullPolicy: IfNotPresent
imagePullPolicy: {{.ImagePullPolicy}}
args:
- -url
- http://localhost:8080
Expand Down
2 changes: 1 addition & 1 deletion tests/e2e/tests/pilot/testdata/ingress-proxy.yaml.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ spec:
{{end}}
- --controlPlaneAuthPolicy
- "{{.ControlPlaneAuthPolicy.String}}"
imagePullPolicy: IfNotPresent
imagePullPolicy: {{.ImagePullPolicy}}
ports:
- containerPort: 443
- containerPort: 80
Expand Down
4 changes: 2 additions & 2 deletions tests/e2e/tests/pilot/testdata/mixer.yaml.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ spec:
containers:
- name: mixer
image: {{.Hub}}/mixer_debug:{{.Tag}}
imagePullPolicy: IfNotPresent
imagePullPolicy: {{.ImagePullPolicy}}
ports:
- containerPort: 9091
- containerPort: 9093
Expand All @@ -61,7 +61,7 @@ spec:
- debug # pilot verification required debug output
- name: istio-proxy
image: {{.Hub}}/proxy_debug:{{.Tag}}
imagePullPolicy: IfNotPresent
imagePullPolicy: {{.ImagePullPolicy}}
ports:
- containerPort: 15004
args:
Expand Down
4 changes: 2 additions & 2 deletions tests/e2e/tests/pilot/testdata/pilot.yaml.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ spec:
containers:
- name: discovery
image: {{.Hub}}/pilot:{{.Tag}}
imagePullPolicy: IfNotPresent
imagePullPolicy: {{.ImagePullPolicy}}
args:
- discovery
- -a
Expand Down Expand Up @@ -126,7 +126,7 @@ spec:
{{end}}
- name: istio-proxy
image: {{.Hub}}/proxy_debug:{{.Tag}}
imagePullPolicy: IfNotPresent
imagePullPolicy: {{.ImagePullPolicy}}
ports:
- containerPort: 15003
- containerPort: 15005
Expand Down
2 changes: 1 addition & 1 deletion tests/e2e/tests/pilot/testdata/sidecar-injector.yaml.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ spec:
containers:
- name: webhook
image: {{.Hub}}/sidecar_injector:{{.Tag}}
imagePullPolicy: IfNotPresent
imagePullPolicy: {{.ImagePullPolicy}}
args:
- --tlsCertFile=/etc/istio/certs/cert.pem
- --tlsKeyFile=/etc/istio/certs/key.pem
Expand Down
2 changes: 1 addition & 1 deletion tests/e2e/tests/pilot/testdata/zipkin.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ spec:
containers:
- name: zipkin
image: docker.io/openzipkin/zipkin
imagePullPolicy: IfNotPresent
imagePullPolicy: {{.ImagePullPolicy}}
ports:
- containerPort: 9411
env:
Expand Down
3 changes: 2 additions & 1 deletion tests/e2e/tests/pilot/util/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,13 @@ type Config struct {
KubeConfig string
Hub string
Tag string
ImagePullPolicy string
Namespace string
IstioNamespace string
Registry string
ErrorLogsDir string
CoreFilesDir string
SelectedTest string
SidecarTemplate string
AdmissionServiceName string
Verbosity int
DebugPort int
Expand Down Expand Up @@ -66,6 +66,7 @@ func NewConfig() *Config {
KubeConfig: os.Getenv("KUBECONFIG"),
Hub: defaultHub,
Tag: "",
ImagePullPolicy: "IfNotPresent",
Namespace: "",
IstioNamespace: "",
Registry: defaultRegistry,
Expand Down
43 changes: 25 additions & 18 deletions tests/e2e/tests/pilot/util/environment.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,8 @@ type Environment struct {
Name string
Config Config

sidecarTemplate string

KubeClient kubernetes.Interface

// Directory where test data files are located.
Expand Down Expand Up @@ -105,6 +107,7 @@ type TemplateData struct {
IstioNamespace string
Registry string
AdmissionServiceName string
ImagePullPolicy string
Verbosity int
DebugPort int
Auth meshconfig.MeshConfig_AuthPolicy
Expand Down Expand Up @@ -138,6 +141,7 @@ func NewEnvironment(config Config) *Environment {
e.MixerCustomConfigFile = mixerConfigAuthFile
e.PilotCustomConfigFile = pilotConfigAuthFile
}

return &e
}

Expand Down Expand Up @@ -182,6 +186,7 @@ func (e *Environment) ToTemplateData() TemplateData {
MixerCustomConfigFile: e.MixerCustomConfigFile,
CABundle: e.CABundle,
RDSv2: e.Config.RDSv2,
ImagePullPolicy: e.Config.ImagePullPolicy,
}
}

Expand Down Expand Up @@ -255,7 +260,7 @@ func (e *Environment) Setup() error {
debugMode := e.Config.DebugImagesAndMode
log.Infof("mesh %s", spew.Sdump(e.meshConfig))

e.Config.SidecarTemplate, err = inject.GenerateTemplateFromParams(&inject.Params{
e.sidecarTemplate, err = inject.GenerateTemplateFromParams(&inject.Params{
InitImage: inject.InitImageName(e.Config.Hub, e.Config.Tag, debugMode),
ProxyImage: inject.ProxyImageName(e.Config.Hub, e.Config.Tag, debugMode),
Verbosity: e.Config.Verbosity,
Expand All @@ -264,6 +269,7 @@ func (e *Environment) Setup() error {
Version: "integration-test",
Mesh: e.meshConfig,
DebugMode: debugMode,
ImagePullPolicy: e.Config.ImagePullPolicy,
})
if err != nil {
return err
Expand Down Expand Up @@ -383,21 +389,22 @@ func (e *Environment) deployApp(deployment, svcName string, port1, port2, port3,
}

w, err := e.Fill("app.yaml.tmpl", map[string]string{
"Hub": e.Config.Hub,
"Tag": e.Config.Tag,
"service": svcName,
"perServiceAuth": strconv.FormatBool(perServiceAuth),
"deployment": deployment,
"port1": strconv.Itoa(port1),
"port2": strconv.Itoa(port2),
"port3": strconv.Itoa(port3),
"port4": strconv.Itoa(port4),
"port5": strconv.Itoa(port5),
"port6": strconv.Itoa(port6),
"version": version,
"istioNamespace": e.Config.IstioNamespace,
"injectProxy": strconv.FormatBool(injectProxy),
"healthPort": healthPort,
"Hub": e.Config.Hub,
"Tag": e.Config.Tag,
"service": svcName,
"perServiceAuth": strconv.FormatBool(perServiceAuth),
"deployment": deployment,
"port1": strconv.Itoa(port1),
"port2": strconv.Itoa(port2),
"port3": strconv.Itoa(port3),
"port4": strconv.Itoa(port4),
"port5": strconv.Itoa(port5),
"port6": strconv.Itoa(port6),
"version": version,
"istioNamespace": e.Config.IstioNamespace,
"injectProxy": strconv.FormatBool(injectProxy),
"healthPort": healthPort,
"ImagePullPolicy": e.Config.ImagePullPolicy,
})
if err != nil {
return err
Expand All @@ -406,7 +413,7 @@ func (e *Environment) deployApp(deployment, svcName string, port1, port2, port3,
writer := new(bytes.Buffer)

if injectProxy && !e.Config.UseAutomaticInjection {
if err := inject.IntoResourceFile(e.Config.SidecarTemplate, e.meshConfig, strings.NewReader(w), writer); err != nil {
if err := inject.IntoResourceFile(e.sidecarTemplate, e.meshConfig, strings.NewReader(w), writer); err != nil {
return err
}
} else {
Expand Down Expand Up @@ -762,7 +769,7 @@ func (e *Environment) deleteAdmissionWebhookSecret() error {
func (e *Environment) createSidecarInjector() error {
configData, err := yaml.Marshal(&inject.Config{
Policy: inject.InjectionPolicyEnabled,
Template: e.Config.SidecarTemplate,
Template: e.sidecarTemplate,
})
if err != nil {
return err
Expand Down
53 changes: 53 additions & 0 deletions tests/util/localregistry/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
# Using a Local Docker Registry

`localregistry.yaml` is a copy of Kubernete's local registry addon, and is included to make it easier to test
Istio by allowing a developer to push docker images locally rather than to some remote registry.

### Run the registry
To run the local registry in your kubernetes cluster:

```shell
$ kubectl apply -f ./tests/util/localregistry/localregistry.yaml
```

### Expose the Registry

After the registry server is running, expose it locally by executing:

```shell
$ kubectl port-forward --namespace kube-system $POD 5000:5000
```

If you're testing locally with minikube, `$POD` can be set with:

```shell
$ POD=$(kubectl get pods --namespace kube-system -l k8s-app=kube-registry \
-o template --template '{{range .items}}{{.metadata.name}} {{.status.phase}}{{"\n"}}{{end}}' \
| grep Running | head -1 | cut -f1 -d' ')
```

### Push Local Images

Build and push the Istio docker images to the local registry, by running:

```shell
$ make docker push HUB=localhost:5000 TAG=latest
```

### Run with Local Images

#### Running E2E Tests
If you're running e2e tests, you can set the test flags:

```shell
$ go test <TEST_PATH> -hub=localhost:5000 -tag=latest
```

#### Hard-coding the Image URL

You can also modify the image URLs in your deployment yaml files directly:

```yaml
image: localhost:5000/<APP_NAME>:latest
```
88 changes: 88 additions & 0 deletions tests/util/localregistry/localregistry.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
# after creating the local registry, to push images to it you need to open up a port to the pod's registry:
# $ POD=$(kubectl get pods --namespace kube-system -l k8s-app=kube-registry \
# -o template --template '{{range .items}}{{.metadata.name}} {{.status.phase}}{{"\n"}}{{end}}' \
# | grep Running | head -1 | cut -f1 -d' ')
# $ kubectl port-forward --namespace kube-system $POD 5000:5000

apiVersion: v1
kind: ReplicationController
metadata:
name: kube-registry-v0
namespace: kube-system
labels:
k8s-app: kube-registry
version: v0
spec:
replicas: 1
selector:
k8s-app: kube-registry
version: v0
template:
metadata:
labels:
k8s-app: kube-registry
version: v0
spec:
containers:
- name: registry
image: registry:2
resources:
limits:
cpu: 100m
memory: 100Mi
env:
- name: REGISTRY_HTTP_ADDR
value: :5000
- name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY
value: /var/lib/registry
volumeMounts:
- name: image-store
mountPath: /var/lib/registry
ports:
- containerPort: 5000
name: registry
protocol: TCP
volumes:
- name: image-store
emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
name: kube-registry
namespace: kube-system
labels:
k8s-app: kube-registry
kubernetes.io/name: "KubeRegistry"
spec:
selector:
k8s-app: kube-registry
ports:
- name: registry
port: 5000
protocol: TCP
---
apiVersion: v1
kind: Pod
metadata:
name: kube-registry-proxy
namespace: kube-system
spec:
containers:
- name: kube-registry-proxy
image: gcr.io/google_containers/kube-registry-proxy:0.3
resources:
limits:
cpu: 100m
memory: 50Mi
env:
- name: REGISTRY_HOST
value: kube-registry.kube-system.svc.cluster.local
- name: REGISTRY_PORT
value: "5000"
- name: FORWARD_PORT
value: "5000"
ports:
- name: registry
containerPort: 5000
hostPort: 5000

0 comments on commit 7ba4568

Please sign in to comment.