Re-organize e2e-suite and reduce retry times (istio#575)

Automatic merge from submit-queue

Re-organize e2e-suite and reduce retry times

**Release note**:

```release-note
none
```
1. Re-organize test: Remove e2e-smoke tests from presubmit. All four combinations are going to run concurrently after presubmit passed.

2. Since we are using nodeport as ingress ip, there is no need to retry so many times, just wasting time. Usually when a request cannot get response in first three attempts, it's going to fail forever, so there is no value to retry 10 times.

Former-commit-id: f01c1ca

prow/e2e-suite-no_rbac-auth.sh

@@ -29,4 +29,4 @@ set -u
 set -x
 
 echo 'Running e2e no rbac, with auth Tests'
-./prow/e2e.sh --auth_enable
+./prow/e2e-suite.sh --auth_enable

prow/e2e-suite-no_rbac-no_auth.sh

@@ -29,4 +29,4 @@ set -u
 set -x
 
 echo 'Running e2e no rbac, no auth Tests'
-./prow/e2e.sh
+./prow/e2e-suite.sh

prow/e2e-suite-rbac-auth.sh

@@ -29,4 +29,4 @@ set -u
 set -x
 
 echo 'Running e2e with rbac, with auth Tests'
-./prow/e2e.sh --rbac_enable --auth_enable
+./prow/e2e-suite.sh --rbac_enable --auth_enable

prow/e2e-suite-rbac-no_auth.sh

@@ -29,4 +29,4 @@ set -u
 set -x
 
 echo 'Running e2e with rbac, no auth Tests'
-./prow/e2e.sh --rbac_enable
+./prow/e2e-suite.sh --rbac_enable

prow/e2e-suite.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Copyright 2017 Istio Authors
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+
+#######################################################
+# e2e-suite triggered after istio/presubmit succeeded #
+#######################################################
+
+# Exit immediately for non zero status
+set -e
+# Check unset variables
+set -u
+# Print commands
+set -x
+
+# Avoid wasting time on getting external IP
+E2E_ARGS=(--use_local_cluster)
+
+if [ "${CI:-}" == 'bootstrap' ]; then
+  # Test harness will checkout code to directory $GOPATH/src/github.com/istio/istio
+  # but we depend on being at path $GOPATH/src/istio.io/istio for imports
+  ln -sf ${GOPATH}/src/github.com/istio ${GOPATH}/src/istio.io
+  cd ${GOPATH}/src/istio.io/istio
+
+  # bootsrap upload all artifacts in _artifacts to the log bucket.
+  ARTIFACTS_DIR=${ARTIFACTS_DIR:-"${GOPATH}/src/istio.io/istio/_artifacts"}
+  LOG_HOST="stackdriver"
+  PROJ_ID="istio-testing"
+  E2E_ARGS+=(--test_logs_path="${ARTIFACTS_DIR}" --log_provider=${LOG_HOST} --project_id=${PROJ_ID})
+fi
+
+echo 'Running Integration Tests'
+./tests/e2e.sh ${E2E_ARGS[@]:-} ${@}

prow/e2e.sh

@@ -15,9 +15,9 @@
 #   limitations under the License.
 
 
-#######################################
-# Presubmit script triggered by Prow. #
-#######################################
+#####################################################################
+# e2e-smoke script triggered after pilot/mixer presubmit succeeded. #
+#####################################################################
 
 # Exit immediately for non zero status
 set -e

prow/istio-presubmit.sh

@@ -26,7 +26,7 @@ set -u
 # Print commands
 set -x
 
-if [ "${CI}" == 'bootstrap' ]; then
+if [ "${CI:-}" == 'bootstrap' ]; then
   # Test harness will checkout code to directory $GOPATH/src/github.com/istio/istio
   # but we depend on being at path $GOPATH/src/istio.io/istio for imports
   ln -sf ${GOPATH}/src/github.com/istio ${GOPATH}/src/istio.io
@@ -47,7 +47,3 @@ bazel test //...
 
 echo 'Pushing Images'
 (cd devel/fortio && make authorize all TAG="${GIT_SHA}")
-
-echo 'Running Integration Tests'
-./prow/e2e.sh
-

tests/e2e/tests/bookinfo/demo_test.go

@@ -48,7 +48,7 @@ const (
 
 var (
 	tc             *testConfig
-	testRetryTimes = 10
+	testRetryTimes = 5
 	defaultRules   = []string{allRule}
 )
 

tests/e2e/tests/mixer/mixer_test.go

@@ -91,8 +91,9 @@ type luaScriptResponseJSON struct {
 }
 
 var (
-	tc    *testConfig
-	rules = []string{rateLimitRule, denialRule, newTelemetryRule, routeAllRule, routeReviewsVersionsRule, emptyRule}
+	tc             *testConfig
+	testRetryTimes = 5
+	rules          = []string{rateLimitRule, denialRule, newTelemetryRule, routeAllRule, routeReviewsVersionsRule, emptyRule}
 )
 
 func (t *testConfig) Setup() error {
@@ -187,7 +188,7 @@ func TestMain(m *testing.M) {
 }
 
 func TestGlobalCheckAndReport(t *testing.T) {
-	if err := visitProductPage(10); err != nil {
+	if err := visitProductPage(testRetryTimes); err != nil {
 		t.Fatalf("Test app setup failure: %v", err)
 	}
 
@@ -231,7 +232,7 @@ func TestNewMetrics(t *testing.T) {
 	// TODO: figure out a better way to confirm rule active
 	time.Sleep(5 * time.Second)
 
-	if err := visitProductPage(10); err != nil {
+	if err := visitProductPage(testRetryTimes); err != nil {
 		t.Fatalf("Test app setup failure: %v", err)
 	}
 
@@ -279,7 +280,7 @@ func TestDenials(t *testing.T) {
 
 	// generate several calls to the product page
 	for i := 0; i < 10; i++ {
-		if err := visitProductPage(10); err != nil {
+		if err := visitProductPage(testRetryTimes); err != nil {
 			t.Fatalf("Test app setup failure: %v", err)
 		}
 	}