integration_test.go

// Copyright 2017 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:build linux
// +build linux

package profiler

import (
	"bytes"
	"context"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"testing"
	"text/template"
	"time"

	"cloud.google.com/go/profiler/proftest"
	"golang.org/x/oauth2/google"
	compute "google.golang.org/api/compute/v1"
)

var runBackoffTest = flag.Bool("run_only_profiler_backoff_test", false, "Enables only the backoff integration test. This integration test requires over 45 mins to run, so it is not run by default.")

const (
	cloudScope        = "https://www.googleapis.com/auth/cloud-platform"
	benchFinishString = "benchmark application(s) complete"
	errorString       = "failed to set up or run the benchmark"
	gceBenchDuration  = 200 * time.Second
	gceTestTimeout    = 25 * time.Minute

	// For any agents to receive backoff, there must be more than 32 agents in
	// the deployment. The initial backoff received will be 33 minutes; each
	// subsequent backoff will be one minute longer. Running 45 benchmarks for
	// 45 minutes will ensure that several agents receive backoff responses and
	// are able to wait for the backoff duration then send another request.
	numBackoffBenchmarks = 45
	backoffBenchDuration = 45 * time.Minute
	backoffTestTimeout   = 60 * time.Minute
)

const startupTemplate = `
{{ define "setup"}}
# Install git
retry apt-get update >/dev/null
retry apt-get -y -q install git >/dev/null

# $GOCACHE is required from Go 1.12. See https://golang.org/doc/go1.11#gocache
# $GOCACHE is explicitly set because $HOME is not set when this code runs
mkdir -p /tmp/gocache
export GOCACHE=/tmp/gocache

# Install gcc, needed to install go master and cgo depencencies when using
# go1.11.
if [ "{{.GoVersion}}" = "master" ] || [[ "{{.GoVersion}}" =~ 1.11.* ]]
then
retry apt-get -y -q install gcc >/dev/null
fi

# Install desired Go version
mkdir -p /tmp/bin
retry curl -sL -o /tmp/bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/master/gimme
chmod +x /tmp/bin/gimme
export PATH=$PATH:/tmp/bin

gimme_retrier() {
  eval "$(gimme {{.GoVersion}})"
  which go # To retry on non-zero code if go not installed.
}
retry gimme_retrier

# Use go modules
export GO111MODULE="on"

# Set $GOPATH
export GOPATH="$HOME/go"

export GOCLOUD_HOME=$GOPATH/src/cloud.google.com/go
mkdir -p $GOCLOUD_HOME

# Install agent
retry git clone https://github.com/googleapis/google-cloud-go.git $GOCLOUD_HOME >/dev/null
cd $GOCLOUD_HOME
retry git fetch origin {{.Commit}}
git reset --hard {{.Commit}}

cd $GOCLOUD_HOME/profiler/busybench
retry go get >/dev/null
{{- end }}

{{ define "integration" -}}
{{- template "prologue" . }}
{{- template "setup" . }}

# Run benchmark with agent.
go run busybench.go --service="{{.Service}}" --duration={{.DurationSec}} --mutex_profiling="{{.MutexProfiling}}"

echo "{{.FinishString}}"

{{ template "epilogue" . -}}
{{ end }}

{{ define "integration_backoff" -}}
{{- template "prologue" . }}
{{- template "setup" . }}

# Compile first so each spawned process can just use the same binary.
go build busybench.go

# Do not display commands being run to simplify logging output.
set +x

# Run benchmarks with agent.
echo "Starting {{.NumBackoffBenchmarks}} benchmarks."
for (( i = 0; i < {{.NumBackoffBenchmarks}}; i++ )); do
	(./busybench --service="{{.Service}}" --duration={{.DurationSec}} \
		--num_busyworkers=1) |& while read line; \
		do echo "benchmark $i: ${line}"; done &
done
echo "Successfully started {{.NumBackoffBenchmarks}} benchmarks."

wait

# Continue displaying commands being run.
set -x

echo "{{.FinishString}}"

{{ template "epilogue" . -}}
{{ end }}
`

type goGCETestCase struct {
	proftest.InstanceConfig
	name          string
	goVersion     string
	benchDuration time.Duration
	timeout       time.Duration

	backoffTest bool

	// mutexProfiling and wantProfileTypes will not be used when the test
	// is a backoff integration test.
	mutexProfiling   bool
	wantProfileTypes []string
}

func (tc *goGCETestCase) initializeStartupScript(template *template.Template, commit string) error {
	params := struct {
		Service              string
		GoVersion            string
		Commit               string
		ErrorString          string
		FinishString         string
		MutexProfiling       bool
		DurationSec          int
		NumBackoffBenchmarks int
	}{
		Service:        tc.InstanceConfig.Name,
		GoVersion:      tc.goVersion,
		Commit:         commit,
		ErrorString:    errorString,
		FinishString:   benchFinishString,
		MutexProfiling: tc.mutexProfiling,
		DurationSec:    int(tc.benchDuration.Seconds()),
	}

	testTemplate := "integration"
	if tc.backoffTest {
		testTemplate = "integration_backoff"
		params.DurationSec = int(backoffBenchDuration.Seconds())
		params.NumBackoffBenchmarks = numBackoffBenchmarks
	}
	var buf bytes.Buffer
	err := template.Lookup(testTemplate).Execute(&buf, params)
	if err != nil {
		return fmt.Errorf("failed to render startup script for %s: %v", tc.name, err)
	}
	tc.StartupScript = buf.String()
	return nil
}

// gitCommit returns the Git commit of the current directory. The source
// checkout in the test VM will run in the same commit. Note that any local
// changes to the profiler agent won't be tested in the integration test. This
// flow only works with code that has been committed and pushed to the public
// repo (either to master or to a branch).
func gitCommit() (string, error) {
	output, err := exec.Command("git", "rev-parse", "HEAD").CombinedOutput()
	if err != nil {
		return "", err
	}
	return strings.Trim(string(output), "\n"), nil
}

// pstTimeStr returns a string representation of the time in the PST timezone.
func pstTimeStr() (string, error) {
	pst, err := time.LoadLocation("America/Los_Angeles")
	if err != nil {
		return "", fmt.Errorf("failed to initialize PST location: %v", err)
	}
	return strings.Replace(time.Now().In(pst).Format("2006-01-02-15-04-05.000000-0700"), ".", "-", -1), nil
}

func TestAgentIntegration(t *testing.T) {
	// Testing against master requires building go code and may take up to 10 minutes.
	// Allow this test to run in parallel with other top level tests to avoid timeouts.
	t.Parallel()

	if testing.Short() {
		t.Skip("skipping profiler integration tests in short mode")
	}

	projectID := os.Getenv("GCLOUD_TESTS_GOLANG_PROJECT_ID")
	if projectID == "" {
		t.Skip("skipping profiler integration test when GCLOUD_TESTS_GOLANG_PROJECT_ID variable is not set")
	}

	// all us-west1 zones
	zones := []string{"us-west1-a", "us-west1-b", "us-west1-c"}

	commit, err := gitCommit()
	if err != nil {
		t.Fatalf("failed to gather the Git revision of the current source: %v", err)
	}
	t.Logf("using Git commit %q for the profiler integration test", commit)

	runID, err := pstTimeStr()
	if err != nil {
		t.Fatalf("failed to get current time to generate a run ID: %v", err)
	}

	ctx := context.Background()

	client, err := google.DefaultClient(ctx, cloudScope)
	if err != nil {
		t.Fatalf("failed to get default client: %v", err)
	}

	computeService, err := compute.New(client)
	if err != nil {
		t.Fatalf("failed to initialize compute service: %v", err)
	}

	template, err := proftest.BaseStartupTmpl.Parse(startupTemplate)
	if err != nil {
		t.Fatalf("failed to parse startup script template: %v", err)
	}

	tr := proftest.TestRunner{
		Client: client,
	}
	gceTr := proftest.GCETestRunner{
		TestRunner:     tr,
		ComputeService: computeService,
	}

	// Determine go version used by current test run
	goVersion := strings.TrimPrefix(runtime.Version(), "go")
	goVersionName := strings.Replace(goVersion, ".", "", -1)

	testcases := []goGCETestCase{
		{
			InstanceConfig: proftest.InstanceConfig{
				ProjectID:    projectID,
				Name:         fmt.Sprintf("profiler-test-gomaster-%s", runID),
				MachineType:  "n1-standard-1",
				ImageProject: "debian-cloud",
				ImageFamily:  "debian-11",
			},
			name:             "profiler-test-gomaster",
			wantProfileTypes: []string{"CPU", "HEAP", "THREADS", "CONTENTION", "HEAP_ALLOC"},
			goVersion:        "master",
			mutexProfiling:   true,
			timeout:          gceTestTimeout,
			benchDuration:    gceBenchDuration,
		},
		{
			InstanceConfig: proftest.InstanceConfig{
				ProjectID:    projectID,
				Name:         fmt.Sprintf("profiler-test-go%s-%s", goVersionName, runID),
				MachineType:  "n1-standard-1",
				ImageProject: "debian-cloud",
				ImageFamily:  "debian-11",
			},
			name:             fmt.Sprintf("profiler-test-go%s", goVersionName),
			wantProfileTypes: []string{"CPU", "HEAP", "THREADS", "CONTENTION", "HEAP_ALLOC"},
			goVersion:        goVersion,
			mutexProfiling:   true,
			timeout:          gceTestTimeout,
			benchDuration:    gceBenchDuration,
		},
	}

	if *runBackoffTest {
		testcases = []goGCETestCase{
			{
				InstanceConfig: proftest.InstanceConfig{
					ProjectID: projectID,
					Name:      fmt.Sprintf("profiler-backoff-test-go%s-%s", goVersionName, runID),

					// Running many copies of the benchmark requires more
					// memory than is available on an n1-standard-1. Use a
					// machine type with more memory for backoff test.
					MachineType: "n1-highmem-2",

					ImageProject: "debian-cloud",
					ImageFamily:  "debian-11",
				},
				name:          fmt.Sprintf("profiler-backoff-test-go%s", goVersionName),
				goVersion:     goVersion,
				backoffTest:   true,
				timeout:       backoffTestTimeout,
				benchDuration: backoffBenchDuration,
			},
		}
	}
	// The number of tests run in parallel is the current value of GOMAXPROCS.
	runtime.GOMAXPROCS(len(testcases))
	for _, tc := range testcases {
		tc := tc // capture range variable
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			if err := tc.initializeStartupScript(template, commit); err != nil {
				t.Fatalf("failed to initialize startup script")
			}

			for i := range zones {
				tc.InstanceConfig.Zone = zones[i]
				if err := gceTr.StartInstance(ctx, &tc.InstanceConfig); err != nil {
					if strings.Contains(err.Error(), "failed to create instance") && i < (len(zones)-1) {
						// try other zones if instance failed to create
						continue
					}
					t.Fatal(err)
				}
				break
			}
			defer func() {
				if err := gceTr.DeleteInstance(ctx, &tc.InstanceConfig); err != nil {
					t.Fatal(err)
				}
			}()

			timeoutCtx, cancel := context.WithTimeout(ctx, tc.timeout)
			defer cancel()
			output, err := gceTr.PollAndLogSerialPort(timeoutCtx, &tc.InstanceConfig, benchFinishString, errorString, t.Logf)
			if err != nil {
				t.Fatalf("PollAndLogSerialPort() got error: %v", err)
			}

			if tc.backoffTest {
				if err := proftest.CheckSerialOutputForBackoffs(output, numBackoffBenchmarks, "action throttled, backoff", "creating a new profile via profiler service", "benchmark"); err != nil {
					t.Errorf("failed to check serial output for backoffs: %v", err)
				}
			}

			timeNow := time.Now()
			endTime := timeNow.Format(time.RFC3339)
			startTime := timeNow.Add(-1 * time.Hour).Format(time.RFC3339)
			for _, pType := range tc.wantProfileTypes {
				pr, err := tr.QueryProfilesWithZone(tc.ProjectID, tc.InstanceConfig.Name, startTime, endTime, pType, tc.Zone)
				if err != nil {
					t.Errorf("QueryProfilesWithZone(%s, %s, %s, %s, %s, %s) got error: %v", tc.ProjectID, tc.InstanceConfig.Name, startTime, endTime, pType, tc.Zone, err)
					continue
				}
				if err := pr.HasFunction("busywork"); err != nil {
					t.Errorf("HasFunction(%s, %s, %s, %s, %s) got error: %v", tc.ProjectID, tc.InstanceConfig.Name, startTime, endTime, pType, err)
				}
			}
		})
	}
}