Refactor basic tests with testcontainers (smartcontractkit#9970)

* wip

* wip - cron

* wip - 5+ tests, ragep2p issues

* rollback ocr/ocr2

* comments, cleanup

* increase timeouts

* rollback go.mod

* re-use mercury env as a boilerplate

* re-use mercury env as a boilerplate

* fix action's Go version

* update go.mod from develop

* explicitly login into aws registry

* update deps

* set explicit go version

* re-trigger

* github-action v2.2.2

* fix the client

* reusable envs

* review fixes

* review fixes

.github/workflows/integration-tests.yml

@@ -208,8 +208,9 @@ jobs:
       ## Run this step when changes that require tests to be run are made
       - name: Run Tests
         if: needs.changes.outputs.src == 'true'
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@ad22fbd6f4d108b82aaf49b527bcf40f32babea8 # v2.2.1
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/[email protected].3
         env:
+          TESTCONTAINERS_RYUK_DISABLED: true
           PYROSCOPE_SERVER: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
           PYROSCOPE_ENVIRONMENT: ${{ matrix.product.pyroscope_env }}
           PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
@@ -218,6 +219,8 @@ jobs:
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
           cl_repo: ${{ env.CHAINLINK_IMAGE }}
           cl_image_tag: ${{ github.sha }}
+          go_version: '1.20.5'
+          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
           artifacts_location: ./integration-tests/smoke/logs
           publish_check_name: EVM Smoke Test Results ${{ matrix.product.name }}
           token: ${{ secrets.GITHUB_TOKEN }}

charts/chainlink-cluster/devspace.yaml

@@ -19,7 +19,7 @@ pipelines:
     run: |-
       run_dependencies --all                            # 1. Deploy any projects this project needs (see "dependencies")
       ensure_pull_secrets --all                         # 2. Ensure pull secrets
-      build_images --all -t $(git describe --always)    # 3. Build, tag (git commit hash) and push all images (see "images")
+      build_images --all -t $(git rev-parse --short HEAD)    # 3. Build, tag (git commit hash) and push all images (see "images")
       create_deployments --all                          # 4. Deploy Helm charts and manifests specfied as "deployments"
 
 images:

integration-tests/client/chainlink.go

@@ -62,12 +62,12 @@ func initRestyClient(url string, email string, password string, timeout *time.Du
 	// Retry the connection on boot up, sometimes pods can still be starting up and not ready to accept connections
 	var resp *resty.Response
 	var err error
-	retryCount := 5
+	retryCount := 20
 	for i := 0; i < retryCount; i++ {
 		resp, err = rc.R().SetBody(session).Post("/sessions")
 		if err != nil {
 			log.Debug().Err(err).Str("URL", url).Interface("Session Details", session).Msg("Error connecting to Chainlink node, retrying")
-			time.Sleep(3 * time.Second)
+			time.Sleep(5 * time.Second)
 		} else {
 			break
 		}

integration-tests/docker/README.md

@@ -0,0 +1,21 @@
+## Docker environment
+This folder contains Chainlink cluster environment created with `testcontainers-go`
+
+### CLI for Local Testing Environment
+
+The command-line interface (CLI) located at `./integration-tests/docker/cmd/test_env.go` can be utilized to initiate a local testing environment. It is intended to replace Docker Compose in the near future.
+
+
+Example: 
+```
+# Set required envs
+export CHAINLINK_IMAGE="<chainlink_node_docker_image_path>"
+export CHAINLINK_VERSION="<chainlink_node_docker_image_version>" 
+# Stream logs to Loki
+export LOKI_TOKEN=...
+export LOKI_URL=https://${loki_host}/loki/api/v1/push
+
+cd ./integration-tests/docker/cmd
+
+go run test_env.go start-env cl-cluster
+```

integration-tests/docker/cmd/test_env.go

@@ -0,0 +1,76 @@
+package main
+
+import (
+	"io"
+	defaultlog "log"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+	"github.com/smartcontractkit/chainlink/integration-tests/docker/test_env"
+	"github.com/smartcontractkit/chainlink/integration-tests/utils"
+	"github.com/spf13/cobra"
+	"github.com/testcontainers/testcontainers-go"
+)
+
+func main() {
+	rootCmd := &cobra.Command{
+		Use:   "test_env",
+		Short: "CL cluster docker test env management tool",
+	}
+
+	startEnvCmd := &cobra.Command{
+		Use:   "start-env",
+		Short: "Start new docker test env",
+	}
+	rootCmd.AddCommand(startEnvCmd)
+
+	startFullEnvCmd := &cobra.Command{
+		Use:   "cl-cluster",
+		Short: "Basic CL cluster",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			utils.SetupGlobalLogger()
+			log.Info().Msg("Starting CL cluster test environment..")
+
+			_, err := test_env.NewCLTestEnvBuilder().
+				WithGeth().
+				WithMockServer(1).
+				WithCLNodes(6).
+				Build()
+			if err != nil {
+				return err
+			}
+
+			log.Info().Msg("Cl cluster is ready")
+
+			handleExitSignal()
+
+			return nil
+		},
+	}
+	startEnvCmd.AddCommand(startFullEnvCmd)
+
+	// Set default log level for non-testcontainer code
+	zerolog.SetGlobalLevel(zerolog.InfoLevel)
+
+	// Discard testcontainers logs
+	testcontainers.Logger = defaultlog.New(io.Discard, "", defaultlog.LstdFlags)
+
+	if err := rootCmd.Execute(); err != nil {
+		log.Error().Err(err).Msg("Error")
+		os.Exit(1)
+	}
+}
+
+func handleExitSignal() {
+	// Create a channel to receive exit signals
+	exitChan := make(chan os.Signal, 1)
+	signal.Notify(exitChan, os.Interrupt, syscall.SIGTERM)
+
+	log.Info().Msg("Press Ctrl+C to destroy the test environment")
+
+	// Block until an exit signal is received
+	<-exitChan
+}

integration-tests/docker/docker.go

@@ -0,0 +1,30 @@
+package docker
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/google/uuid"
+	"github.com/rs/zerolog/log"
+	tc "github.com/testcontainers/testcontainers-go"
+)
+
+func CreateNetwork() (*tc.DockerNetwork, error) {
+	uuidObj, _ := uuid.NewRandom()
+	var networkName = fmt.Sprintf("network-%s", uuidObj.String())
+	network, err := tc.GenericNetwork(context.Background(), tc.GenericNetworkRequest{
+		NetworkRequest: tc.NetworkRequest{
+			Name:           networkName,
+			CheckDuplicate: true,
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+	dockerNetwork, ok := network.(*tc.DockerNetwork)
+	if !ok {
+		return nil, fmt.Errorf("failed to cast network to *dockertest.Network")
+	}
+	log.Trace().Any("network", dockerNetwork).Msgf("created network")
+	return dockerNetwork, nil
+}