[TT-291] Adds Generic Test Runner (smartcontractkit#9318)

.github/workflows/generic-test-runner.yml

@@ -2,31 +2,87 @@ name: Generic Test Runner
 on:
   workflow_dispatch:
     inputs:
-      directtory:
+      network:
+        description: 'Network to run tests against'
+        required: true
+        default: 'SIMULATED'
+      wsURL:
+        description: 'Chain WS URL (Skip with SIMULATED)'
+        required: false
+      httpURL:
+        description: 'Chain HTTP URL (Skip with SIMULATED)'
+        required: false
+      fundingKey:
+        description: 'Private key to fund test (Skip with SIMULATED)'
+        required: false
+      directory:
         description: 'Directory to run tests from'
         required: true
         default: 'smoke'
       test:
         description: 'Test to run'
         required: true
-        default: 'all'
-      test-inputs:
+        default: 'OCRBasic'
+      testInputs:
         description: 'Custom test inputs'
         required: false
         default: ''
 env:
   ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }}
   CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
-  TEST_SUITE: ${{ github.event.inputs.directory }}
 
 jobs:
+  build-chainlink:
+    environment: integration
+    permissions:
+      id-token: write
+      contents: read
+    strategy:
+      matrix:
+        image:
+          - name: ""
+            dockerfile: core/chainlink.Dockerfile
+            tag-suffix: ""
+          - name: (plugins)
+            dockerfile: plugins/chainlink.Dockerfile
+            tag-suffix: -plugins
+    name: Build Chainlink Image ${{ matrix.image.name }}
+    runs-on: ubuntu20.04-16cores-64GB
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+      - name: Check if image exists
+        id: check-image
+        uses: smartcontractkit/chainlink-github-actions/docker/image-exists@ab595504ae9cf10c60eb8d2c5ce025284e58b210 #v2.1.5
+        with:
+          repository: chainlink
+          tag: ${{ github.sha }}${{ matrix.image.tag-suffix }}
+          AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+          AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+      - name: Build Image
+        if: steps.check-image.outputs.exists == 'false'
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@2c9f401149f6c25fb632067b7e6626aebeee5d69
+        with:
+          cl_repo: smartcontractkit/chainlink
+          cl_ref: ${{ github.sha }}
+          cl_dockerfile: ${{ matrix.image.dockerfile }}
+          push_tag: ${{ env.CHAINLINK_IMAGE }}:${{ github.sha }}${{ matrix.image.tag-suffix }}
+          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+      - name: Print Chainlink Image Built
+        run: |
+          echo "### chainlink node image tag used for this test run :link:" >>$GITHUB_STEP_SUMMARY
+          echo "\`${GITHUB_SHA}\`" >>$GITHUB_STEP_SUMMARY
+
   build-test-image:
     environment: integration
     permissions:
       id-token: write
       contents: read
     name: Build Test Image
     runs-on: ubuntu20.04-16cores-64GB
+    env:
+      TEST_SUITE: ${{ github.event.inputs.directory }}
     steps:
       - name: Checkout the repo
         uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
@@ -39,15 +95,40 @@ jobs:
 
   test:
     runs-on: ubuntu-latest
-    needs: build-test-image
+    needs: [build-chainlink, build-test-image]
+    name: Run Test
     environment: integration
+    permissions:
+      id-token: write
+      contents: read
+    env:
+      CHAINLINK_ENV_USER: ${{ github.actor }}
+      CHAINLINK_COMMIT_SHA: ${{ github.sha }}
+      TEST_SUITE: ${{ github.event.inputs.directory }}
+      SELECTED_NETWORKS: ${{ github.event.inputs.network }}
+      TEST_LOG_LEVEL: debug
     steps:
+      - name: Mask Inputs
+        run: |
+          EVM_URLS=$(jq -r '.inputs.wsURL' $GITHUB_EVENT_PATH)
+          EVM_HTTP_URLS=$(jq -r '.inputs.httpURL' $GITHUB_EVENT_PATH)
+          EVM_KEYS=$(jq -r '.inputs.fundingPrivateKey' $GITHUB_EVENT_PATH)
+
+          echo ::add-mask::$EVM_URLS
+          echo ::add-mask::$EVM_HTTP_URLS
+          echo ::add-mask::$EVM_KEYS
+
+          echo EVM_URLS=$EVM_URLS >> $GITHUB_ENV
+          echo EVM_HTTP_URLS=$EVM_HTTP_URLS >> $GITHUB_ENV
+          echo EVM_KEYS=$EVM_KEYS >> $GITHUB_ENV
+      - name: Debug Input
+        run: echo ${{ github.event.inputs.testInputs }}
       - name: Checkout the repo
         uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
       - name: Run Tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@ab7c0b6189783e8ee29ca4cefbec896b30819831 #v2.1.7
         with:
-          test_command_to_run: make test_need_operator_assets && cd ./integration-tests && go test -timeout 30m -count=1 ./${{ github.event.inputs.directory }} -run ${{ github.event.inputs.test }} -v -args ${{ github.event.inputs.test-inputs }}
+          test_command_to_run: make test_need_operator_assets && cd ./integration-tests && go test -timeout 1h -count=1 ./${{ github.event.inputs.directory }} -run ${{ github.event.inputs.test }} -v -args ${{ github.event.inputs.test-inputs }}
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
           cl_repo: ${{ env.CHAINLINK_IMAGE }}
           cl_image_tag: ${{ github.sha }}
@@ -57,4 +138,4 @@ jobs:
           go_mod_path: ./integration-tests/go.mod
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}

integration-tests/README.md

@@ -2,9 +2,12 @@
 
 Here lives the integration tests for chainlink, utilizing our [chainlink-testing-framework](https://github.com/smartcontractkit/chainlink-testing-framework).
 
-## Setup
+## Full Setup
 
-Prerequisites to run the tests.
+Prerequisites to run the tests from your local machine. Best for debugging and developing new tests, or checking changes to the framework. This can be a bit complex however, so if you just want to run the tests, see the [Just Run](#just-run) section.
+
+<details>
+  <summary>Details</summary>
 
 ### Install Dependencies
 
@@ -141,3 +144,16 @@ make test_perf
 ## Common Issues
 
 When upgrading to a new version, it's possible the helm charts have changed. There are a myriad of errors that can result from this, so it's best to just try running `helm repo update` when encountering an error you're unsure of.
+
+</details>
+
+
+## Just Run
+
+If you're making changes to chainlink code, or just want to run some tests without a complex setup, follow the below steps.
+
+1. [Install Go](https://go.dev/doc/install)
+2. [Install GitHub CLI](https://cli.github.com/)
+3. Authenticate with GitHub CLI: `gh auth login`
+4. `go run .`
+5. Follow the setup wizard and watch your tests run in the GitHub Action.

integration-tests/benchmark/keeper_test.go

@@ -9,7 +9,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/smartcontractkit/chainlink-testing-framework/utils"
 	"github.com/stretchr/testify/require"
 
 	env_client "github.com/smartcontractkit/chainlink-env/client"
@@ -18,14 +17,14 @@ import (
 	"github.com/smartcontractkit/chainlink-env/pkg/helm/chainlink"
 	"github.com/smartcontractkit/chainlink-env/pkg/helm/ethereum"
 	"github.com/smartcontractkit/chainlink-env/pkg/helm/reorg"
-
 	"github.com/smartcontractkit/chainlink-testing-framework/blockchain"
+	"github.com/smartcontractkit/chainlink-testing-framework/utils"
 
-	networks "github.com/smartcontractkit/chainlink/integration-tests"
 	"github.com/smartcontractkit/chainlink/integration-tests/actions"
 	"github.com/smartcontractkit/chainlink/integration-tests/client"
 	"github.com/smartcontractkit/chainlink/integration-tests/contracts"
 	eth_contracts "github.com/smartcontractkit/chainlink/integration-tests/contracts/ethereum"
+	"github.com/smartcontractkit/chainlink/integration-tests/networks"
 	"github.com/smartcontractkit/chainlink/integration-tests/testsetups"
 )
 
@@ -294,7 +293,7 @@ func getEnv(key, fallback string) string {
 
 func SetupAutomationBenchmarkEnv(t *testing.T) (*environment.Environment, blockchain.EVMNetwork) {
 	l := utils.GetTestLogger(t)
-	activeEVMNetwork := networks.SelectedNetwork // Environment currently being used to run benchmark test on
+	testNetwork := networks.SelectedNetwork // Environment currently being used to run benchmark test on
 	blockTime := "1"
 	networkDetailTOML := `MinIncomingConfirmations = 1`
 
@@ -308,7 +307,7 @@ func SetupAutomationBenchmarkEnv(t *testing.T) (*environment.Environment, blockc
 		NamespacePrefix: fmt.Sprintf(
 			"automation-%s-%s-%s",
 			testType,
-			strings.ReplaceAll(strings.ToLower(activeEVMNetwork.Name), " ", "-"),
+			strings.ReplaceAll(strings.ToLower(testNetwork.Name), " ", "-"),
 			strings.ReplaceAll(strings.ToLower(RegistryToTest), "_", "-"),
 		),
 		Test: t,
@@ -331,11 +330,11 @@ func SetupAutomationBenchmarkEnv(t *testing.T) (*environment.Environment, blockc
 	}
 
 	// Test can run on simulated, simulated-non-dev, testnets
-	if activeEVMNetwork.Name == networks.SimulatedEVMNonDev.Name {
+	if testNetwork.Name == networks.SimulatedEVMNonDev.Name {
 		keeperBenchmarkBaseTOML = keeperBenchmarkBaseTOML + simulatedEVMNonDevTOML
 		testEnvironment.
 			AddHelm(reorg.New(&reorg.Props{
-				NetworkName: activeEVMNetwork.Name,
+				NetworkName: testNetwork.Name,
 				Values: map[string]interface{}{
 					"geth": map[string]interface{}{
 						"tx": map[string]interface{}{
@@ -350,9 +349,9 @@ func SetupAutomationBenchmarkEnv(t *testing.T) (*environment.Environment, blockc
 	} else {
 		testEnvironment.
 			AddHelm(ethereum.New(&ethereum.Props{
-				NetworkName: activeEVMNetwork.Name,
-				Simulated:   activeEVMNetwork.Simulated,
-				WsURLs:      activeEVMNetwork.URLs,
+				NetworkName: testNetwork.Name,
+				Simulated:   testNetwork.Simulated,
+				WsURLs:      testNetwork.URLs,
 				Values: map[string]interface{}{
 					"resources": map[string]interface{}{
 						"requests": map[string]interface{}{
@@ -372,55 +371,55 @@ func SetupAutomationBenchmarkEnv(t *testing.T) (*environment.Environment, blockc
 	}
 
 	// deploy blockscout if running on simulated
-	if activeEVMNetwork.Simulated {
+	if testNetwork.Simulated {
 		testEnvironment.
 			AddChart(blockscout.New(&blockscout.Props{
 				Name:    "geth-blockscout",
-				WsURL:   activeEVMNetwork.URLs[0],
-				HttpURL: activeEVMNetwork.HTTPURLs[0]}))
+				WsURL:   testNetwork.URLs[0],
+				HttpURL: testNetwork.HTTPURLs[0]}))
 	}
 	err := testEnvironment.Run()
 	require.NoError(t, err, "Error launching test environment")
 
 	if testEnvironment.WillUseRemoteRunner() {
-		return testEnvironment, activeEVMNetwork
+		return testEnvironment, testNetwork
 	}
 
 	// separate RPC urls per CL node
 	internalWsURLs := make([]string, 0)
 	internalHttpURLs := make([]string, 0)
 	for i := 0; i < NumberOfNodes; i++ {
 		// for simulated-nod-dev each CL node gets its own RPC node
-		if activeEVMNetwork.Name == networks.SimulatedEVMNonDev.Name {
-			podName := fmt.Sprintf("%s-ethereum-geth:%d", activeEVMNetwork.Name, i)
+		if testNetwork.Name == networks.SimulatedEVMNonDev.Name {
+			podName := fmt.Sprintf("%s-ethereum-geth:%d", testNetwork.Name, i)
 			txNodeInternalWs, err := testEnvironment.Fwd.FindPort(podName, "geth", "ws-rpc").As(env_client.RemoteConnection, env_client.WS)
 			require.NoError(t, err, "Error finding WS ports")
 			internalWsURLs = append(internalWsURLs, txNodeInternalWs)
 			txNodeInternalHttp, err := testEnvironment.Fwd.FindPort(podName, "geth", "http-rpc").As(env_client.RemoteConnection, env_client.HTTP)
 			require.NoError(t, err, "Error finding HTTP ports")
 			internalHttpURLs = append(internalHttpURLs, txNodeInternalHttp)
 			// for testnets with more than 1 RPC nodes
-		} else if len(activeEVMNetwork.URLs) > 1 {
-			internalWsURLs = append(internalWsURLs, activeEVMNetwork.URLs[i%len(activeEVMNetwork.URLs)])
-			internalHttpURLs = append(internalHttpURLs, activeEVMNetwork.HTTPURLs[i%len(activeEVMNetwork.URLs)])
+		} else if len(testNetwork.URLs) > 1 {
+			internalWsURLs = append(internalWsURLs, testNetwork.URLs[i%len(testNetwork.URLs)])
+			internalHttpURLs = append(internalHttpURLs, testNetwork.HTTPURLs[i%len(testNetwork.URLs)])
 			// for simulated and testnets with 1 RPC node
 		} else {
-			internalWsURLs = append(internalWsURLs, activeEVMNetwork.URLs[0])
-			internalHttpURLs = append(internalHttpURLs, activeEVMNetwork.HTTPURLs[0])
+			internalWsURLs = append(internalWsURLs, testNetwork.URLs[0])
+			internalHttpURLs = append(internalHttpURLs, testNetwork.HTTPURLs[0])
 		}
 	}
 	l.Debug().Strs("internalWsURLs", internalWsURLs).Strs("internalHttpURLs", internalHttpURLs).Msg("internalURLs")
 
 	for i := 0; i < NumberOfNodes; i++ {
-		activeEVMNetwork.HTTPURLs = []string{internalHttpURLs[i]}
-		activeEVMNetwork.URLs = []string{internalWsURLs[i]}
+		testNetwork.HTTPURLs = []string{internalHttpURLs[i]}
+		testNetwork.URLs = []string{internalWsURLs[i]}
 		testEnvironment.AddHelm(chainlink.New(i, map[string]any{
-			"toml":      client.AddNetworkDetailedConfig(keeperBenchmarkBaseTOML, networkDetailTOML, activeEVMNetwork),
+			"toml":      client.AddNetworkDetailedConfig(keeperBenchmarkBaseTOML, networkDetailTOML, testNetwork),
 			"chainlink": chainlinkResources,
 			"db":        dbResources,
 		}))
 	}
 	err = testEnvironment.Run()
 	require.NoError(t, err, "Error launching test environment")
-	return testEnvironment, activeEVMNetwork
+	return testEnvironment, testNetwork
 }

integration-tests/chaos/automation_chaos_test.go

@@ -8,6 +8,9 @@ import (
 	"time"
 
 	"github.com/onsi/gomega"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/zap/zapcore"
+
 	"github.com/smartcontractkit/chainlink-env/chaos"
 	"github.com/smartcontractkit/chainlink-env/environment"
 	a "github.com/smartcontractkit/chainlink-env/pkg/alias"
@@ -16,14 +19,12 @@ import (
 	"github.com/smartcontractkit/chainlink-env/pkg/helm/ethereum"
 	"github.com/smartcontractkit/chainlink-testing-framework/blockchain"
 	"github.com/smartcontractkit/chainlink-testing-framework/utils"
-	"github.com/stretchr/testify/require"
-	"go.uber.org/zap/zapcore"
 
-	networks "github.com/smartcontractkit/chainlink/integration-tests"
 	"github.com/smartcontractkit/chainlink/integration-tests/actions"
 	"github.com/smartcontractkit/chainlink/integration-tests/client"
 	"github.com/smartcontractkit/chainlink/integration-tests/contracts"
 	eth_contracts "github.com/smartcontractkit/chainlink/integration-tests/contracts/ethereum"
+	"github.com/smartcontractkit/chainlink/integration-tests/networks"
 )
 
 var (
@@ -38,6 +39,7 @@ Enabled = true
 Enabled = true
 AnnounceAddresses = ["0.0.0.0:8090"]
 ListenAddresses = ["0.0.0.0:8090"]`
+
 	defaultAutomationSettings = map[string]interface{}{
 		"toml":     client.AddNetworksConfig(baseTOML, networks.SelectedNetwork),
 		"replicas": "6",
@@ -107,6 +109,7 @@ const (
 func TestAutomationChaos(t *testing.T) {
 	t.Parallel()
 	l := utils.GetTestLogger(t)
+
 	testCases := map[string]struct {
 		networkChart environment.ConnectedChart
 		clChart      environment.ConnectedChart
@@ -169,7 +172,7 @@ func TestAutomationChaos(t *testing.T) {
 		testCase := tst
 		t.Run(fmt.Sprintf("Automation_%s", name), func(t *testing.T) {
 			t.Parallel()
-			network := networks.SelectedNetwork
+			network := networks.SelectedNetwork // Need a new copy of the network for each test
 
 			testEnvironment := environment.
 				New(&environment.Config{