diff --git a/.github/actions/build-test-image/action.yml b/.github/actions/build-test-image/action.yml index b1c694ea8bc..02d798c75b7 100644 --- a/.github/actions/build-test-image/action.yml +++ b/.github/actions/build-test-image/action.yml @@ -48,7 +48,7 @@ runs: file: ./integration-tests/test.Dockerfile build-args: | BASE_IMAGE=${{ inputs.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ inputs.QA_AWS_REGION }}.amazonaws.com/test-base-image - IMAGE_VERSION=v0.3.29 + IMAGE_VERSION=v0.32.3 SUITES="${{ inputs.suites }}" AWS_REGION: ${{ inputs.QA_AWS_REGION }} AWS_ROLE_TO_ASSUME: ${{ inputs.QA_AWS_ROLE_TO_ASSUME }} diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index b9cbef11079..ac51b491c30 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -252,6 +252,7 @@ jobs: this-job-name: ETH Smoke Tests ${{ matrix.product.name }} test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' continue-on-error: true + ### Used to check the required checks box when the matrix completes eth-smoke-tests: if: always() @@ -263,6 +264,64 @@ jobs: if: needs.eth-smoke-tests-matrix.result != 'success' run: exit 1 + ### Migration tests + node-migration-tests: + name: Version Migration Tests + environment: integration + permissions: + checks: write + pull-requests: write + id-token: write + contents: read + runs-on: ubuntu-latest + needs: [build-chainlink, changes, build-test-image] + env: + SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2 + CHAINLINK_COMMIT_SHA: ${{ github.sha }} + CHAINLINK_ENV_USER: ${{ github.actor }} + TEST_UPGRADE_VERSION: ${{ github.sha }} + TEST_LOG_LEVEL: debug + TEST_SUITE: migration + steps: + - name: Checkout the repo + uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + with: + ref: ${{ github.event.pull_request.head.sha }} + ## Run this step when changes that require tests to be run are made + - name: Run Migration Tests + if: needs.changes.outputs.src == 'true' + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@ab595504ae9cf10c60eb8d2c5ce025284e58b210 #v2.1.5 + with: + test_command_to_run: make test_need_operator_assets && cd ./integration-tests && go test -timeout 30m -count=1 -json ./migration 2>&1 | tee /tmp/gotest.log | gotestfmt + test_download_vendor_packages_command: cd ./integration-tests && go mod download + cl_repo: ${{ env.CHAINLINK_IMAGE }} + cl_image_tag: develop + artifacts_location: ./integration-tests/migration/logs + publish_check_name: Node Migration Test Results + token: ${{ secrets.GITHUB_TOKEN }} + go_mod_path: ./integration-tests/go.mod + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + - name: Upload test log + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + if: failure() + with: + name: test-log-${{ matrix.product.name }} + path: /tmp/gotest.log + retention-days: 7 + continue-on-error: true + - name: Collect Metrics + if: always() + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@8163dcea2f01a0a8fec84b284406ff7af1d2e1c0 + with: + basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_CLOUD_HOST }} + this-job-name: Version Migration Tests + test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' + continue-on-error: true + ### Solana Section get_solana_sha: name: Get Solana Sha From Go Mod diff --git a/integration-tests/.tool-versions b/integration-tests/.tool-versions index 80fe1955065..aadd7caf06f 100644 --- a/integration-tests/.tool-versions +++ b/integration-tests/.tool-versions @@ -1,4 +1,4 @@ -golang 1.20.3 +golang 1.20.4 k3d 5.4.6 kubectl 1.25.5 nodejs 18.13.0 diff --git a/integration-tests/Makefile b/integration-tests/Makefile index f0ec7c19571..50d81185d6a 100644 --- a/integration-tests/Makefile +++ b/integration-tests/Makefile @@ -125,6 +125,27 @@ test_perf: test_need_operator_assets ## Run core node performance tests. SELECTED_NETWORKS="SIMULATED,SIMULATED_1,SIMULATED_2" \ go test -timeout 1h -count=1 -json $(args) ./performance 2>&1 | tee /tmp/gotest.log | gotestfmt +# Migrations +.PHONY: test_node_migrations +test_node_migrations: install_gotestfmt ## Run all node migration tests + TEST_LOG_LEVEL="disabled" \ + go test -timeout 1h -count=1 -json $(args) ./migration 2>&1 | tee /tmp/gotest.log | gotestfmt + +.PHONY: test_node_migrations_simulated +test_node_migrations_simulated: install_gotestfmt + TEST_LOG_LEVEL="disabled" \ + SELECTED_NETWORKS="SIMULATED,SIMULATED_1,SIMULATED_2" \ + go test -timeout 1h -count=1 -json $(args) ./migration 2>&1 | tee /tmp/gotest.log | gotestfmt + +.PHONY: test_node_migrations_verbose +test_node_migrations_verbose: + go test -timeout 1h -count=1 -v $(args) ./migration + +.PHONY: test_node_migrations_simulated_verbose +test_node_migrations_simulated_verbose: + SELECTED_NETWORKS="SIMULATED,SIMULATED_1,SIMULATED_2" \ + go test -timeout 1h -count=1 -v $(args) ./migration + # Soak .PHONY: test_soak_ocr test_soak_ocr: diff --git a/integration-tests/actions/actions.go b/integration-tests/actions/actions.go index 1e4e8eab85b..9d72b4f576b 100644 --- a/integration-tests/actions/actions.go +++ b/integration-tests/actions/actions.go @@ -229,7 +229,7 @@ func TeardownSuite( logsFolderPath string, chainlinkNodes []*client.Chainlink, optionalTestReporter testreporters.TestReporter, // Optionally pass in a test reporter to log further metrics - failingLogLevel zapcore.Level, // Examines logs after the test, and fails the test if any Chainlink logs are found at or above provided level + failingLogLevel zapcore.Level, // Examines logs after the test, and fails the test if any Chainlink logs are found at or above provided level clients ...blockchain.EVMClient, ) error { l := utils.GetTestLogger(t) @@ -362,3 +362,25 @@ func EncodeOnChainExternalJobID(jobID uuid.UUID) [32]byte { copy(ji[:], strings.Replace(jobID.String(), "-", "", 4)) return ji } + +// UpgradeChainlinkNodeVersions upgrades all Chainlink nodes to a new version, and then runs the test environment +// to apply the upgrades +func UpgradeChainlinkNodeVersions( + testEnvironment *environment.Environment, + newImage, newVersion string, + nodes ...*client.Chainlink, +) error { + if newImage == "" && newVersion == "" { + return errors.New("unable to upgrade node version, found empty image and version, must provide either a new image or a new version") + } + for _, node := range nodes { + if err := node.UpgradeVersion(testEnvironment, newImage, newVersion); err != nil { + return err + } + } + err := testEnvironment.RunUpdated(len(nodes)) + if err != nil { // Run the new environment and wait for changes to show + return err + } + return client.ReconnectChainlinkNodes(testEnvironment, nodes) +} diff --git a/integration-tests/actions/automation_ocr_helpers.go b/integration-tests/actions/automation_ocr_helpers.go index ce26eab50c3..24f79ea8980 100644 --- a/integration-tests/actions/automation_ocr_helpers.go +++ b/integration-tests/actions/automation_ocr_helpers.go @@ -120,7 +120,7 @@ func CreateOCRKeeperJobs( ) { l := utils.GetTestLogger(t) bootstrapNode := chainlinkNodes[0] - bootstrapNode.RemoteIP() + bootstrapNode.InternalIP() bootstrapP2PIds, err := bootstrapNode.MustReadP2PKeys() require.NoError(t, err, "Shouldn't fail reading P2P keys from bootstrap node") bootstrapP2PId := bootstrapP2PIds.Data[0].Attributes.PeerID @@ -139,7 +139,7 @@ func CreateOCRKeeperJobs( } _, err = bootstrapNode.MustCreateJob(bootstrapSpec) require.NoError(t, err, "Shouldn't fail creating bootstrap job on bootstrap node") - P2Pv2Bootstrapper := fmt.Sprintf("%s@%s:%d", bootstrapP2PId, bootstrapNode.RemoteIP(), 6690) + P2Pv2Bootstrapper := fmt.Sprintf("%s@%s:%d", bootstrapP2PId, bootstrapNode.InternalIP(), 6690) for nodeIndex := 1; nodeIndex < len(chainlinkNodes); nodeIndex++ { nodeTransmitterAddress, err := chainlinkNodes[nodeIndex].EthAddresses() diff --git a/integration-tests/actions/ocr2_helpers.go b/integration-tests/actions/ocr2_helpers.go index 23586f1a802..fe9ff746191 100644 --- a/integration-tests/actions/ocr2_helpers.go +++ b/integration-tests/actions/ocr2_helpers.go @@ -268,7 +268,7 @@ func CreateOCRv2Jobs( if err != nil { return err } - p2pV2Bootstrapper := fmt.Sprintf("%s@%s:%d", bootstrapP2PIds.Data[0].Attributes.PeerID, bootstrapNode.RemoteIP(), 6690) + p2pV2Bootstrapper := fmt.Sprintf("%s@%s:%d", bootstrapP2PIds.Data[0].Attributes.PeerID, bootstrapNode.InternalIP(), 6690) // Set the value for the jobs to report on err = mockserver.SetValuePath(mockServerPath, mockServerValue) if err != nil { diff --git a/integration-tests/actions/ocr2vrf_actions/ocr2vrf_config_helpers.go b/integration-tests/actions/ocr2vrf_actions/ocr2vrf_config_helpers.go index 1cda2c2b871..78e101ef378 100644 --- a/integration-tests/actions/ocr2vrf_actions/ocr2vrf_config_helpers.go +++ b/integration-tests/actions/ocr2vrf_actions/ocr2vrf_config_helpers.go @@ -113,7 +113,7 @@ func createBootstrapJob(t *testing.T, bootstrapNode *client.Chainlink, dkgAddres } _, err = bootstrapNode.MustCreateJob(bootstrapSpec) require.NoError(t, err, "Shouldn't fail creating bootstrap job on bootstrap node") - return fmt.Sprintf("%s@%s:%d", bootstrapP2PId, bootstrapNode.RemoteIP(), 6690) + return fmt.Sprintf("%s@%s:%d", bootstrapP2PId, bootstrapNode.InternalIP(), 6690) } func BuildOCR2DKGConfigVars( diff --git a/integration-tests/actions/ocr_helpers.go b/integration-tests/actions/ocr_helpers.go index e3091aed4c3..9b3e81aff78 100644 --- a/integration-tests/actions/ocr_helpers.go +++ b/integration-tests/actions/ocr_helpers.go @@ -12,6 +12,7 @@ import ( "github.com/smartcontractkit/chainlink-testing-framework/blockchain" ctfClient "github.com/smartcontractkit/chainlink-testing-framework/client" + "github.com/smartcontractkit/chainlink/integration-tests/client" "github.com/smartcontractkit/chainlink/integration-tests/contracts" ) diff --git a/integration-tests/actions/operator_forwarder_helpers.go b/integration-tests/actions/operator_forwarder_helpers.go index 98498a77bfb..b812fb18b71 100644 --- a/integration-tests/actions/operator_forwarder_helpers.go +++ b/integration-tests/actions/operator_forwarder_helpers.go @@ -9,15 +9,14 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - "github.com/smartcontractkit/chainlink-testing-framework/utils" "github.com/stretchr/testify/require" "github.com/smartcontractkit/chainlink-testing-framework/blockchain" + "github.com/smartcontractkit/chainlink-testing-framework/utils" + "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/operator_factory" "github.com/smartcontractkit/chainlink/integration-tests/client" "github.com/smartcontractkit/chainlink/integration-tests/contracts" - - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/operator_factory" ) func DeployForwarderContracts( diff --git a/integration-tests/benchmark/keeper_test.go b/integration-tests/benchmark/keeper_test.go index 18dff6832ad..741f4f83d23 100644 --- a/integration-tests/benchmark/keeper_test.go +++ b/integration-tests/benchmark/keeper_test.go @@ -561,11 +561,6 @@ func SetupAutomationBenchmarkEnv(t *testing.T) (*environment.Environment, blockc return testEnvironment, activeEVMNetwork, registryToTest } - // For if we end up using env vars - keeperBenchmarkEnvVars["ETH_URL"] = activeEVMNetwork.URLs[0] - keeperBenchmarkEnvVars["ETH_HTTP_URL"] = activeEVMNetwork.HTTPURLs[0] - keeperBenchmarkEnvVars["ETH_CHAIN_ID"] = fmt.Sprint(activeEVMNetwork.ChainID) - // separate RPC urls per CL node internalWsURLs := make([]string, 0) internalHttpURLs := make([]string, 0) @@ -592,22 +587,13 @@ func SetupAutomationBenchmarkEnv(t *testing.T) (*environment.Environment, blockc l.Debug().Strs("internalWsURLs", internalWsURLs).Strs("internalHttpURLs", internalHttpURLs).Msg("internalURLs") for i := 0; i < NumberOfNodes; i++ { - useEnvVars := strings.ToLower(os.Getenv("TEST_USE_ENV_VAR_CONFIG")) - if useEnvVars == "true" { - testEnvironment.AddHelm(chainlink.NewVersioned(i, "0.0.11", map[string]any{ - "env": keeperBenchmarkEnvVars, - "chainlink": chainlinkResources, - "db": dbResources, - })) - } else { - activeEVMNetwork.HTTPURLs = []string{internalHttpURLs[i]} - activeEVMNetwork.URLs = []string{internalWsURLs[i]} - testEnvironment.AddHelm(chainlink.New(i, map[string]any{ - "toml": client.AddNetworkDetailedConfig(keeperBenchmarkBaseTOML, networkDetailTOML, activeEVMNetwork), - "chainlink": chainlinkResources, - "db": dbResources, - })) - } + activeEVMNetwork.HTTPURLs = []string{internalHttpURLs[i]} + activeEVMNetwork.URLs = []string{internalWsURLs[i]} + testEnvironment.AddHelm(chainlink.New(i, map[string]any{ + "toml": client.AddNetworkDetailedConfig(keeperBenchmarkBaseTOML, networkDetailTOML, activeEVMNetwork), + "chainlink": chainlinkResources, + "db": dbResources, + })) } err = testEnvironment.Run() require.NoError(t, err, "Error launching test environment") diff --git a/integration-tests/client/chainlink.go b/integration-tests/client/chainlink.go index 28d0501735a..2be9d726e88 100644 --- a/integration-tests/client/chainlink.go +++ b/integration-tests/client/chainlink.go @@ -5,6 +5,7 @@ import ( "fmt" "math/big" "net/http" + "os" "regexp" "strings" "sync" @@ -16,7 +17,6 @@ import ( "golang.org/x/sync/errgroup" "github.com/smartcontractkit/chainlink-env/environment" - chainlinkChart "github.com/smartcontractkit/chainlink-env/pkg/helm/chainlink" ) const ( @@ -991,9 +991,9 @@ func (c *Chainlink) CreateStarkNetNode(node *StarkNetNodeAttributes) (*StarkNetN return &response, resp.RawResponse, err } -// RemoteIP retrieves the inter-cluster IP of the Chainlink node, for use with inter-node communications -func (c *Chainlink) RemoteIP() string { - return c.Config.RemoteIP +// InternalIP retrieves the inter-cluster IP of the Chainlink node, for use with inter-node communications +func (c *Chainlink) InternalIP() string { + return c.Config.InternalIP } // Profile starts a profile session on the Chainlink node for a pre-determined length, then runs the provided function @@ -1066,24 +1066,58 @@ func (c *Chainlink) SetPageSize(size int) { // ConnectChainlinkNodes creates new Chainlink clients func ConnectChainlinkNodes(e *environment.Environment) ([]*Chainlink, error) { var clients []*Chainlink - localURLs := e.URLs[chainlinkChart.NodesLocalURLsKey] - internalURLs := e.URLs[chainlinkChart.NodesInternalURLsKey] - for i, localURL := range localURLs { - internalHost := parseHostname(internalURLs[i]) + for _, nodeDetails := range e.ChainlinkNodeDetails { c, err := NewChainlink(&ChainlinkConfig{ - URL: localURL, - Email: "notreal@fakeemail.ch", - Password: "fj293fbBnlQ!f9vNs", - RemoteIP: internalHost, + URL: nodeDetails.LocalIP, + Email: "notreal@fakeemail.ch", + Password: "fj293fbBnlQ!f9vNs", + InternalIP: parseHostname(nodeDetails.InternalIP), + ChartName: nodeDetails.ChartName, + PodName: nodeDetails.PodName, }) if err != nil { return nil, err } + log.Debug(). + Str("URL", c.Config.URL). + Str("Internal IP", c.Config.InternalIP). + Str("Chart Name", c.Config.ChartName). + Str("Pod Name", c.Config.PodName). + Msg("Connected to Chainlink node") clients = append(clients, c) } return clients, nil } +// ReconnectChainlinkNodes reconnects to Chainlink nodes after they have been modified, say through a Helm upgrade +// Note: Experimental as of now, will likely not work predictably. +func ReconnectChainlinkNodes(testEnvironment *environment.Environment, nodes []*Chainlink) (err error) { + for _, node := range nodes { + for _, details := range testEnvironment.ChainlinkNodeDetails { + if details.ChartName == node.Config.ChartName { // Make the link from client to pod consistent + node, err = NewChainlink(&ChainlinkConfig{ + URL: details.LocalIP, + Email: "notreal@fakeemail.ch", + Password: "fj293fbBnlQ!f9vNs", + InternalIP: parseHostname(details.InternalIP), + ChartName: details.ChartName, + PodName: details.PodName, + }) + if err != nil { + return err + } + log.Debug(). + Str("URL", node.Config.URL). + Str("Internal IP", node.Config.InternalIP). + Str("Chart Name", node.Config.ChartName). + Str("Pod Name", node.Config.PodName). + Msg("Reconnected to Chainlink node") + } + } + } + return nil +} + func parseHostname(s string) string { r := regexp.MustCompile(`://(?P.*):`) return r.FindStringSubmatch(s)[1] @@ -1208,3 +1242,32 @@ func (c *Chainlink) GetForwarders() (*Forwarders, *http.Response, error) { } return response, resp.RawResponse, err } + +// UpgradeVersion upgrades the chainlink node to the new version +// Note: You need to call Run() on the test environment for changes to take effect +// Note: This function is not thread safe, call from a single thread +func (c *Chainlink) UpgradeVersion(testEnvironment *environment.Environment, newImage, newVersion string) error { + if newVersion == "" { + return fmt.Errorf("new version is empty") + } + if newImage == "" { + newImage = os.Getenv("CHAINLINK_IMAGE") + } + log.Info(). + Str("Chart Name", c.Config.ChartName). + Str("Old Image", os.Getenv("CHAINLINK_IMAGE")). + Str("Old Version", os.Getenv("CHAINLINK_VERSION")). + Str("New Image", newImage). + Str("New Version", newVersion). + Msg("Upgrading Chainlink Node") + upgradeVals := map[string]any{ + "chainlink": map[string]any{ + "image": map[string]any{ + "image": newImage, + "version": newVersion, + }, + }, + } + testEnvironment, err := testEnvironment.UpdateHelm(c.Config.ChartName, upgradeVals) + return err +} diff --git a/integration-tests/client/chainlink_models.go b/integration-tests/client/chainlink_models.go index 1722538b5c3..06796986731 100644 --- a/integration-tests/client/chainlink_models.go +++ b/integration-tests/client/chainlink_models.go @@ -6,9 +6,9 @@ import ( "text/template" "time" + "github.com/ethereum/go-ethereum/common" "gopkg.in/guregu/null.v4" - "github.com/ethereum/go-ethereum/common" "github.com/smartcontractkit/chainlink/v2/core/services/job" ) @@ -19,10 +19,12 @@ type EIServiceConfig struct { // ChainlinkConfig represents the variables needed to connect to a Chainlink node type ChainlinkConfig struct { - URL string - Email string - Password string - RemoteIP string + URL string + Email string + Password string + InternalIP string // Can change if the node is restarted. Prefer RemoteURL if possible + ChartName string + PodName string } // ResponseSlice is the generic model that can be used for all Chainlink API responses that are an slice @@ -867,16 +869,16 @@ type OCRTaskJobSpec struct { // P2PData holds the remote ip and the peer id and port type P2PData struct { - RemoteIP string - RemotePort string - PeerID string + InternalIP string + InternalPort string + PeerID string } func (p *P2PData) P2PV2Bootstrapper() string { - if p.RemotePort == "" { - p.RemotePort = "6690" + if p.InternalPort == "" { + p.InternalPort = "6690" } - return fmt.Sprintf("%s@%s:%s", p.PeerID, p.RemoteIP, p.RemotePort) + return fmt.Sprintf("%s@%s:%s", p.PeerID, p.InternalIP, p.InternalPort) } // Type returns the type of the job @@ -892,8 +894,8 @@ func (o *OCRTaskJobSpec) String() (string, error) { return "", err } peers = append(peers, P2PData{ - RemoteIP: peer.RemoteIP(), - PeerID: p2pKeys.Data[0].Attributes.PeerID, + InternalIP: peer.InternalIP(), + PeerID: p2pKeys.Data[0].Attributes.PeerID, }) } specWrap := struct { @@ -938,7 +940,7 @@ contractAddress = "{{.ContractAddress}}" {{if .P2PBootstrapPeers}} p2pBootstrapPeers = [ {{range $peer := .P2PBootstrapPeers}} - "/dns4/{{$peer.RemoteIP}}/tcp/6690/p2p/{{$peer.PeerID}}", + "/dns4/{{$peer.InternalIP}}/tcp/6690/p2p/{{$peer.PeerID}}", {{end}} ] {{else}} diff --git a/integration-tests/contracts/ethereum_contracts.go b/integration-tests/contracts/ethereum_contracts.go index a6b25f11f92..012e03cdc41 100644 --- a/integration-tests/contracts/ethereum_contracts.go +++ b/integration-tests/contracts/ethereum_contracts.go @@ -776,6 +776,9 @@ func (o *EthereumOffchainAggregator) SetConfig( if err != nil { return err } + if len(ocrKeys.Data) == 0 { + return fmt.Errorf("no OCR keys found for node %s", node.Config.ChartName) + } primaryOCRKey := ocrKeys.Data[0] if err != nil { return err diff --git a/integration-tests/example.env b/integration-tests/example.env index ee0a30d7064..8570fa86e1e 100644 --- a/integration-tests/example.env +++ b/integration-tests/example.env @@ -1,27 +1,33 @@ # An example template that you can use for your own .env file for integration test settings # `source ./integration-tests/.env` -# Test Settings +########## General Test Settings ########## export KEEP_ENVIRONMENTS="Never" # Always | OnFail | Never export CHAINLINK_IMAGE="public.ecr.aws/chainlink/chainlink" # Image repo to pull the Chainlink image from export CHAINLINK_VERSION="1.13.0" # Version of the Chainlink image to pull export CHAINLINK_ENV_USER="Satoshi-Nakamoto" # Name of the person running the tests (change to your own) export TEST_LOG_LEVEL="info" # info | debug | trace -# Soak/Chaos/Load Test Specific Settings +########## Soak/Chaos/Load Test Specific Settings ########## +# Remote Runner export ENV_JOB_IMAGE="image-location/chainlink-tests:test-tag" # Image repo to pull the remote-test-runner image from. Check the Integration Tests workflow. export DETACH_RUNNER="true" # true 99% of the time, false if you are debugging soak test issues export TEST_SUITE="soak" # soak | chaos | load -export OCR_TEST_DURATION="15m" # For an OCRv1 soak test, how long to run the test for -export OCR_CHAINLINK_NODE_FUNDING=".01" # For an OCRv1 soak test, how much ETH to send to each node -export OCR_TIME_BETWEEN_ROUNDS="1m" # For an OCRv1 soak test, how long to wait between starting new rounds - # Slack Notification Settings export SLACK_API_KEY="xoxb-example-key" # API key used to report soak test results to slack export SLACK_CHANNEL="C000000000" # Channel ID for the slack bot to post test results export SLACK_USER="U000000000" # User ID of the person running the soak tests to properly notify them +# OCR Soak Test Settings +export OCR_TEST_DURATION="15m" # For an OCRv1 soak test, how long to run the test for +export OCR_CHAINLINK_NODE_FUNDING=".01" # For an OCRv1 soak test, how much ETH to send to each node +export OCR_TIME_BETWEEN_ROUNDS="1m" # For an OCRv1 soak test, how long to wait between starting new rounds + +# Node Version Upgrade Settings +export TEST_UPGRADE_IMAGE="" # Image path to use for the node version upgrade test, likely empty +export TEST_UPGRADE_VERSION="2.0.0" # Version of the Chainlink image to upgrade to for upgrade tests. Should be newer than the current version + ########## Network Settings ########## # Select a pre-defined network(s) diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 162c7447305..e377d040073 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -15,13 +15,14 @@ require ( github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.29.1 github.com/slack-go/slack v0.12.2 - github.com/smartcontractkit/chainlink-env v0.32.1 + github.com/smartcontractkit/chainlink-env v0.32.2 github.com/smartcontractkit/chainlink-testing-framework v1.11.6 github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 github.com/smartcontractkit/libocr v0.0.0-20230413082317-9561d14087cc github.com/smartcontractkit/ocr2keepers v0.6.15 github.com/smartcontractkit/ocr2vrf v0.0.0-20230510102715-c58be582bf19 github.com/stretchr/testify v1.8.2 + github.com/test-go/testify v1.1.4 github.com/umbracle/ethgo v0.1.3 go.dedis.ch/kyber/v3 v3.0.14 go.uber.org/zap v1.24.0 @@ -370,7 +371,7 @@ require ( ) replace ( - // Fix go mod tidy issue for ambiguous imports from go-ethereum + // Fixes go mod tidy issue for ambiguous imports from go-ethereum // See https://github.com/ugorji/go/issues/279 github.com/btcsuite/btcd => github.com/btcsuite/btcd v0.22.1 diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 557d209969d..2556132766b 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1341,8 +1341,8 @@ github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs github.com/slack-go/slack v0.12.2 h1:x3OppyMyGIbbiyFhsBmpf9pwkUzMhthJMRNmNlA4LaQ= github.com/slack-go/slack v0.12.2/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw= github.com/smartcontractkit/chainlink-cosmos v0.4.0 h1:xYLAcJJIm0cyMtYtMaosO45bykEwcwQOmZz3mz46Y2A= -github.com/smartcontractkit/chainlink-env v0.32.1 h1:gHM3s+LtjYKyqIp+A4a3tfm4WJSIlXKFbjRUPNPtEwg= -github.com/smartcontractkit/chainlink-env v0.32.1/go.mod h1:9c0Czq4a6wZKY20BcoAlK29DnejQIiLo/MwKYtSFnHk= +github.com/smartcontractkit/chainlink-env v0.32.2 h1:4o9TyvseEIGV/MGqkklQKfrsXUII4hlC8yq6NWC6dmU= +github.com/smartcontractkit/chainlink-env v0.32.2/go.mod h1:9c0Czq4a6wZKY20BcoAlK29DnejQIiLo/MwKYtSFnHk= github.com/smartcontractkit/chainlink-relay v0.1.7-0.20230517200758-40dada2543a1 h1:/gqM+lMSfoxdNSeruMgEdqA7a/RJuz6l36LPuKoI1EU= github.com/smartcontractkit/chainlink-relay v0.1.7-0.20230517200758-40dada2543a1/go.mod h1:f7/HKcJWWFzINrkDDlpKsFaU6D+8D4B4OrQxkkCX4oc= github.com/smartcontractkit/chainlink-solana v1.0.3-0.20230516001004-0216997a2508 h1:hsCcqkK7ZgABURMRYbNgRdVbNJv7JJPGlgKJbukjMsk= diff --git a/integration-tests/migration/migration_up_test.go b/integration-tests/migration/migration_up_test.go deleted file mode 100644 index b626726cbee..00000000000 --- a/integration-tests/migration/migration_up_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package migration_test - -import ( - "os" - "strings" - "testing" - "time" - - "github.com/smartcontractkit/chainlink-testing-framework/utils" - "github.com/stretchr/testify/require" - - "github.com/smartcontractkit/chainlink-env/environment" - "github.com/smartcontractkit/chainlink-env/logging" - ctfClient "github.com/smartcontractkit/chainlink-testing-framework/client" - "github.com/smartcontractkit/chainlink-testing-framework/testsetups" -) - -type Data struct { - ID int `db:"id"` - Cfg []byte `db:"cfg"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt time.Time `db:"updated_at"` - Enabled bool `db:"enabled"` -} - -func TestMain(m *testing.M) { - logging.Init() - os.Exit(m.Run()) -} - -func TestMigrationDatabase(t *testing.T) { - l := utils.GetTestLogger(t) - testEnvironment, err := testsetups.DBMigration(&testsetups.DBMigrationSpec{ - FromSpec: testsetups.FromVersionSpec{ - Image: "public.ecr.aws/chainlink/chainlink", - Tag: "1.6.0-nonroot", - }, - ToSpec: testsetups.ToVersionSpec{ - Image: "public.ecr.aws/chainlink/chainlink", - Tag: "1.7.1-nonroot", - }, - }) - require.NoError(t, err, "Error setting up DBMigration test") - // if test haven't failed after that assertion we know that migration is complete - // check other stuff via queries if needed - db := getDB(t, testEnvironment) - var d []Data - err = db.Select(&d, "select * from evm_chains;") - require.NoError(t, err, "Error running SELECT") - l.Info().Interface("Rows", d).Send() -} - -func getDB(t *testing.T, testEnvironment *environment.Environment) *ctfClient.PostgresConnector { - spl := strings.Split(testEnvironment.URLs["chainlink_db"][1], ":") - port := spl[len(spl)-1] - db, err := ctfClient.NewPostgresConnector(&ctfClient.PostgresConfig{ - Host: "localhost", - Port: port, - User: "postgres", - Password: "postgres", - DBName: "chainlink", - }) - require.NoError(t, err, "Error connecting to postgres") - return db -} diff --git a/integration-tests/migration/upgrade_version_test.go b/integration-tests/migration/upgrade_version_test.go new file mode 100644 index 00000000000..695a3354dca --- /dev/null +++ b/integration-tests/migration/upgrade_version_test.go @@ -0,0 +1,78 @@ +package migration + +import ( + "fmt" + "strings" + "testing" + + "github.com/test-go/testify/require" + "go.uber.org/zap/zapcore" + + "github.com/smartcontractkit/chainlink-env/environment" + "github.com/smartcontractkit/chainlink-env/pkg/helm/chainlink" + "github.com/smartcontractkit/chainlink-env/pkg/helm/ethereum" + "github.com/smartcontractkit/chainlink-testing-framework/blockchain" + "github.com/smartcontractkit/chainlink-testing-framework/utils" + + networks "github.com/smartcontractkit/chainlink/integration-tests" + "github.com/smartcontractkit/chainlink/integration-tests/actions" + "github.com/smartcontractkit/chainlink/integration-tests/client" + "github.com/smartcontractkit/chainlink/integration-tests/config" +) + +func TestVersionUpgrade(t *testing.T) { + t.Parallel() + testEnvironment, testNetwork := setupUpgradeTest(t) + if testEnvironment.WillUseRemoteRunner() { + return + } + + upgradeImage, err := utils.GetEnv("UPGRADE_IMAGE") + require.NoError(t, err, "Error getting upgrade image") + upgradeVersion, err := utils.GetEnv("UPGRADE_VERSION") + require.NoError(t, err, "Error getting upgrade version") + + chainClient, err := blockchain.NewEVMClient(testNetwork, testEnvironment) + require.NoError(t, err, "Connecting to blockchain nodes shouldn't fail") + chainlinkNodes, err := client.ConnectChainlinkNodes(testEnvironment) + require.NoError(t, err, "Connecting to chainlink nodes shouldn't fail") + + t.Cleanup(func() { + err := actions.TeardownSuite(t, testEnvironment, utils.ProjectRoot, chainlinkNodes, nil, zapcore.ErrorLevel, chainClient) + require.NoError(t, err, "Error tearing down environment") + }) + + err = actions.UpgradeChainlinkNodeVersions(testEnvironment, upgradeImage, upgradeVersion, chainlinkNodes[0]) + require.NoError(t, err, "Upgrading chainlink nodes shouldn't fail") +} + +func setupUpgradeTest(t *testing.T) ( + testEnvironment *environment.Environment, + testNetwork blockchain.EVMNetwork, +) { + testNetwork = networks.SelectedNetwork + evmConfig := ethereum.New(nil) + if !testNetwork.Simulated { + evmConfig = ethereum.New(ðereum.Props{ + NetworkName: testNetwork.Name, + Simulated: testNetwork.Simulated, + WsURLs: testNetwork.URLs, + }) + } + charts, err := chainlink.NewDeployment(1, map[string]any{ + "toml": client.AddNetworksConfig(config.BaseOCRP2PV1Config, testNetwork), + "db": map[string]any{ + "stateful": true, + }, + }) + require.NoError(t, err, "Error creating chainlink deployments") + testEnvironment = environment.New(&environment.Config{ + NamespacePrefix: fmt.Sprintf("upgrade-version-%s", strings.ReplaceAll(strings.ToLower(testNetwork.Name), " ", "-")), + Test: t, + }). + AddHelm(evmConfig). + AddHelmCharts(charts) + err = testEnvironment.Run() + require.NoError(t, err, "Error launching test environment") + return testEnvironment, testNetwork +} diff --git a/integration-tests/smoke/automation_test.go b/integration-tests/smoke/automation_test.go index 6fdc3e272b8..8b1ed725b03 100644 --- a/integration-tests/smoke/automation_test.go +++ b/integration-tests/smoke/automation_test.go @@ -778,10 +778,6 @@ func setupAutomationTest( Simulated: network.Simulated, WsURLs: network.URLs, }) - // For if we end up using env vars - automationEnvVars["ETH_URL"] = network.URLs[0] - automationEnvVars["ETH_HTTP_URL"] = network.HTTPURLs[0] - automationEnvVars["ETH_CHAIN_ID"] = fmt.Sprint(network.ChainID) } chainlinkChart := chainlink.New(0, map[string]any{ "replicas": "5", @@ -789,14 +785,6 @@ func setupAutomationTest( "secretsToml": client.AddSecretTomlConfig("https://google.com", "username1", "password1"), }) - useEnvVars := strings.ToLower(os.Getenv("TEST_USE_ENV_VAR_CONFIG")) - if useEnvVars == "true" { - chainlinkChart = chainlink.NewVersioned(0, "0.0.11", map[string]any{ - "replicas": "5", - "env": automationEnvVars, - }) - } - testEnvironment := environment.New(&environment.Config{ NamespacePrefix: fmt.Sprintf("smoke-automation-%s-%s", testName, strings.ReplaceAll(strings.ToLower(network.Name), " ", "-")), Test: t, diff --git a/integration-tests/smoke/keeper_test.go b/integration-tests/smoke/keeper_test.go index 781baa9fdba..d1b569bf516 100644 --- a/integration-tests/smoke/keeper_test.go +++ b/integration-tests/smoke/keeper_test.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "math/big" - "os" "strconv" "strings" "testing" @@ -1125,10 +1124,6 @@ func setupKeeperTest( Simulated: network.Simulated, WsURLs: network.URLs, }) - // For if we end up using env vars - keeperEnvVars["ETH_URL"] = network.URLs[0] - keeperEnvVars["ETH_HTTP_URL"] = network.HTTPURLs[0] - keeperEnvVars["ETH_CHAIN_ID"] = fmt.Sprint(network.ChainID) } chainlinkChart := chainlink.New(0, map[string]interface{}{ @@ -1136,14 +1131,6 @@ func setupKeeperTest( "toml": client.AddNetworksConfig(keeperBaseTOML, network), }) - useEnvVars := strings.ToLower(os.Getenv("TEST_USE_ENV_VAR_CONFIG")) - if useEnvVars == "true" { - chainlinkChart = chainlink.NewVersioned(0, "0.0.11", map[string]any{ - "replicas": "5", - "env": keeperEnvVars, - }) - } - networkName := strings.ReplaceAll(strings.ToLower(network.Name), " ", "-") testEnvironment := environment.New( &environment.Config{ diff --git a/integration-tests/smoke/ocr_test.go b/integration-tests/smoke/ocr_test.go index 690d9f4c9fe..8e8f5d9493e 100644 --- a/integration-tests/smoke/ocr_test.go +++ b/integration-tests/smoke/ocr_test.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "math/big" - "os" "strings" "testing" @@ -81,13 +80,10 @@ func TestOCRBasic(t *testing.T) { require.Equal(t, int64(10), answer.Int64(), "Expected latest answer from OCR contract to be 10 but got %d", answer.Int64()) } -var ocrEnvVars = map[string]any{} - func setupOCRTest(t *testing.T) ( testEnvironment *environment.Environment, testNetwork blockchain.EVMNetwork, ) { - l := utils.GetTestLogger(t) testNetwork = networks.SelectedNetwork evmConfig := ethereum.New(nil) if !testNetwork.Simulated { @@ -96,25 +92,12 @@ func setupOCRTest(t *testing.T) ( Simulated: testNetwork.Simulated, WsURLs: testNetwork.URLs, }) - // For if we end up using env vars - ocrEnvVars["ETH_URL"] = testNetwork.URLs[0] - ocrEnvVars["ETH_HTTP_URL"] = testNetwork.HTTPURLs[0] - ocrEnvVars["ETH_CHAIN_ID"] = fmt.Sprint(testNetwork.ChainID) } chainlinkChart := chainlink.New(0, map[string]interface{}{ "toml": client.AddNetworksConfig(config.BaseOCRP2PV1Config, testNetwork), "replicas": 6, }) - useEnvVars := strings.ToLower(os.Getenv("TEST_USE_ENV_VAR_CONFIG")) - if useEnvVars == "true" { - chainlinkChart = chainlink.NewVersioned(0, "0.0.11", map[string]any{ - "replicas": 6, - "env": ocrEnvVars, - }) - l.Debug().Interface("Env", ocrEnvVars).Msg("Using Environment Variable Config") - } - testEnvironment = environment.New(&environment.Config{ NamespacePrefix: fmt.Sprintf("smoke-ocr-%s", strings.ReplaceAll(strings.ToLower(testNetwork.Name), " ", "-")), Test: t, diff --git a/integration-tests/soak/forwarder_ocr_test.go b/integration-tests/soak/forwarder_ocr_test.go index 7604bc038fc..723d7ec5318 100644 --- a/integration-tests/soak/forwarder_ocr_test.go +++ b/integration-tests/soak/forwarder_ocr_test.go @@ -8,7 +8,6 @@ import ( "testing" "time" - "github.com/smartcontractkit/chainlink-testing-framework/utils" "github.com/stretchr/testify/require" "github.com/smartcontractkit/chainlink-env/environment" @@ -18,6 +17,7 @@ import ( "github.com/smartcontractkit/chainlink-env/pkg/helm/mockserver" mockservercfg "github.com/smartcontractkit/chainlink-env/pkg/helm/mockserver-cfg" "github.com/smartcontractkit/chainlink-testing-framework/blockchain" + "github.com/smartcontractkit/chainlink-testing-framework/utils" networks "github.com/smartcontractkit/chainlink/integration-tests" "github.com/smartcontractkit/chainlink/integration-tests/actions" @@ -62,13 +62,6 @@ func TestForwarderOCRSoak(t *testing.T) { func SetupForwarderOCRSoakEnv(t *testing.T) (*environment.Environment, blockchain.EVMNetwork) { var ( - ocrForwarderEnvVars = map[string]any{ - "FEATURE_LOG_POLLER": "true", - "ETH_USE_FORWARDERS": "true", - "P2P_LISTEN_IP": "0.0.0.0", - "P2P_LISTEN_PORT": "6690", - } - ocrForwarderBaseTOML = `[OCR] Enabled = true @@ -85,9 +78,6 @@ func SetupForwarderOCRSoakEnv(t *testing.T) (*environment.Environment, blockchai ForwardersEnabled = true` ) network := networks.SelectedNetwork // Environment currently being used to soak test on - ocrForwarderEnvVars["ETH_URL"] = network.URLs[0] - ocrForwarderEnvVars["ETH_HTTP_URL"] = network.HTTPURLs[0] - ocrForwarderEnvVars["ETH_CHAIN_ID"] = fmt.Sprint(network.ChainID) baseEnvironmentConfig := &environment.Config{ TTL: time.Hour * 720, // 30 days, @@ -109,16 +99,9 @@ func SetupForwarderOCRSoakEnv(t *testing.T) (*environment.Environment, blockchai WsURLs: network.URLs, })) for i := 0; i < replicas; i++ { - useEnvVars := strings.ToLower(os.Getenv("TEST_USE_ENV_VAR_CONFIG")) - if useEnvVars == "true" { - testEnvironment.AddHelm(chainlink.NewVersioned(i, "0.0.11", map[string]any{ - "env": ocrForwarderEnvVars, - })) - } else { - testEnvironment.AddHelm(chainlink.New(i, map[string]interface{}{ - "toml": client.AddNetworkDetailedConfig(ocrForwarderBaseTOML, ocrForwarderNetworkDetailTOML, network), - })) - } + testEnvironment.AddHelm(chainlink.New(i, map[string]interface{}{ + "toml": client.AddNetworkDetailedConfig(ocrForwarderBaseTOML, ocrForwarderNetworkDetailTOML, network), + })) } err := testEnvironment.Run() diff --git a/integration-tests/soak/ocr_test.go b/integration-tests/soak/ocr_test.go index a0617afc341..2f8a5181421 100644 --- a/integration-tests/soak/ocr_test.go +++ b/integration-tests/soak/ocr_test.go @@ -10,7 +10,6 @@ import ( "time" "github.com/kelseyhightower/envconfig" - "github.com/smartcontractkit/chainlink-testing-framework/utils" "github.com/stretchr/testify/require" "github.com/smartcontractkit/chainlink-env/environment" @@ -19,6 +18,7 @@ import ( "github.com/smartcontractkit/chainlink-env/pkg/helm/mockserver" mockservercfg "github.com/smartcontractkit/chainlink-env/pkg/helm/mockserver-cfg" "github.com/smartcontractkit/chainlink-testing-framework/blockchain" + "github.com/smartcontractkit/chainlink-testing-framework/utils" networks "github.com/smartcontractkit/chainlink/integration-tests" "github.com/smartcontractkit/chainlink/integration-tests/actions"