Skip to content

Commit

Permalink
[TT-294] Node Version Tests (smartcontractkit#9177)
Browse files Browse the repository at this point in the history
  • Loading branch information
kalverra authored May 18, 2023
1 parent 6c11975 commit 3a31f1e
Show file tree
Hide file tree
Showing 24 changed files with 313 additions and 196 deletions.
2 changes: 1 addition & 1 deletion .github/actions/build-test-image/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ runs:
file: ./integration-tests/test.Dockerfile
build-args: |
BASE_IMAGE=${{ inputs.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ inputs.QA_AWS_REGION }}.amazonaws.com/test-base-image
IMAGE_VERSION=v0.3.29
IMAGE_VERSION=v0.32.3
SUITES="${{ inputs.suites }}"
AWS_REGION: ${{ inputs.QA_AWS_REGION }}
AWS_ROLE_TO_ASSUME: ${{ inputs.QA_AWS_ROLE_TO_ASSUME }}
Expand Down
59 changes: 59 additions & 0 deletions .github/workflows/integration-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,7 @@ jobs:
this-job-name: ETH Smoke Tests ${{ matrix.product.name }}
test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
continue-on-error: true

### Used to check the required checks box when the matrix completes
eth-smoke-tests:
if: always()
Expand All @@ -263,6 +264,64 @@ jobs:
if: needs.eth-smoke-tests-matrix.result != 'success'
run: exit 1

### Migration tests
node-migration-tests:
name: Version Migration Tests
environment: integration
permissions:
checks: write
pull-requests: write
id-token: write
contents: read
runs-on: ubuntu-latest
needs: [build-chainlink, changes, build-test-image]
env:
SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2
CHAINLINK_COMMIT_SHA: ${{ github.sha }}
CHAINLINK_ENV_USER: ${{ github.actor }}
TEST_UPGRADE_VERSION: ${{ github.sha }}
TEST_LOG_LEVEL: debug
TEST_SUITE: migration
steps:
- name: Checkout the repo
uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
with:
ref: ${{ github.event.pull_request.head.sha }}
## Run this step when changes that require tests to be run are made
- name: Run Migration Tests
if: needs.changes.outputs.src == 'true'
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@ab595504ae9cf10c60eb8d2c5ce025284e58b210 #v2.1.5
with:
test_command_to_run: make test_need_operator_assets && cd ./integration-tests && go test -timeout 30m -count=1 -json ./migration 2>&1 | tee /tmp/gotest.log | gotestfmt
test_download_vendor_packages_command: cd ./integration-tests && go mod download
cl_repo: ${{ env.CHAINLINK_IMAGE }}
cl_image_tag: develop
artifacts_location: ./integration-tests/migration/logs
publish_check_name: Node Migration Test Results
token: ${{ secrets.GITHUB_TOKEN }}
go_mod_path: ./integration-tests/go.mod
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
- name: Upload test log
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
if: failure()
with:
name: test-log-${{ matrix.product.name }}
path: /tmp/gotest.log
retention-days: 7
continue-on-error: true
- name: Collect Metrics
if: always()
id: collect-gha-metrics
uses: smartcontractkit/push-gha-metrics-action@8163dcea2f01a0a8fec84b284406ff7af1d2e1c0
with:
basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
this-job-name: Version Migration Tests
test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
continue-on-error: true

### Solana Section
get_solana_sha:
name: Get Solana Sha From Go Mod
Expand Down
2 changes: 1 addition & 1 deletion integration-tests/.tool-versions
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
golang 1.20.3
golang 1.20.4
k3d 5.4.6
kubectl 1.25.5
nodejs 18.13.0
21 changes: 21 additions & 0 deletions integration-tests/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,27 @@ test_perf: test_need_operator_assets ## Run core node performance tests.
SELECTED_NETWORKS="SIMULATED,SIMULATED_1,SIMULATED_2" \
go test -timeout 1h -count=1 -json $(args) ./performance 2>&1 | tee /tmp/gotest.log | gotestfmt

# Migrations
.PHONY: test_node_migrations
test_node_migrations: install_gotestfmt ## Run all node migration tests
TEST_LOG_LEVEL="disabled" \
go test -timeout 1h -count=1 -json $(args) ./migration 2>&1 | tee /tmp/gotest.log | gotestfmt

.PHONY: test_node_migrations_simulated
test_node_migrations_simulated: install_gotestfmt
TEST_LOG_LEVEL="disabled" \
SELECTED_NETWORKS="SIMULATED,SIMULATED_1,SIMULATED_2" \
go test -timeout 1h -count=1 -json $(args) ./migration 2>&1 | tee /tmp/gotest.log | gotestfmt

.PHONY: test_node_migrations_verbose
test_node_migrations_verbose:
go test -timeout 1h -count=1 -v $(args) ./migration

.PHONY: test_node_migrations_simulated_verbose
test_node_migrations_simulated_verbose:
SELECTED_NETWORKS="SIMULATED,SIMULATED_1,SIMULATED_2" \
go test -timeout 1h -count=1 -v $(args) ./migration

# Soak
.PHONY: test_soak_ocr
test_soak_ocr:
Expand Down
24 changes: 23 additions & 1 deletion integration-tests/actions/actions.go
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,7 @@ func TeardownSuite(
logsFolderPath string,
chainlinkNodes []*client.Chainlink,
optionalTestReporter testreporters.TestReporter, // Optionally pass in a test reporter to log further metrics
failingLogLevel zapcore.Level, // Examines logs after the test, and fails the test if any Chainlink logs are found at or above provided level
failingLogLevel zapcore.Level, // Examines logs after the test, and fails the test if any Chainlink logs are found at or above provided level
clients ...blockchain.EVMClient,
) error {
l := utils.GetTestLogger(t)
Expand Down Expand Up @@ -362,3 +362,25 @@ func EncodeOnChainExternalJobID(jobID uuid.UUID) [32]byte {
copy(ji[:], strings.Replace(jobID.String(), "-", "", 4))
return ji
}

// UpgradeChainlinkNodeVersions upgrades all Chainlink nodes to a new version, and then runs the test environment
// to apply the upgrades
func UpgradeChainlinkNodeVersions(
testEnvironment *environment.Environment,
newImage, newVersion string,
nodes ...*client.Chainlink,
) error {
if newImage == "" && newVersion == "" {
return errors.New("unable to upgrade node version, found empty image and version, must provide either a new image or a new version")
}
for _, node := range nodes {
if err := node.UpgradeVersion(testEnvironment, newImage, newVersion); err != nil {
return err
}
}
err := testEnvironment.RunUpdated(len(nodes))
if err != nil { // Run the new environment and wait for changes to show
return err
}
return client.ReconnectChainlinkNodes(testEnvironment, nodes)
}
4 changes: 2 additions & 2 deletions integration-tests/actions/automation_ocr_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ func CreateOCRKeeperJobs(
) {
l := utils.GetTestLogger(t)
bootstrapNode := chainlinkNodes[0]
bootstrapNode.RemoteIP()
bootstrapNode.InternalIP()
bootstrapP2PIds, err := bootstrapNode.MustReadP2PKeys()
require.NoError(t, err, "Shouldn't fail reading P2P keys from bootstrap node")
bootstrapP2PId := bootstrapP2PIds.Data[0].Attributes.PeerID
Expand All @@ -139,7 +139,7 @@ func CreateOCRKeeperJobs(
}
_, err = bootstrapNode.MustCreateJob(bootstrapSpec)
require.NoError(t, err, "Shouldn't fail creating bootstrap job on bootstrap node")
P2Pv2Bootstrapper := fmt.Sprintf("%s@%s:%d", bootstrapP2PId, bootstrapNode.RemoteIP(), 6690)
P2Pv2Bootstrapper := fmt.Sprintf("%s@%s:%d", bootstrapP2PId, bootstrapNode.InternalIP(), 6690)

for nodeIndex := 1; nodeIndex < len(chainlinkNodes); nodeIndex++ {
nodeTransmitterAddress, err := chainlinkNodes[nodeIndex].EthAddresses()
Expand Down
2 changes: 1 addition & 1 deletion integration-tests/actions/ocr2_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -268,7 +268,7 @@ func CreateOCRv2Jobs(
if err != nil {
return err
}
p2pV2Bootstrapper := fmt.Sprintf("%s@%s:%d", bootstrapP2PIds.Data[0].Attributes.PeerID, bootstrapNode.RemoteIP(), 6690)
p2pV2Bootstrapper := fmt.Sprintf("%s@%s:%d", bootstrapP2PIds.Data[0].Attributes.PeerID, bootstrapNode.InternalIP(), 6690)
// Set the value for the jobs to report on
err = mockserver.SetValuePath(mockServerPath, mockServerValue)
if err != nil {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ func createBootstrapJob(t *testing.T, bootstrapNode *client.Chainlink, dkgAddres
}
_, err = bootstrapNode.MustCreateJob(bootstrapSpec)
require.NoError(t, err, "Shouldn't fail creating bootstrap job on bootstrap node")
return fmt.Sprintf("%s@%s:%d", bootstrapP2PId, bootstrapNode.RemoteIP(), 6690)
return fmt.Sprintf("%s@%s:%d", bootstrapP2PId, bootstrapNode.InternalIP(), 6690)
}

func BuildOCR2DKGConfigVars(
Expand Down
1 change: 1 addition & 0 deletions integration-tests/actions/ocr_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (

"github.com/smartcontractkit/chainlink-testing-framework/blockchain"
ctfClient "github.com/smartcontractkit/chainlink-testing-framework/client"

"github.com/smartcontractkit/chainlink/integration-tests/client"
"github.com/smartcontractkit/chainlink/integration-tests/contracts"
)
Expand Down
5 changes: 2 additions & 3 deletions integration-tests/actions/operator_forwarder_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,14 @@ import (
"github.com/ethereum/go-ethereum/accounts/abi"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/smartcontractkit/chainlink-testing-framework/utils"
"github.com/stretchr/testify/require"

"github.com/smartcontractkit/chainlink-testing-framework/blockchain"
"github.com/smartcontractkit/chainlink-testing-framework/utils"
"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/operator_factory"

"github.com/smartcontractkit/chainlink/integration-tests/client"
"github.com/smartcontractkit/chainlink/integration-tests/contracts"

"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/operator_factory"
)

func DeployForwarderContracts(
Expand Down
28 changes: 7 additions & 21 deletions integration-tests/benchmark/keeper_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -561,11 +561,6 @@ func SetupAutomationBenchmarkEnv(t *testing.T) (*environment.Environment, blockc
return testEnvironment, activeEVMNetwork, registryToTest
}

// For if we end up using env vars
keeperBenchmarkEnvVars["ETH_URL"] = activeEVMNetwork.URLs[0]
keeperBenchmarkEnvVars["ETH_HTTP_URL"] = activeEVMNetwork.HTTPURLs[0]
keeperBenchmarkEnvVars["ETH_CHAIN_ID"] = fmt.Sprint(activeEVMNetwork.ChainID)

// separate RPC urls per CL node
internalWsURLs := make([]string, 0)
internalHttpURLs := make([]string, 0)
Expand All @@ -592,22 +587,13 @@ func SetupAutomationBenchmarkEnv(t *testing.T) (*environment.Environment, blockc
l.Debug().Strs("internalWsURLs", internalWsURLs).Strs("internalHttpURLs", internalHttpURLs).Msg("internalURLs")

for i := 0; i < NumberOfNodes; i++ {
useEnvVars := strings.ToLower(os.Getenv("TEST_USE_ENV_VAR_CONFIG"))
if useEnvVars == "true" {
testEnvironment.AddHelm(chainlink.NewVersioned(i, "0.0.11", map[string]any{
"env": keeperBenchmarkEnvVars,
"chainlink": chainlinkResources,
"db": dbResources,
}))
} else {
activeEVMNetwork.HTTPURLs = []string{internalHttpURLs[i]}
activeEVMNetwork.URLs = []string{internalWsURLs[i]}
testEnvironment.AddHelm(chainlink.New(i, map[string]any{
"toml": client.AddNetworkDetailedConfig(keeperBenchmarkBaseTOML, networkDetailTOML, activeEVMNetwork),
"chainlink": chainlinkResources,
"db": dbResources,
}))
}
activeEVMNetwork.HTTPURLs = []string{internalHttpURLs[i]}
activeEVMNetwork.URLs = []string{internalWsURLs[i]}
testEnvironment.AddHelm(chainlink.New(i, map[string]any{
"toml": client.AddNetworkDetailedConfig(keeperBenchmarkBaseTOML, networkDetailTOML, activeEVMNetwork),
"chainlink": chainlinkResources,
"db": dbResources,
}))
}
err = testEnvironment.Run()
require.NoError(t, err, "Error launching test environment")
Expand Down
87 changes: 75 additions & 12 deletions integration-tests/client/chainlink.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"fmt"
"math/big"
"net/http"
"os"
"regexp"
"strings"
"sync"
Expand All @@ -16,7 +17,6 @@ import (
"golang.org/x/sync/errgroup"

"github.com/smartcontractkit/chainlink-env/environment"
chainlinkChart "github.com/smartcontractkit/chainlink-env/pkg/helm/chainlink"
)

const (
Expand Down Expand Up @@ -991,9 +991,9 @@ func (c *Chainlink) CreateStarkNetNode(node *StarkNetNodeAttributes) (*StarkNetN
return &response, resp.RawResponse, err
}

// RemoteIP retrieves the inter-cluster IP of the Chainlink node, for use with inter-node communications
func (c *Chainlink) RemoteIP() string {
return c.Config.RemoteIP
// InternalIP retrieves the inter-cluster IP of the Chainlink node, for use with inter-node communications
func (c *Chainlink) InternalIP() string {
return c.Config.InternalIP
}

// Profile starts a profile session on the Chainlink node for a pre-determined length, then runs the provided function
Expand Down Expand Up @@ -1066,24 +1066,58 @@ func (c *Chainlink) SetPageSize(size int) {
// ConnectChainlinkNodes creates new Chainlink clients
func ConnectChainlinkNodes(e *environment.Environment) ([]*Chainlink, error) {
var clients []*Chainlink
localURLs := e.URLs[chainlinkChart.NodesLocalURLsKey]
internalURLs := e.URLs[chainlinkChart.NodesInternalURLsKey]
for i, localURL := range localURLs {
internalHost := parseHostname(internalURLs[i])
for _, nodeDetails := range e.ChainlinkNodeDetails {
c, err := NewChainlink(&ChainlinkConfig{
URL: localURL,
Email: "[email protected]",
Password: "fj293fbBnlQ!f9vNs",
RemoteIP: internalHost,
URL: nodeDetails.LocalIP,
Email: "[email protected]",
Password: "fj293fbBnlQ!f9vNs",
InternalIP: parseHostname(nodeDetails.InternalIP),
ChartName: nodeDetails.ChartName,
PodName: nodeDetails.PodName,
})
if err != nil {
return nil, err
}
log.Debug().
Str("URL", c.Config.URL).
Str("Internal IP", c.Config.InternalIP).
Str("Chart Name", c.Config.ChartName).
Str("Pod Name", c.Config.PodName).
Msg("Connected to Chainlink node")
clients = append(clients, c)
}
return clients, nil
}

// ReconnectChainlinkNodes reconnects to Chainlink nodes after they have been modified, say through a Helm upgrade
// Note: Experimental as of now, will likely not work predictably.
func ReconnectChainlinkNodes(testEnvironment *environment.Environment, nodes []*Chainlink) (err error) {
for _, node := range nodes {
for _, details := range testEnvironment.ChainlinkNodeDetails {
if details.ChartName == node.Config.ChartName { // Make the link from client to pod consistent
node, err = NewChainlink(&ChainlinkConfig{
URL: details.LocalIP,
Email: "[email protected]",
Password: "fj293fbBnlQ!f9vNs",
InternalIP: parseHostname(details.InternalIP),
ChartName: details.ChartName,
PodName: details.PodName,
})
if err != nil {
return err
}
log.Debug().
Str("URL", node.Config.URL).
Str("Internal IP", node.Config.InternalIP).
Str("Chart Name", node.Config.ChartName).
Str("Pod Name", node.Config.PodName).
Msg("Reconnected to Chainlink node")
}
}
}
return nil
}

func parseHostname(s string) string {
r := regexp.MustCompile(`://(?P<Host>.*):`)
return r.FindStringSubmatch(s)[1]
Expand Down Expand Up @@ -1208,3 +1242,32 @@ func (c *Chainlink) GetForwarders() (*Forwarders, *http.Response, error) {
}
return response, resp.RawResponse, err
}

// UpgradeVersion upgrades the chainlink node to the new version
// Note: You need to call Run() on the test environment for changes to take effect
// Note: This function is not thread safe, call from a single thread
func (c *Chainlink) UpgradeVersion(testEnvironment *environment.Environment, newImage, newVersion string) error {
if newVersion == "" {
return fmt.Errorf("new version is empty")
}
if newImage == "" {
newImage = os.Getenv("CHAINLINK_IMAGE")
}
log.Info().
Str("Chart Name", c.Config.ChartName).
Str("Old Image", os.Getenv("CHAINLINK_IMAGE")).
Str("Old Version", os.Getenv("CHAINLINK_VERSION")).
Str("New Image", newImage).
Str("New Version", newVersion).
Msg("Upgrading Chainlink Node")
upgradeVals := map[string]any{
"chainlink": map[string]any{
"image": map[string]any{
"image": newImage,
"version": newVersion,
},
},
}
testEnvironment, err := testEnvironment.UpdateHelm(c.Config.ChartName, upgradeVals)
return err
}
Loading

0 comments on commit 3a31f1e

Please sign in to comment.