From 896b5d52df27e1c4372db77af88df609a08c9e5a Mon Sep 17 00:00:00 2001 From: Matthew Slipper Date: Thu, 20 Oct 2022 14:22:40 -0600 Subject: [PATCH] ops: Add bedrock-devnet script (#3734) * ops: Add bedrock-devnet script Adds a bedrock-devnet script that deploys a local network using the Hardhat deploy scripts. I opted to write this in Python rather than Bash because we're starting to add complexity to our devnet setup, and I find Python much easier and safer to maintain. Over time, I'll migrate the other devnet-up script to use this Python script as well. To invoke this script, run `make devnet-up-deploy`. It's included in CI already as a separate testing job. * review updates * Revert "review updates" This reverts commit d01ba16b971294b44c5bfbc3124354be30477daf. --- .circleci/config.yml | 70 ++++++-- .gitignore | 3 + Makefile | 6 +- bedrock-devnet/README.md | 5 + bedrock-devnet/devnet/__init__.py | 161 ++++++++++++++++++ bedrock-devnet/devnet/genesis.py | 55 ++++++ bedrock-devnet/devnet/log_setup.py | 30 ++++ bedrock-devnet/main.py | 9 + op-node/cmd/genesis/cmd.go | 5 +- .../deploy-config/devnetL1.json | 2 + packages/sdk/src/cross-chain-messenger.ts | 1 + packages/sdk/src/utils/contracts.ts | 9 +- packages/sdk/tasks/deposit-erc20.ts | 26 ++- packages/sdk/tasks/deposit-eth.ts | 26 ++- 14 files changed, 382 insertions(+), 26 deletions(-) create mode 100644 bedrock-devnet/README.md create mode 100644 bedrock-devnet/devnet/__init__.py create mode 100644 bedrock-devnet/devnet/genesis.py create mode 100644 bedrock-devnet/devnet/log_setup.py create mode 100644 bedrock-devnet/main.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 158bfa3a6885..f84453e51a41 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -508,6 +508,11 @@ jobs: machine: image: ubuntu-2004:2022.07.1 docker_layer_caching: true + parameters: + deploy: + description: Deploy contracts + default: false + type: boolean environment: DOCKER_BUILDKIT: 1 steps: @@ -539,22 +544,48 @@ jobs: command: | yarn install yarn build - - run: - name: Bring up the stack - command: | - make devnet-up - - run: - name: Deposit ERC20 through the bridge - command: timeout 5m npx hardhat deposit-erc20 --network devnetL1 - working_directory: packages/sdk - - run: - name: Deposit ETH through the bridge - command: timeout 5m npx hardhat deposit-eth --network devnetL1 - working_directory: packages/sdk - - run: - name: Check the status - command: npx hardhat check-op-node - working_directory: packages/contracts-bedrock + - when: + condition: + and: + - equal: [ true, <> ] + steps: + - run: + name: Bring up the stack + command: | + make devnet-up-deploy + - run: + name: Deposit ERC20 through the bridge + command: timeout 5m npx hardhat deposit-erc20 --network devnetL1 --l1-contracts-json-path ../../.devnet/sdk-addresses.json + working_directory: packages/sdk + - run: + name: Deposit ETH through the bridge + command: timeout 5m npx hardhat deposit-eth --network devnetL1 --l1-contracts-json-path ../../.devnet/sdk-addresses.json + working_directory: packages/sdk + - run: + name: Check the status + command: npx hardhat check-op-node + working_directory: packages/contracts-bedrock + - when: + condition: + and: + - equal: [ false, <> ] + steps: + - run: + name: Bring up the stack + command: | + make devnet-up + - run: + name: Deposit ERC20 through the bridge + command: timeout 5m npx hardhat deposit-erc20 --network devnetL1 + working_directory: packages/sdk + - run: + name: Deposit ETH through the bridge + command: timeout 5m npx hardhat deposit-eth --network devnetL1 + working_directory: packages/sdk + - run: + name: Check the status + command: npx hardhat check-op-node + working_directory: packages/contracts-bedrock integration-tests: machine: @@ -773,7 +804,12 @@ workflows: - bedrock-go-tests - fuzz-op-node - bedrock-markdown - - devnet + - devnet: + name: devnet (with deployed contracts) + deploy: true + - devnet: + name: devnet (with genesis contracts) + deploy: false - go-lint-test-build: name: batch-submitter-tests binary_name: batch-submitter diff --git a/.gitignore b/.gitignore index 6b1278b25998..8bfae506506e 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,6 @@ coverage.out # Ignore bedrock go bindings local output files op-bindings/bin op-exporter + + +__pycache__ diff --git a/Makefile b/Makefile index 076bccd48d5d..b778d041e86b 100644 --- a/Makefile +++ b/Makefile @@ -68,6 +68,10 @@ devnet-up: @bash ./ops-bedrock/devnet-up.sh .PHONY: devnet-up +devnet-up-deploy: + PYTHONPATH=./bedrock-devnet python3 ./bedrock-devnet/main.py --monorepo-dir=. +.PHONY: devnet-up-deploy + devnet-down: @(cd ./ops-bedrock && GENESIS_TIMESTAMP=$(shell date +%s) docker-compose stop) .PHONY: devnet-down @@ -114,4 +118,4 @@ tag-bedrock-go-modules: update-op-geth: ./ops/scripts/update-op-geth.py -.PHONY: update-op-geth \ No newline at end of file +.PHONY: update-op-geth diff --git a/bedrock-devnet/README.md b/bedrock-devnet/README.md new file mode 100644 index 000000000000..aaa4741fa105 --- /dev/null +++ b/bedrock-devnet/README.md @@ -0,0 +1,5 @@ +# bedrock-devnet + +This is a utility for running a local Bedrock devnet. It is designed to replace the legacy Bash-based devnet runner as part of a progressive migration away from Bash automation. + +The easiest way to invoke this script is to run `make devnet-up-deploy` from the root of this repository. Otherwise, to use this script run `python3 main.py --monorepo-path=`. You may need to set `PYTHONPATH` to this directory if you are invoking the script from somewhere other than `bedrock-devnet`. \ No newline at end of file diff --git a/bedrock-devnet/devnet/__init__.py b/bedrock-devnet/devnet/__init__.py new file mode 100644 index 000000000000..8a76a30df8dc --- /dev/null +++ b/bedrock-devnet/devnet/__init__.py @@ -0,0 +1,161 @@ +import argparse +import logging +import os +import subprocess +import json +import socket + +import time + +import shutil + +import devnet.log_setup +from devnet.genesis import GENESIS_TMPL + +parser = argparse.ArgumentParser(description='Bedrock devnet launcher') +parser.add_argument('--monorepo-dir', help='Directory of the monorepo', default=os.getcwd()) + +log = logging.getLogger() + + +def main(): + args = parser.parse_args() + + pjoin = os.path.join + monorepo_dir = os.path.abspath(args.monorepo_dir) + devnet_dir = pjoin(monorepo_dir, '.devnet') + ops_bedrock_dir = pjoin(monorepo_dir, 'ops-bedrock') + contracts_bedrock_dir = pjoin(monorepo_dir, 'packages', 'contracts-bedrock') + deployment_dir = pjoin(contracts_bedrock_dir, 'deployments', 'devnetL1') + op_node_dir = pjoin(args.monorepo_dir, 'op-node') + genesis_l1_path = pjoin(devnet_dir, 'genesis-l1.json') + genesis_l2_path = pjoin(devnet_dir, 'genesis-l2.json') + addresses_json_path = pjoin(devnet_dir, 'addresses.json') + sdk_addresses_json_path = pjoin(devnet_dir, 'sdk-addresses.json') + rollup_config_path = pjoin(devnet_dir, 'rollup.json') + os.makedirs(devnet_dir, exist_ok=True) + + if os.path.exists(genesis_l1_path): + log.info('L2 genesis already generated.') + else: + log.info('Generating L1 genesis.') + write_json(genesis_l1_path, GENESIS_TMPL) + + log.info('Starting L1.') + run_command(['docker-compose', 'up', '-d', 'l1'], cwd=ops_bedrock_dir, env={ + 'PWD': ops_bedrock_dir + }) + wait_up(8545) + + log.info('Generating network config.') + devnet_cfg_orig = pjoin(contracts_bedrock_dir, 'deploy-config', 'devnetL1.json') + devnet_cfg_backup = pjoin(devnet_dir, 'devnetL1.json.bak') + shutil.copy(devnet_cfg_orig, devnet_cfg_backup) + deploy_config = read_json(devnet_cfg_orig) + deploy_config['l1GenesisBlockTimestamp'] = GENESIS_TMPL['timestamp'] + deploy_config['l1StartingBlockTag'] = 'earliest' + write_json(devnet_cfg_orig, deploy_config) + + if os.path.exists(addresses_json_path): + log.info('Contracts already deployed.') + addresses = read_json(addresses_json_path) + else: + log.info('Deploying contracts.') + run_command(['yarn', 'hardhat', '--network', 'devnetL1', 'deploy'], env={ + 'CHAIN_ID': '900', + 'L1_RPC': 'http://localhost:8545', + 'PRIVATE_KEY_DEPLOYER': 'ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80' + }, cwd=contracts_bedrock_dir) + contracts = os.listdir(deployment_dir) + addresses = {} + for c in contracts: + if not c.endswith('.json'): + continue + data = read_json(pjoin(deployment_dir, c)) + addresses[c.replace('.json', '')] = data['address'] + sdk_addresses = {} + sdk_addresses.update({ + 'StateCommitmentChain': '0x0000000000000000000000000000000000000000', + 'CanonicalTransactionChain': '0x0000000000000000000000000000000000000000', + 'BondManager': '0x0000000000000000000000000000000000000000', + }) + sdk_addresses['AddressManager'] = addresses['AddressManager'] + sdk_addresses['L1CrossDomainMessenger'] = addresses['L1CrossDomainMessengerProxy'] + sdk_addresses['L1StandardBridge'] = addresses['L1StandardBridgeProxy'] + sdk_addresses['OptimismPortal'] = addresses['OptimismPortalProxy'] + sdk_addresses['L2OutputOracle'] = addresses['L2OutputOracleProxy'] + write_json(addresses_json_path, addresses) + write_json(sdk_addresses_json_path, sdk_addresses) + + if os.path.exists(genesis_l2_path): + log.info('L2 genesis and rollup configs already generated.') + else: + log.info('Generating L2 genesis and rollup configs.') + run_command([ + 'go', 'run', 'cmd/main.go', 'genesis', 'l2', + '--l1-rpc', 'http://localhost:8545', + '--deploy-config', devnet_cfg_orig, + '--deployment-dir', deployment_dir, + '--outfile.l2', pjoin(devnet_dir, 'genesis-l2.json'), + '--outfile.rollup', pjoin(devnet_dir, 'rollup.json') + ], cwd=op_node_dir) + + rollup_config = read_json(rollup_config_path) + + if os.path.exists(devnet_cfg_backup): + shutil.move(devnet_cfg_backup, devnet_cfg_orig) + + log.info('Bringing up L2.') + run_command(['docker-compose', 'up', '-d', 'l2'], cwd=ops_bedrock_dir, env={ + 'PWD': ops_bedrock_dir + }) + wait_up(9545) + + log.info('Bringing up everything else.') + run_command(['docker-compose', 'up', '-d', 'op-node', 'op-proposer', 'op-batcher'], cwd=ops_bedrock_dir, env={ + 'PWD': ops_bedrock_dir, + 'L2OO_ADDRESS': addresses['L2OutputOracleProxy'], + 'SEQUENCER_GENESIS_HASH': rollup_config['genesis']['l2']['hash'], + 'SEQUENCER_BATCH_INBOX_ADDRESS': rollup_config['batch_inbox_address'] + }) + + log.info('Devnet ready.') + + +def run_command(args, check=True, shell=False, cwd=None, env=None): + env = env if env else {} + return subprocess.run( + args, + check=check, + shell=shell, + env={ + **os.environ, + **env + }, + cwd=cwd + ) + + +def wait_up(port, retries=10, wait_secs=1): + for i in range(0, retries): + log.info(f'Trying 127.0.0.1:{port}') + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + s.connect(('127.0.0.1', int(port))) + s.shutdown(2) + log.info(f'Connected 127.0.0.1:{port}') + return True + except Exception: + time.sleep(wait_secs) + + raise Exception(f'Timed out waiting for port {port}.') + + +def write_json(path, data): + with open(path, 'w+') as f: + json.dump(data, f, indent=' ') + + +def read_json(path): + with open(path, 'r') as f: + return json.load(f) diff --git a/bedrock-devnet/devnet/genesis.py b/bedrock-devnet/devnet/genesis.py new file mode 100644 index 000000000000..b16bab21c06b --- /dev/null +++ b/bedrock-devnet/devnet/genesis.py @@ -0,0 +1,55 @@ +import time + +DEV_ACCOUNTS = [ + '3c44cdddb6a900fa2b585dd299e03d12fa4293bc', + '70997970c51812dc3a010c7d01b50e0d17dc79c8', + 'f39fd6e51aad88f6f4ce6ab8827279cfffb92266' +] + +GENESIS_TMPL = { + 'config': { + 'chainId': 900, + "homesteadBlock": 0, + "eip150Block": 0, + "eip150Hash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "eip155Block": 0, + "eip158Block": 0, + "byzantiumBlock": 0, + "constantinopleBlock": 0, + "petersburgBlock": 0, + "istanbulBlock": 0, + "muirGlacierBlock": 0, + "berlinBlock": 0, + "londonBlock": 0, + "arrowGlacierBlock": 0, + "grayGlacierBlock": 0, + "shanghaiBlock": None, + "cancunBlock": None, + 'clique': { + 'period': 15, + 'epoch': 30000 + } + }, + 'nonce': '0x0', + 'timestamp': '{:#x}'.format(int(time.time())), + 'extraData': '0x0000000000000000000000000000000000000000000000000000000000000000ca062b0fd91172d89bcd4bb084ac4e21972cc4670000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + 'gasLimit': '0xE4E1C0', + 'difficulty': '0x1', + 'mixHash': '0x0000000000000000000000000000000000000000000000000000000000000000', + 'coinbase': '0x0000000000000000000000000000000000000000', + 'alloc': { + '{:x}'.format(i).ljust(40, '0'): { + 'balance': '0x1' + } for i in range(0, 255) + }, + 'number': '0x0', + 'gasUsed': '0x0', + 'parentHash': '0x0000000000000000000000000000000000000000000000000000000000000000', + 'baseFeePergas': '0x3B9ACA00' +} + +GENESIS_TMPL['alloc'].update({ + d: { + 'balance': '0x200000000000000000000000000000000000000000000000000000000000000' + } for d in DEV_ACCOUNTS +}) diff --git a/bedrock-devnet/devnet/log_setup.py b/bedrock-devnet/devnet/log_setup.py new file mode 100644 index 000000000000..2a1083f01cef --- /dev/null +++ b/bedrock-devnet/devnet/log_setup.py @@ -0,0 +1,30 @@ +import os + +from logging.config import dictConfig + +log_level = os.getenv('LOG_LEVEL') + +log_config = { + 'version': 1, + 'loggers': { + '': { + 'handlers': ['console'], + 'level': log_level if log_level is not None else 'INFO' + }, + }, + 'handlers': { + 'console': { + 'formatter': 'stderr', + 'class': 'logging.StreamHandler', + 'stream': 'ext://sys.stdout' + } + }, + 'formatters': { + 'stderr': { + 'format': '[%(levelname)s|%(asctime)s] %(message)s', + 'datefmt': '%m-%d-%Y %I:%M:%S' + } + }, +} + +dictConfig(log_config) diff --git a/bedrock-devnet/main.py b/bedrock-devnet/main.py new file mode 100644 index 000000000000..13c2d209c8bb --- /dev/null +++ b/bedrock-devnet/main.py @@ -0,0 +1,9 @@ +import devnet + + +def main(): + devnet.main() + + +if __name__ == '__main__': + main() diff --git a/op-node/cmd/genesis/cmd.go b/op-node/cmd/genesis/cmd.go index aa94bb3d7190..46a20a1a670c 100644 --- a/op-node/cmd/genesis/cmd.go +++ b/op-node/cmd/genesis/cmd.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "math/big" "os" "path/filepath" @@ -125,7 +126,7 @@ var Subcommands = cli.Commands{ l1StartBlock, err = client.BlockByNumber(context.Background(), big.NewInt(config.L1StartingBlockTag.BlockNumber.Int64())) } if err != nil { - return err + return fmt.Errorf("error getting l1 start block: %w", err) } depPath, network := filepath.Split(ctx.String("deployment-dir")) @@ -157,7 +158,7 @@ var Subcommands = cli.Commands{ } l2Genesis, err := genesis.BuildL2DeveloperGenesis(config, l1StartBlock, l2Addrs) if err != nil { - return err + return fmt.Errorf("error creating l2 developer genesis: %w", err) } rollupConfig := makeRollupConfig(config, l1StartBlock, l2Genesis, portalProxy.Address) diff --git a/packages/contracts-bedrock/deploy-config/devnetL1.json b/packages/contracts-bedrock/deploy-config/devnetL1.json index 1f7d2646c5e8..051f626591ab 100644 --- a/packages/contracts-bedrock/deploy-config/devnetL1.json +++ b/packages/contracts-bedrock/deploy-config/devnetL1.json @@ -23,6 +23,8 @@ "optimismBaseFeeRecipient": "0xBcd4042DE499D14e55001CcbB24a551F3b954096", "optimismL1FeeRecipient": "0x71bE63f3384f5fb98995898A86B02Fb2426c5788", + "finalizationPeriodSeconds": 2, + "deploymentWaitConfirmations": 1, "fundDevAccounts": true } diff --git a/packages/sdk/src/cross-chain-messenger.ts b/packages/sdk/src/cross-chain-messenger.ts index 39eaec0aee96..4a8ee1538ef7 100644 --- a/packages/sdk/src/cross-chain-messenger.ts +++ b/packages/sdk/src/cross-chain-messenger.ts @@ -176,6 +176,7 @@ export class CrossChainMessenger { this.bridges = getBridgeAdapters(this.l2ChainId, this, { overrides: opts.bridges, + contracts: opts.contracts, }) } diff --git a/packages/sdk/src/utils/contracts.ts b/packages/sdk/src/utils/contracts.ts index 1095d304f5ae..83882736b66c 100644 --- a/packages/sdk/src/utils/contracts.ts +++ b/packages/sdk/src/utils/contracts.ts @@ -161,6 +161,7 @@ export const getBridgeAdapters = ( messenger: CrossChainMessenger, opts?: { overrides?: BridgeAdapterData + contracts?: DeepPartial } ): BridgeAdapters => { const adapterData: BridgeAdapterData = { @@ -168,12 +169,16 @@ export const getBridgeAdapters = ( ? { Standard: { Adapter: StandardBridgeAdapter, - l1Bridge: CONTRACT_ADDRESSES[l2ChainId].l1.L1StandardBridge, + l1Bridge: + opts.contracts?.l1?.L1StandardBridge || + CONTRACT_ADDRESSES[l2ChainId].l1.L1StandardBridge, l2Bridge: predeploys.L2StandardBridge, }, ETH: { Adapter: ETHBridgeAdapter, - l1Bridge: CONTRACT_ADDRESSES[l2ChainId].l1.L1StandardBridge, + l1Bridge: + opts.contracts?.l1?.L1StandardBridge || + CONTRACT_ADDRESSES[l2ChainId].l1.L1StandardBridge, l2Bridge: predeploys.L2StandardBridge, }, } diff --git a/packages/sdk/tasks/deposit-erc20.ts b/packages/sdk/tasks/deposit-erc20.ts index 6062198ca111..5edec25fc153 100644 --- a/packages/sdk/tasks/deposit-erc20.ts +++ b/packages/sdk/tasks/deposit-erc20.ts @@ -1,3 +1,5 @@ +import { promises as fs } from 'fs' + import { task, types } from 'hardhat/config' import { HardhatRuntimeEnvironment } from 'hardhat/types' import '@nomiclabs/hardhat-ethers' @@ -8,7 +10,13 @@ import { } from '@eth-optimism/contracts-bedrock' import { Event, Contract, Wallet, providers, utils } from 'ethers' -import { CrossChainMessenger, MessageStatus, CONTRACT_ADDRESSES } from '../src' +import { + CrossChainMessenger, + MessageStatus, + CONTRACT_ADDRESSES, + OEContractsLike, + DEFAULT_L2_CONTRACT_ADDRESSES, +} from '../src' const deployWETH9 = async ( hre: HardhatRuntimeEnvironment, @@ -102,6 +110,12 @@ task('deposit-erc20', 'Deposits WETH9 onto L2.') 'http://localhost:7545', types.string ) + .addOptionalParam( + 'l1ContractsJsonPath', + 'Path to a JSON with L1 contract addresses in it', + '', + types.string + ) .setAction(async (args, hre) => { const signers = await hre.ethers.getSigners() if (signers.length === 0) { @@ -127,7 +141,14 @@ task('deposit-erc20', 'Deposits WETH9 onto L2.') ) const l2ChainId = await l2Signer.getChainId() - const contractAddrs = CONTRACT_ADDRESSES[l2ChainId] + let contractAddrs = CONTRACT_ADDRESSES[l2ChainId] + if (args.l1ContractsJsonPath) { + const data = await fs.readFile(args.l1ContractsJsonPath) + contractAddrs = { + l1: JSON.parse(data.toString()), + l2: DEFAULT_L2_CONTRACT_ADDRESSES, + } as OEContractsLike + } const Artifact__L2ToL1MessagePasser = await getContractDefinition( 'L2ToL1MessagePasser' @@ -192,6 +213,7 @@ task('deposit-erc20', 'Deposits WETH9 onto L2.') l1ChainId: await signer.getChainId(), l2ChainId, bedrock: true, + contracts: contractAddrs, }) console.log('Deploying WETH9 to L1') diff --git a/packages/sdk/tasks/deposit-eth.ts b/packages/sdk/tasks/deposit-eth.ts index b2855a6ececd..8c4bdf35f265 100644 --- a/packages/sdk/tasks/deposit-eth.ts +++ b/packages/sdk/tasks/deposit-eth.ts @@ -1,3 +1,5 @@ +import { promises as fs } from 'fs' + import { task, types } from 'hardhat/config' import '@nomiclabs/hardhat-ethers' import 'hardhat-deploy' @@ -7,7 +9,13 @@ import { } from '@eth-optimism/contracts-bedrock' import { providers, utils } from 'ethers' -import { CrossChainMessenger, MessageStatus, CONTRACT_ADDRESSES } from '../src' +import { + CrossChainMessenger, + MessageStatus, + CONTRACT_ADDRESSES, + OEContractsLike, + DEFAULT_L2_CONTRACT_ADDRESSES, +} from '../src' task('deposit-eth', 'Deposits WETH9 onto L2.') .addParam( @@ -35,6 +43,12 @@ task('deposit-eth', 'Deposits WETH9 onto L2.') true, types.boolean ) + .addOptionalParam( + 'l1ContractsJsonPath', + 'Path to a JSON with L1 contract addresses in it', + '', + types.string + ) .addOptionalParam('withdrawAmount', 'Amount to withdraw', '', types.string) .setAction(async (args, hre) => { const signers = await hre.ethers.getSigners() @@ -70,7 +84,14 @@ task('deposit-eth', 'Deposits WETH9 onto L2.') ) const l2ChainId = await l2Signer.getChainId() - const contractAddrs = CONTRACT_ADDRESSES[l2ChainId] + let contractAddrs = CONTRACT_ADDRESSES[l2ChainId] + if (args.l1ContractsJsonPath) { + const data = await fs.readFile(args.l1ContractsJsonPath) + contractAddrs = { + l1: JSON.parse(data.toString()), + l2: DEFAULT_L2_CONTRACT_ADDRESSES, + } as OEContractsLike + } const Artifact__L2ToL1MessagePasser = await getContractDefinition( 'L2ToL1MessagePasser' @@ -135,6 +156,7 @@ task('deposit-eth', 'Deposits WETH9 onto L2.') l1ChainId: await signer.getChainId(), l2ChainId, bedrock: true, + contracts: contractAddrs, }) const opBalanceBefore = await signer.provider.getBalance(