Skip to content

Commit

Permalink
EIP-4844: Make the spec executable
Browse files Browse the repository at this point in the history
- Implement all the required glue code to make things executable
- Implement a dummy KZG trusted setup

Co-authored-by: Hsiao-Wei Wang <[email protected]>
  • Loading branch information
asn-d6 and hwwhww committed Jul 13, 2022
1 parent 5b9bf41 commit 567a25f
Show file tree
Hide file tree
Showing 19 changed files with 327 additions and 36 deletions.
17 changes: 16 additions & 1 deletion .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,19 @@ jobs:
command: make citest fork=capella
- store_test_results:
path: tests/core/pyspec/test-reports

test-eip4844:
docker:
- image: circleci/python:3.8
working_directory: ~/specs-repo
steps:
- restore_cache:
key: v3-specs-repo-{{ .Branch }}-{{ .Revision }}
- restore_pyspec_cached_venv
- run:
name: Run py-tests
command: make citest fork=eip4844
- store_test_results:
path: tests/core/pyspec/test-reports
table_of_contents:
docker:
- image: circleci/node:10.16.3
Expand Down Expand Up @@ -260,6 +272,9 @@ workflows:
- test-capella:
requires:
- install_pyspec_test
- test-eip4844:
requires:
- install_pyspec_test
- table_of_contents
- codespell
- lint:
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ tests/core/pyspec/eth2spec/phase0/
tests/core/pyspec/eth2spec/altair/
tests/core/pyspec/eth2spec/bellatrix/
tests/core/pyspec/eth2spec/capella/
tests/core/pyspec/eth2spec/eip4844/

# coverage reports
.htmlcov
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ codespell:
lint: pyspec
. venv/bin/activate; cd $(PY_SPEC_DIR); \
flake8 --config $(LINTER_CONFIG_FILE) ./eth2spec \
&& pylint --disable=all --enable unused-argument ./eth2spec/phase0 ./eth2spec/altair ./eth2spec/bellatrix \
&& pylint --disable=all --enable unused-argument ./eth2spec/phase0 ./eth2spec/altair ./eth2spec/bellatrix ./eth2spec/capella \
&& mypy --config-file $(LINTER_CONFIG_FILE) -p eth2spec.phase0 -p eth2spec.altair -p eth2spec.bellatrix -p eth2spec.capella

lint_generators: pyspec
Expand Down
6 changes: 3 additions & 3 deletions configs/mainnet.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,9 @@ BELLATRIX_FORK_EPOCH: 18446744073709551615
# Capella
CAPELLA_FORK_VERSION: 0x03000000
CAPELLA_FORK_EPOCH: 18446744073709551615
# Sharding
SHARDING_FORK_VERSION: 0x04000000
SHARDING_FORK_EPOCH: 18446744073709551615
# EIP4844
EIP4844_FORK_VERSION: 0x04000000
EIP4844_FORK_EPOCH: 18446744073709551615



Expand Down
6 changes: 3 additions & 3 deletions configs/minimal.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,9 @@ BELLATRIX_FORK_EPOCH: 18446744073709551615
# Capella
CAPELLA_FORK_VERSION: 0x03000001
CAPELLA_FORK_EPOCH: 18446744073709551615
# Sharding
SHARDING_FORK_VERSION: 0x04000001
SHARDING_FORK_EPOCH: 18446744073709551615
# EIP4844
EIP4844_FORK_VERSION: 0x04000001
EIP4844_FORK_EPOCH: 18446744073709551615


# Time parameters
Expand Down
8 changes: 8 additions & 0 deletions presets/mainnet/eip4844.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# Mainnet preset - Phase0

# Misc
# ---------------------------------------------------------------
# `uint64(4096)`
FIELD_ELEMENTS_PER_BLOB: 4096
# `uint64(2**4)` (= 16)
MAX_BLOBS_PER_BLOCK: 16
8 changes: 8 additions & 0 deletions presets/minimal/eip4844.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# Minimal preset - Phase0

# Misc
# ---------------------------------------------------------------
# [customized]
FIELD_ELEMENTS_PER_BLOB: 4
# `uint64(2**4)` (= 16)
MAX_BLOBS_PER_BLOCK: 16
79 changes: 67 additions & 12 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ def installPackage(package: str):
ALTAIR = 'altair'
BELLATRIX = 'bellatrix'
CAPELLA = 'capella'
EIP4844 = 'eip4844'


# The helper functions that are used when defining constants
Expand Down Expand Up @@ -208,6 +209,9 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str]) ->
elif source.startswith("class"):
class_name, parent_class = _get_class_info_from_source(source)
# check consistency with spec
if class_name != current_name:
print('class_name', class_name, 'current_name', current_name)

assert class_name == current_name
if parent_class:
assert parent_class == "Container"
Expand All @@ -230,7 +234,7 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str]) ->

if not _is_constant_id(name):
# Check for short type declarations
if value.startswith(("uint", "Bytes", "ByteList", "Union")):
if value.startswith(("uint", "Bytes", "ByteList", "Union", "Vector", "List")):
custom_types[name] = value
continue

Expand Down Expand Up @@ -304,7 +308,7 @@ def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]:

@classmethod
@abstractmethod
def hardcoded_custom_type_dep_constants(cls) -> Dict[str, str]: # TODO
def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]: # TODO
"""
The constants that are required for custom types.
"""
Expand Down Expand Up @@ -432,7 +436,7 @@ def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]:
return {}

@classmethod
def hardcoded_custom_type_dep_constants(cls) -> Dict[str, str]:
def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]:
return {}

@classmethod
Expand Down Expand Up @@ -547,11 +551,11 @@ def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> ExecutionPayloa


@classmethod
def hardcoded_custom_type_dep_constants(cls) -> str:
def hardcoded_custom_type_dep_constants(cls, spec_object) -> str:
constants = {
'MAX_BYTES_PER_TRANSACTION': 'uint64(2**30)',
'MAX_BYTES_PER_TRANSACTION': spec_object.preset_vars['MAX_BYTES_PER_TRANSACTION'].value,
}
return {**super().hardcoded_custom_type_dep_constants(), **constants}
return {**super().hardcoded_custom_type_dep_constants(spec_object), **constants}


#
Expand All @@ -567,14 +571,57 @@ def imports(cls, preset_name: str):
'''


#
# EIP4844SpecBuilder
#
class EIP4844SpecBuilder(BellatrixSpecBuilder):
fork: str = EIP4844

@classmethod
def imports(cls, preset_name: str):
return super().imports(preset_name) + f'''
from eth2spec.utils import kzg
from eth2spec.bellatrix import {preset_name} as bellatrix
'''

@classmethod
def sundry_functions(cls) -> str:
return super().sundry_functions() + '''
# TODO: for mainnet, load pre-generated trusted setup file to reduce building time.
# TESTING_FIELD_ELEMENTS_PER_BLOB is hardcoded copy from minimal presets
TESTING_FIELD_ELEMENTS_PER_BLOB = 4
TESTING_SECRET = 1337
TESTING_KZG_SETUP_G1 = kzg.generate_setup(bls.G1, TESTING_SECRET, TESTING_FIELD_ELEMENTS_PER_BLOB)
TESTING_KZG_SETUP_G2 = kzg.generate_setup(bls.G2, TESTING_SECRET, TESTING_FIELD_ELEMENTS_PER_BLOB)
TESTING_KZG_SETUP_LAGRANGE = kzg.get_lagrange(TESTING_KZG_SETUP_G1)
KZG_SETUP_G1 = [bls.G1_to_bytes48(p) for p in TESTING_KZG_SETUP_G1]
KZG_SETUP_G2 = [bls.G2_to_bytes96(p) for p in TESTING_KZG_SETUP_G2]
KZG_SETUP_LAGRANGE = TESTING_KZG_SETUP_LAGRANGE
ROOTS_OF_UNITY = kzg.compute_roots_of_unity(TESTING_FIELD_ELEMENTS_PER_BLOB)
def retrieve_blobs_sidecar(slot: Slot, beacon_block_root: Root) -> BlobsSidecar:
pass'''

@classmethod
def hardcoded_custom_type_dep_constants(cls, spec_object) -> str:
constants = {
'FIELD_ELEMENTS_PER_BLOB': spec_object.preset_vars['FIELD_ELEMENTS_PER_BLOB'].value,
'MAX_BLOBS_PER_BLOCK': spec_object.preset_vars['MAX_BLOBS_PER_BLOCK'].value,
}
return {**super().hardcoded_custom_type_dep_constants(spec_object), **constants}



spec_builders = {
builder.fork: builder
for builder in (Phase0SpecBuilder, AltairSpecBuilder, BellatrixSpecBuilder, CapellaSpecBuilder)
for builder in (Phase0SpecBuilder, AltairSpecBuilder, BellatrixSpecBuilder, CapellaSpecBuilder, EIP4844SpecBuilder)
}


def is_spec_defined_type(value: str) -> bool:
return value.startswith('ByteList') or value.startswith('Union')
return value.startswith(('ByteList', 'Union', 'Vector', 'List'))


def objects_to_spec(preset_name: str,
Expand Down Expand Up @@ -652,7 +699,7 @@ def format_constant(name: str, vardef: VariableDefinition) -> str:
ordered_class_objects_spec = '\n\n\n'.join(ordered_class_objects.values())
ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, builder.hardcoded_ssz_dep_constants()[x]), builder.hardcoded_ssz_dep_constants()))
ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), builder.hardcoded_ssz_dep_constants()))
custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, builder.hardcoded_custom_type_dep_constants()[x]), builder.hardcoded_custom_type_dep_constants()))
custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, builder.hardcoded_custom_type_dep_constants(spec_object)[x]), builder.hardcoded_custom_type_dep_constants(spec_object)))
spec = (
builder.imports(preset_name)
+ builder.preparations()
Expand Down Expand Up @@ -869,14 +916,14 @@ def finalize_options(self):
if len(self.md_doc_paths) == 0:
print("no paths were specified, using default markdown file paths for pyspec"
" build (spec fork: %s)" % self.spec_fork)
if self.spec_fork in (PHASE0, ALTAIR, BELLATRIX, CAPELLA):
if self.spec_fork in (PHASE0, ALTAIR, BELLATRIX, CAPELLA, EIP4844):
self.md_doc_paths = """
specs/phase0/beacon-chain.md
specs/phase0/fork-choice.md
specs/phase0/validator.md
specs/phase0/weak-subjectivity.md
"""
if self.spec_fork in (ALTAIR, BELLATRIX, CAPELLA):
if self.spec_fork in (ALTAIR, BELLATRIX, CAPELLA, EIP4844):
self.md_doc_paths += """
specs/altair/beacon-chain.md
specs/altair/bls.md
Expand All @@ -885,7 +932,7 @@ def finalize_options(self):
specs/altair/p2p-interface.md
specs/altair/sync-protocol.md
"""
if self.spec_fork in (BELLATRIX, CAPELLA):
if self.spec_fork in (BELLATRIX, CAPELLA, EIP4844):
self.md_doc_paths += """
specs/bellatrix/beacon-chain.md
specs/bellatrix/fork.md
Expand All @@ -902,6 +949,14 @@ def finalize_options(self):
specs/capella/validator.md
specs/capella/p2p-interface.md
"""
if self.spec_fork == EIP4844:
self.md_doc_paths += """
specs/eip4844/beacon-chain.md
specs/eip4844/fork.md
specs/eip4844/polynomial-commitments.md
specs/eip4844/p2p-interface.md
specs/eip4844/validator.md
"""
if len(self.md_doc_paths) == 0:
raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork)

Expand Down
2 changes: 1 addition & 1 deletion tests/core/pyspec/eth2spec/test/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ def pytest_addoption(parser):

def _validate_fork_name(forks):
for fork in forks:
if fork not in ALL_PHASES:
if fork not in set(ALL_PHASES):
raise ValueError(
f'The given --fork argument "{fork}" is not an available fork.'
f' The available forks: {ALL_PHASES}'
Expand Down
12 changes: 8 additions & 4 deletions tests/core/pyspec/eth2spec/test/context.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,14 @@
from eth2spec.altair import mainnet as spec_altair_mainnet, minimal as spec_altair_minimal
from eth2spec.bellatrix import mainnet as spec_bellatrix_mainnet, minimal as spec_bellatrix_minimal
from eth2spec.capella import mainnet as spec_capella_mainnet, minimal as spec_capella_minimal
from eth2spec.eip4844 import mainnet as spec_eip4844_mainnet, minimal as spec_eip4844_minimal
from eth2spec.utils import bls

from .exceptions import SkippedTest
from .helpers.constants import (
PHASE0, ALTAIR, BELLATRIX, CAPELLA,
PHASE0, ALTAIR, BELLATRIX, CAPELLA, EIP4844,
MINIMAL, MAINNET,
ALL_PHASES, FORKS_BEFORE_ALTAIR, FORKS_BEFORE_BELLATRIX, FORKS_BEFORE_CAPELLA,
ALL_PHASES, FORKS_BEFORE_ALTAIR, FORKS_BEFORE_BELLATRIX,
ALL_FORK_UPGRADES,
)
from .helpers.typing import SpecForkName, PresetBaseName
Expand Down Expand Up @@ -76,12 +77,14 @@ class ForkMeta:
ALTAIR: spec_altair_minimal,
BELLATRIX: spec_bellatrix_minimal,
CAPELLA: spec_capella_minimal,
EIP4844: spec_eip4844_minimal,
},
MAINNET: {
PHASE0: spec_phase0_mainnet,
ALTAIR: spec_altair_mainnet,
BELLATRIX: spec_bellatrix_mainnet,
CAPELLA: spec_capella_mainnet,
EIP4844: spec_eip4844_mainnet
},
}

Expand Down Expand Up @@ -576,12 +579,13 @@ def is_post_bellatrix(spec):


def is_post_capella(spec):
return spec.fork not in FORKS_BEFORE_CAPELLA
return spec.fork == CAPELLA


with_altair_and_later = with_all_phases_except([PHASE0])
with_bellatrix_and_later = with_all_phases_except([PHASE0, ALTAIR])
with_capella_and_later = with_all_phases_except([PHASE0, ALTAIR, BELLATRIX])
with_capella_and_later = with_all_phases_except([PHASE0, ALTAIR, BELLATRIX, EIP4844])
with_eip4844_and_later = with_all_phases_except([PHASE0, ALTAIR, BELLATRIX, CAPELLA])


def only_generator(reason):
Expand Down
12 changes: 9 additions & 3 deletions tests/core/pyspec/eth2spec/test/helpers/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,15 @@
SHARDING = SpecForkName('sharding')
CUSTODY_GAME = SpecForkName('custody_game')
DAS = SpecForkName('das')

# The forks that pytest runs with.
ALL_PHASES = (PHASE0, ALTAIR, BELLATRIX, CAPELLA)
EIP4844 = SpecForkName('eip4844')

# The forks that pytest can run with.
ALL_PHASES = (
# Formal forks
PHASE0, ALTAIR, BELLATRIX, CAPELLA,
# Experimental patches
EIP4844,
)
# The forks that output to the test vectors.
TESTGEN_FORKS = (PHASE0, ALTAIR, BELLATRIX)

Expand Down
6 changes: 3 additions & 3 deletions tests/core/pyspec/eth2spec/test/helpers/execution_payload.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from eth2spec.test.helpers.constants import FORKS_BEFORE_CAPELLA
from eth2spec.test.context import is_post_capella


def build_empty_execution_payload(spec, state, randao_mix=None):
Expand Down Expand Up @@ -28,7 +28,7 @@ def build_empty_execution_payload(spec, state, randao_mix=None):
block_hash=spec.Hash32(),
transactions=empty_txs,
)
if spec.fork not in FORKS_BEFORE_CAPELLA:
if is_post_capella(spec):
num_withdrawals = min(spec.MAX_WITHDRAWALS_PER_PAYLOAD, len(state.withdrawal_queue))
payload.withdrawals = state.withdrawal_queue[:num_withdrawals]

Expand All @@ -55,7 +55,7 @@ def get_execution_payload_header(spec, execution_payload):
block_hash=execution_payload.block_hash,
transactions_root=spec.hash_tree_root(execution_payload.transactions)
)
if spec.fork not in FORKS_BEFORE_CAPELLA:
if is_post_capella(spec):
payload_header.withdrawals_root = spec.hash_tree_root(execution_payload.withdrawals)
return payload_header

Expand Down
3 changes: 3 additions & 0 deletions tests/core/pyspec/eth2spec/test/helpers/fork_transition.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
ALTAIR,
BELLATRIX,
CAPELLA,
EIP4844,
)
from eth2spec.test.helpers.deposits import (
prepare_state_and_deposit,
Expand Down Expand Up @@ -150,6 +151,8 @@ def do_fork(state, spec, post_spec, fork_epoch, with_block=True, operation_dict=
state = post_spec.upgrade_to_bellatrix(state)
elif post_spec.fork == CAPELLA:
state = post_spec.upgrade_to_capella(state)
elif post_spec.fork == EIP4844:
state = post_spec.upgrade_to_eip4844(state)

assert state.fork.epoch == fork_epoch

Expand Down
6 changes: 3 additions & 3 deletions tests/core/pyspec/eth2spec/test/helpers/genesis.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from eth2spec.test.helpers.constants import (
ALTAIR, BELLATRIX,
FORKS_BEFORE_ALTAIR, FORKS_BEFORE_BELLATRIX, FORKS_BEFORE_CAPELLA,
ALTAIR, BELLATRIX, CAPELLA,
FORKS_BEFORE_ALTAIR, FORKS_BEFORE_BELLATRIX,
)
from eth2spec.test.helpers.keys import pubkeys

Expand All @@ -20,7 +20,7 @@ def build_mock_validator(spec, i: int, balance: int):
effective_balance=min(balance - balance % spec.EFFECTIVE_BALANCE_INCREMENT, spec.MAX_EFFECTIVE_BALANCE)
)

if spec.fork not in FORKS_BEFORE_CAPELLA:
if spec.fork in (CAPELLA):
validator.fully_withdrawn_epoch = spec.FAR_FUTURE_EPOCH

return validator
Expand Down
Loading

0 comments on commit 567a25f

Please sign in to comment.