From 675d404c380c48629e0705b06a5ea1f53e884f30 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 25 Jan 2020 01:26:10 +0100 Subject: [PATCH] Package eth2spec for tooling and experimentation See tests/core/pyspec/README.md for usage description. This commit: - refactors config loading to be part of the pyspec package - updates requirements and main files to use new config loading - cleans up the build script - converts the build script to a distutil command - runs pyspec build as part of build package command - provides pyspecdev command to get editable spec python files --- .circleci/config.yml | 22 +- .gitignore | 6 +- Makefile | 63 +-- deposit_contract/tester/requirements.txt | 4 +- scripts/README.md | 32 -- scripts/__init__.py | 0 scripts/build_spec.py | 309 ------------ scripts/function_puller.py | 72 --- setup.py | 465 ++++++++++++++++++ tests/core/config_helpers/README.md | 19 - .../config_helpers/preset_loader/__init__.py | 0 .../config_helpers/preset_loader/loader.py | 27 - tests/core/config_helpers/requirements.txt | 1 - tests/core/config_helpers/setup.py | 9 - tests/core/pyspec/README.md | 33 +- tests/core/pyspec/eth2spec/config/README.md | 20 + .../pyspec/eth2spec/config/apply_config.py | 22 - .../pyspec/eth2spec/config/config_util.py | 44 ++ tests/core/pyspec/eth2spec/phase0/__init__.py | 0 tests/core/pyspec/eth2spec/phase1/__init__.py | 0 tests/core/pyspec/eth2spec/test/conftest.py | 4 +- tests/core/pyspec/requirements-testing.txt | 7 - tests/core/pyspec/requirements.txt | 6 - tests/core/pyspec/setup.py | 16 - tests/generators/epoch_processing/main.py | 9 +- .../epoch_processing/requirements.txt | 3 +- tests/generators/genesis/main.py | 7 +- tests/generators/genesis/requirements.txt | 3 +- tests/generators/operations/main.py | 9 +- tests/generators/operations/requirements.txt | 3 +- tests/generators/sanity/main.py | 11 +- tests/generators/sanity/requirements.txt | 3 +- tests/generators/shuffling/main.py | 8 +- tests/generators/shuffling/requirements.txt | 3 +- tests/generators/ssz_generic/requirements.txt | 3 +- tests/generators/ssz_static/main.py | 10 +- tests/generators/ssz_static/requirements.txt | 3 +- 37 files changed, 632 insertions(+), 624 deletions(-) delete mode 100644 scripts/README.md delete mode 100644 scripts/__init__.py delete mode 100644 scripts/build_spec.py delete mode 100644 scripts/function_puller.py create mode 100644 setup.py delete mode 100644 tests/core/config_helpers/README.md delete mode 100644 tests/core/config_helpers/preset_loader/__init__.py delete mode 100644 tests/core/config_helpers/preset_loader/loader.py delete mode 100644 tests/core/config_helpers/requirements.txt delete mode 100644 tests/core/config_helpers/setup.py create mode 100644 tests/core/pyspec/eth2spec/config/README.md delete mode 100644 tests/core/pyspec/eth2spec/config/apply_config.py create mode 100644 tests/core/pyspec/eth2spec/config/config_util.py delete mode 100644 tests/core/pyspec/eth2spec/phase0/__init__.py delete mode 100644 tests/core/pyspec/eth2spec/phase1/__init__.py delete mode 100644 tests/core/pyspec/requirements-testing.txt delete mode 100644 tests/core/pyspec/requirements.txt delete mode 100644 tests/core/pyspec/setup.py diff --git a/.circleci/config.yml b/.circleci/config.yml index bd77f2e5e6..6b243921ff 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,40 +35,40 @@ commands: description: "Restore the cache with pyspec keys" steps: - restore_cached_venv: - venv_name: v17-pyspec - reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "tests/core/pyspec/requirements-testing.txt" }} + venv_name: v19-pyspec + reqs_checksum: cache-{{ checksum "setup.py" }} save_pyspec_cached_venv: description: Save a venv into a cache with pyspec keys" steps: - save_cached_venv: - venv_name: v17-pyspec - reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "tests/core/pyspec/requirements-testing.txt" }} - venv_path: ./tests/core/pyspec/venv + venv_name: v19-pyspec + reqs_checksum: cache-{{ checksum "setup.py" }} + venv_path: ./venv restore_deposit_contract_compiler_cached_venv: description: "Restore the venv from cache for the deposit contract compiler" steps: - restore_cached_venv: - venv_name: v16-deposit-contract-compiler + venv_name: v18-deposit-contract-compiler reqs_checksum: cache-{{ checksum "deposit_contract/compiler/requirements.txt" }} save_deposit_contract_compiler_cached_venv: description: "Save the venv to cache for later use of the deposit contract compiler" steps: - save_cached_venv: - venv_name: v16-deposit-contract-compiler + venv_name: v18-deposit-contract-compiler reqs_checksum: cache-{{ checksum "deposit_contract/compiler/requirements.txt" }} venv_path: ./deposit_contract/compiler/venv restore_deposit_contract_tester_cached_venv: description: "Restore the venv from cache for the deposit contract tester" steps: - restore_cached_venv: - venv_name: v17-deposit-contract-tester - reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/tester/requirements.txt" }} + venv_name: v18-deposit-contract-tester + reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "deposit_contract/tester/requirements.txt" }} save_deposit_contract_tester_cached_venv: description: "Save the venv to cache for later use of the deposit contract tester" steps: - save_cached_venv: - venv_name: v17-deposit-contract-tester - reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/tester/requirements.txt" }} + venv_name: v18-deposit-contract-tester + reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "deposit_contract/tester/requirements.txt" }} venv_path: ./deposit_contract/tester/venv jobs: checkout_specs: diff --git a/.gitignore b/.gitignore index c4256032c0..bcd96f8858 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ venv build/ output/ +dist/ eth2.0-spec-tests/ @@ -14,8 +15,8 @@ eth2.0-spec-tests/ .mypy_cache # Dynamically built from Markdown spec -tests/core/pyspec/eth2spec/phase0/spec.py -tests/core/pyspec/eth2spec/phase1/spec.py +tests/core/pyspec/eth2spec/phase0/ +tests/core/pyspec/eth2spec/phase1/ # coverage reports .htmlcov @@ -24,5 +25,6 @@ tests/core/pyspec/eth2spec/phase1/spec.py # local CI testing output tests/core/pyspec/test-reports +tests/core/pyspec/eth2spec/test_results.xml *.egg-info diff --git a/Makefile b/Makefile index c779c5e8ea..efa7769977 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ SSZ_DIR = ./ssz SCRIPT_DIR = ./scripts TEST_LIBS_DIR = ./tests/core PY_SPEC_DIR = $(TEST_LIBS_DIR)/pyspec -TEST_VECTOR_DIR = ./eth2.0-spec-tests/tests +TEST_VECTOR_DIR = ../eth2.0-spec-tests/tests GENERATOR_DIR = ./tests/generators DEPOSIT_CONTRACT_COMPILER_DIR = ./deposit_contract/compiler DEPOSIT_CONTRACT_TESTER_DIR = ./deposit_contract/tester @@ -18,24 +18,12 @@ GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENER # To check generator matching: #$(info $$GENERATOR_TARGETS is [${GENERATOR_TARGETS}]) -PHASE0_SPEC_DIR = $(SPEC_DIR)/phase0 -PY_SPEC_PHASE_0_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase0/spec.py -PY_SPEC_PHASE_0_DEPS = $(wildcard $(SPEC_DIR)/phase0/*.md) - -PHASE1_SPEC_DIR = $(SPEC_DIR)/phase1 -PY_SPEC_PHASE_1_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase1/spec.py -PY_SPEC_PHASE_1_DEPS = $(wildcard $(SPEC_DIR)/phase1/*.md) - -PY_SPEC_ALL_DEPS = $(PY_SPEC_PHASE_0_DEPS) $(PY_SPEC_PHASE_1_DEPS) - -PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS) $(PY_SPEC_PHASE_1_TARGETS) - -MARKDOWN_FILES = $(PY_SPEC_ALL_DEPS) $(wildcard $(SPEC_DIR)/*.md) $(wildcard $(SSZ_DIR)/*.md) $(wildcard $(SPEC_DIR)/networking/*.md) $(wildcard $(SPEC_DIR)/validator/*.md) +MARKDOWN_FILES = $(wildcard $(SPEC_DIR)/phase0/*.md) $(wildcard $(SPEC_DIR)/phase1/*.md) $(wildcard $(SSZ_DIR)/*.md) $(wildcard $(SPEC_DIR)/networking/*.md) $(wildcard $(SPEC_DIR)/validator/*.md) COV_HTML_OUT=.htmlcov COV_INDEX_FILE=$(PY_SPEC_DIR)/$(COV_HTML_OUT)/index.html -.PHONY: clean partial_clean all test citest lint generate_tests pyspec phase0 phase1 install_test open_cov \ +.PHONY: clean partial_clean all test citest lint generate_tests pyspec install_test open_cov \ install_deposit_contract_tester test_deposit_contract install_deposit_contract_compiler \ compile_deposit_contract test_compile_deposit_contract check_toc @@ -45,33 +33,43 @@ all: $(PY_SPEC_ALL_TARGETS) partial_clean: rm -rf $(TEST_VECTOR_DIR) rm -rf $(GENERATOR_VENVS) + rm -rf .pytest_cache + rm -f .coverage rm -rf $(PY_SPEC_DIR)/.pytest_cache - rm -rf $(PY_SPEC_ALL_TARGETS) rm -rf $(DEPOSIT_CONTRACT_COMPILER_DIR)/.pytest_cache rm -rf $(DEPOSIT_CONTRACT_TESTER_DIR)/.pytest_cache + rm -rf $(PY_SPEC_DIR)/phase0 + rm -rf $(PY_SPEC_DIR)/phase1 rm -rf $(PY_SPEC_DIR)/$(COV_HTML_OUT) rm -rf $(PY_SPEC_DIR)/.coverage rm -rf $(PY_SPEC_DIR)/test-reports + rm -rf eth2spec.egg-info dist build + clean: partial_clean + rm -rf venv rm -rf $(PY_SPEC_DIR)/venv rm -rf $(DEPOSIT_CONTRACT_COMPILER_DIR)/venv rm -rf $(DEPOSIT_CONTRACT_TESTER_DIR)/venv # "make generate_tests" to run all generators -generate_tests: $(PY_SPEC_ALL_TARGETS) $(GENERATOR_TARGETS) +generate_tests: $(GENERATOR_TARGETS) + +# "make pyspec" to create the pyspec for all phases. +pyspec: + . venv/bin/activate; python3 setup.py pyspecdev # installs the packages to run pyspec tests install_test: - cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements-testing.txt; + python3 -m venv venv; . venv/bin/activate; pip3 install .[testing] .[linting] -test: $(PY_SPEC_ALL_TARGETS) - cd $(PY_SPEC_DIR); . venv/bin/activate; export PYTHONPATH="./"; \ +test: pyspec + . venv/bin/activate; cd $(PY_SPEC_DIR); \ python -m pytest -n 4 --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec -citest: $(PY_SPEC_ALL_TARGETS) - cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; export PYTHONPATH="./"; \ - python -m pytest -n 4 --junitxml=test-reports/eth2spec/test_results.xml eth2spec +citest: pyspec + mkdir -p tests/core/pyspec/test-reports/eth2spec; . venv/bin/activate; cd $(PY_SPEC_DIR); \ + python -m pytest -n 4 --junitxml=eth2spec/test_results.xml eth2spec open_cov: ((open "$(COV_INDEX_FILE)" || xdg-open "$(COV_INDEX_FILE)") &> /dev/null) & @@ -87,13 +85,13 @@ check_toc: $(MARKDOWN_FILES:=.toc) codespell: codespell . --skip ./.git -I .codespell-whitelist -lint: $(PY_SPEC_ALL_TARGETS) - cd $(PY_SPEC_DIR); . venv/bin/activate; \ +lint: pyspec + . venv/bin/activate; cd $(PY_SPEC_DIR); \ flake8 --ignore=E252,W504,W503 --max-line-length=120 ./eth2spec \ && cd ./eth2spec && mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase0 \ && mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase1; -install_deposit_contract_tester: $(PY_SPEC_ALL_TARGETS) +install_deposit_contract_tester: cd $(DEPOSIT_CONTRACT_TESTER_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements.txt test_deposit_contract: @@ -111,17 +109,6 @@ test_compile_deposit_contract: cd $(DEPOSIT_CONTRACT_COMPILER_DIR); . venv/bin/activate; \ python3.7 -m pytest . -# "make pyspec" to create the pyspec for all phases. -pyspec: $(PY_SPEC_ALL_TARGETS) - -$(PY_SPEC_PHASE_0_TARGETS): $(PY_SPEC_PHASE_0_DEPS) - python3 $(SCRIPT_DIR)/build_spec.py -p0 $(PHASE0_SPEC_DIR)/beacon-chain.md $(PHASE0_SPEC_DIR)/fork-choice.md $(PHASE0_SPEC_DIR)/validator.md $@ - -$(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS) - python3 $(SCRIPT_DIR)/build_spec.py -p1 $(PHASE0_SPEC_DIR)/beacon-chain.md $(PHASE0_SPEC_DIR)/fork-choice.md $(PHASE1_SPEC_DIR)/custody-game.md $(PHASE1_SPEC_DIR)/beacon-chain.md $(PHASE1_SPEC_DIR)/fraud-proofs.md $(PHASE1_SPEC_DIR)/fork-choice.md $(PHASE1_SPEC_DIR)/phase1-fork.md $@ - -# TODO: also build validator spec and light-client-sync - CURRENT_DIR = ${CURDIR} # Runs a generator, identified by param 1 @@ -154,5 +141,5 @@ $(TEST_VECTOR_DIR)/: # For any generator, build it using the run_generator function. # (creation of output dir is a dependency) -gen_%: $(PY_SPEC_ALL_TARGETS) $(TEST_VECTOR_DIR) +gen_%: $(TEST_VECTOR_DIR) $(call run_generator,$*) diff --git a/deposit_contract/tester/requirements.txt b/deposit_contract/tester/requirements.txt index e6acaf825e..8dadab0bc2 100644 --- a/deposit_contract/tester/requirements.txt +++ b/deposit_contract/tester/requirements.txt @@ -1,5 +1,5 @@ eth-tester[py-evm]>=0.3.0b1,<0.4 web3==5.4.0 pytest==3.6.1 -../../tests/core/pyspec -../../tests/core/config_helpers \ No newline at end of file +# The eth2spec +../../ diff --git a/scripts/README.md b/scripts/README.md deleted file mode 100644 index 9d5849053f..0000000000 --- a/scripts/README.md +++ /dev/null @@ -1,32 +0,0 @@ -# Building pyspecs from specs.md - -The benefit of the particular spec design is that the given Markdown files can be converted to a `spec.py` file for the purposes of testing and linting. As a result, bugs are discovered and patched more quickly. - -Specs can be built from either a single Markdown document or multiple files that must be combined in a given order. Given 2 spec objects, `build_spec.combine_spec_objects` will combine them into a single spec object which, subsequently, can be converted into a `specs.py`. - -## Usage - -For usage of the spec builder, run `python3 -m build_spec --help`. - -## `@Labels` and inserts - -The functioning of the spec combiner is largely automatic in that given `spec0.md` and `spec1.md`, SSZ Objects will be extended and old functions will be overwritten. Extra functionality is provided for more granular control over how files are combined. In the event that only a small portion of code is to be added to an existing function, insert functionality is provided. This saves having to completely redefine the old function from `spec0.md` in `spec1.md`. This is done by marking where the change is to occur in the old file and marking which code is to be inserted in the new file. This is done as follows: - -* In the old file, a label is added as a Python comment marking where the code is to be inserted. This would appear as follows in `spec0.md`: - -```python -def foo(x): - x << 1 - # @YourLabelHere - return x -``` - -* In spec1, the new code can then be inserted by having a code-block that looks as follows: - -```python -#begin insert @YourLabelHere - x += x -#end insert @YourLabelHere -``` - -*Note*: The code to be inserted has the **same level of indentation** as the surrounding code of its destination insert point. diff --git a/scripts/__init__.py b/scripts/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/scripts/build_spec.py b/scripts/build_spec.py deleted file mode 100644 index fa351db2f8..0000000000 --- a/scripts/build_spec.py +++ /dev/null @@ -1,309 +0,0 @@ -import re -from function_puller import ( - get_spec, - SpecObject, -) -from argparse import ArgumentParser -from typing import ( - Dict, - Optional, -) - -CONFIG_LOADER = ''' -apply_constants_preset(globals()) -''' - -PHASE0_IMPORTS = '''from eth2spec.config.apply_config import apply_constants_preset -from typing import ( - Any, Callable, Dict, Set, Sequence, Tuple, Optional, TypeVar -) - -from dataclasses import ( - dataclass, - field, -) - -from eth2spec.utils.ssz.ssz_impl import hash_tree_root -from eth2spec.utils.ssz.ssz_typing import ( - View, boolean, Container, List, Vector, uint64, - Bytes1, Bytes4, Bytes8, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, -) -from eth2spec.utils import bls - -from eth2spec.utils.hash_function import hash - -SSZObject = TypeVar('SSZObject', bound=View) -''' -PHASE1_IMPORTS = '''from eth2spec.phase0 import spec as phase0 -from eth2spec.config.apply_config import apply_constants_preset -from typing import ( - Any, Callable, Dict, Set, Sequence, NewType, Optional, Tuple, TypeVar -) - -from dataclasses import ( - dataclass, - field, -) - -from eth2spec.utils.ssz.ssz_impl import hash_tree_root -from eth2spec.utils.ssz.ssz_typing import ( - View, boolean, Container, List, Vector, uint64, uint8, bit, - ByteVector, ByteList, Bytes1, Bytes4, Bytes8, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, -) -from eth2spec.utils import bls - -from eth2spec.utils.hash_function import hash - - -SSZVariableName = str -GeneralizedIndex = NewType('GeneralizedIndex', int) -SSZObject = TypeVar('SSZObject', bound=View) -''' -SUNDRY_CONSTANTS_FUNCTIONS = ''' -def ceillog2(x: uint64) -> int: - return (x - 1).bit_length() -''' -SUNDRY_FUNCTIONS = ''' -# Monkey patch hash cache -_hash = hash -hash_cache: Dict[bytes, Bytes32] = {} - - -def get_eth1_data(distance: uint64) -> Bytes32: - return hash(distance) - - -def hash(x: bytes) -> Bytes32: # type: ignore - if x not in hash_cache: - hash_cache[x] = Bytes32(_hash(x)) - return hash_cache[x] - - -def cache_this(key_fn, value_fn): # type: ignore - cache_dict = {} # type: ignore - - def wrapper(*args, **kw): # type: ignore - key = key_fn(*args, **kw) - nonlocal cache_dict - if key not in cache_dict: - cache_dict[key] = value_fn(*args, **kw) - return cache_dict[key] - return wrapper - - -get_base_reward = cache_this( - lambda state, index: (state.validators.hash_tree_root(), state.slot), - get_base_reward) - -get_committee_count_at_slot = cache_this( - lambda state, epoch: (state.validators.hash_tree_root(), epoch), - get_committee_count_at_slot) - -get_active_validator_indices = cache_this( - lambda state, epoch: (state.validators.hash_tree_root(), epoch), - get_active_validator_indices) - -get_beacon_committee = cache_this( - lambda state, slot, index: (state.validators.hash_tree_root(), state.randao_mixes.hash_tree_root(), slot, index), - get_beacon_committee)''' - - -def objects_to_spec(functions: Dict[str, str], - custom_types: Dict[str, str], - constants: Dict[str, str], - ssz_objects: Dict[str, str], - imports: str, - version: str, - ) -> str: - """ - Given all the objects that constitute a spec, combine them into a single pyfile. - """ - new_type_definitions = ( - '\n\n'.join( - [ - f"class {key}({value}):\n pass\n" - for key, value in custom_types.items() - ] - ) - ) - for k in list(functions): - if "ceillog2" in k: - del functions[k] - functions_spec = '\n\n'.join(functions.values()) - for k in list(constants.keys()): - if k == "BLS12_381_Q": - constants[k] += " # noqa: E501" - constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, constants[x]), constants)) - ssz_objects_instantiation_spec = '\n\n'.join(ssz_objects.values()) - spec = ( - imports - + '\n\n' + f"version = \'{version}\'\n" - + '\n\n' + new_type_definitions - + '\n' + SUNDRY_CONSTANTS_FUNCTIONS - + '\n\n' + constants_spec - + '\n\n' + CONFIG_LOADER - + '\n\n' + ssz_objects_instantiation_spec - + '\n\n' + functions_spec - + '\n' + SUNDRY_FUNCTIONS - + '\n' - ) - return spec - - -def combine_functions(old_functions: Dict[str, str], new_functions: Dict[str, str]) -> Dict[str, str]: - for key, value in new_functions.items(): - old_functions[key] = value - return old_functions - - -def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, str]) -> Dict[str, str]: - for key, value in new_constants.items(): - old_constants[key] = value - return old_constants - - -ignored_dependencies = [ - 'bit', 'boolean', 'Vector', 'List', 'Container', 'BLSPubkey', 'BLSSignature', - 'Bytes1', 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', 'Bitlist', 'Bitvector', - 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', - 'bytes', 'byte', 'ByteList', 'ByteVector' -] - - -def dependency_order_ssz_objects(objects: Dict[str, str], custom_types: Dict[str, str]) -> None: - """ - Determines which SSZ Object is dependent on which other and orders them appropriately - """ - items = list(objects.items()) - for key, value in items: - dependencies = [] - for line in value.split('\n'): - if not re.match(r'\s+\w+: .+', line): - continue # skip whitespace etc. - line = line[line.index(':') + 1:] # strip of field name - if '#' in line: - line = line[:line.index('#')] # strip of comment - dependencies.extend(re.findall(r'(\w+)', line)) # catch all legible words, potential dependencies - dependencies = filter(lambda x: '_' not in x and x.upper() != x, dependencies) # filter out constants - dependencies = filter(lambda x: x not in ignored_dependencies, dependencies) - dependencies = filter(lambda x: x not in custom_types, dependencies) - for dep in dependencies: - key_list = list(objects.keys()) - for item in [dep, key] + key_list[key_list.index(dep)+1:]: - objects[item] = objects.pop(item) - - -def combine_ssz_objects(old_objects: Dict[str, str], new_objects: Dict[str, str], custom_types) -> Dict[str, str]: - """ - Takes in old spec and new spec ssz objects, combines them, - and returns the newer versions of the objects in dependency order. - """ - for key, value in new_objects.items(): - old_objects[key] = value - return old_objects - - -def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject: - """ - Takes in two spec variants (as tuples of their objects) and combines them using the appropriate combiner function. - """ - functions0, custom_types0, constants0, ssz_objects0 = spec0 - functions1, custom_types1, constants1, ssz_objects1 = spec1 - functions = combine_functions(functions0, functions1) - custom_types = combine_constants(custom_types0, custom_types1) - constants = combine_constants(constants0, constants1) - ssz_objects = combine_ssz_objects(ssz_objects0, ssz_objects1, custom_types) - return SpecObject((functions, custom_types, constants, ssz_objects)) - - -def dependency_order_spec(objs: SpecObject): - functions, custom_types, constants, ssz_objects = objs - dependency_order_ssz_objects(ssz_objects, custom_types) - - -def build_phase0_spec(phase0_sourcefile: str, fork_choice_sourcefile: str, - v_guide_sourcefile: str, outfile: str=None) -> Optional[str]: - phase0_spec = get_spec(phase0_sourcefile) - fork_choice_spec = get_spec(fork_choice_sourcefile) - v_guide = get_spec(v_guide_sourcefile) - spec_objects = phase0_spec - for value in [fork_choice_spec, v_guide]: - spec_objects = combine_spec_objects(spec_objects, value) - dependency_order_spec(spec_objects) - spec = objects_to_spec(*spec_objects, PHASE0_IMPORTS, 'phase0') - if outfile is not None: - with open(outfile, 'w') as out: - out.write(spec) - return spec - - -def build_phase1_spec(phase0_beacon_sourcefile: str, - phase0_fork_choice_sourcefile: str, - phase1_custody_sourcefile: str, - phase1_beacon_sourcefile: str, - phase1_fraud_sourcefile: str, - phase1_fork_choice_sourcefile: str, - phase1_fork_sourcefile: str, - outfile: str=None) -> Optional[str]: - all_sourcefiles = ( - phase0_beacon_sourcefile, - phase0_fork_choice_sourcefile, - phase1_custody_sourcefile, - phase1_beacon_sourcefile, - phase1_fraud_sourcefile, - phase1_fork_choice_sourcefile, - phase1_fork_sourcefile, - ) - all_spescs = [get_spec(spec) for spec in all_sourcefiles] - spec_objects = all_spescs[0] - for value in all_spescs[1:]: - spec_objects = combine_spec_objects(spec_objects, value) - dependency_order_spec(spec_objects) - spec = objects_to_spec(*spec_objects, PHASE1_IMPORTS, 'phase1') - if outfile is not None: - with open(outfile, 'w') as out: - out.write(spec) - return spec - - -if __name__ == '__main__': - description = ''' -Build the specs from the md docs. -If building phase 0: - 1st argument is input phase0/beacon-chain.md - 2nd argument is input phase0/fork-choice.md - 3rd argument is input phase0/validator.md - 4th argument is output spec.py - -If building phase 1: - 1st argument is input phase0/beacon-chain.md - 2nd argument is input phase0/fork-choice.md - 3rd argument is input phase1/custody-game.md - 4th argument is input phase1/beacon-chain.md - 5th argument is input phase1/fraud-proofs.md - 6th argument is input phase1/fork-choice.md - 7th argument is input phase1/phase1-fork.md - 8th argument is output spec.py -''' - parser = ArgumentParser(description=description) - parser.add_argument("-p", "--phase", dest="phase", type=int, default=0, help="Build for phase #") - parser.add_argument(dest="files", help="Input and output files", nargs="+") - - args = parser.parse_args() - if args.phase == 0: - if len(args.files) == 4: - build_phase0_spec(*args.files) - else: - print(" Phase 0 requires spec, forkchoice, and v-guide inputs as well as an output file.") - elif args.phase == 1: - if len(args.files) == 8: - build_phase1_spec(*args.files) - else: - print( - " Phase 1 requires input files as well as an output file:\n" - "\t phase0: (beacon-chain.md, fork-choice.md)\n" - "\t phase1: (custody-game.md, beacon-chain.md, fraud-proofs.md, fork-choice.md, phase1-fork.md)\n" - "\t and output.py" - ) - else: - print("Invalid phase: {0}".format(args.phase)) diff --git a/scripts/function_puller.py b/scripts/function_puller.py deleted file mode 100644 index 1a134007e0..0000000000 --- a/scripts/function_puller.py +++ /dev/null @@ -1,72 +0,0 @@ -import re -from typing import Dict, Tuple, NewType - - -FUNCTION_REGEX = r'^def [\w_]*' - -SpecObject = NewType('SpecObjects', Tuple[Dict[str, str], Dict[str, str], Dict[str, str], Dict[str, str]]) - - -def get_spec(file_name: str) -> SpecObject: - """ - Takes in the file name of a spec.md file, opens it and returns the following objects: - functions = {function_name: function_code} - constants= {constant_name: constant_code} - ssz_objects= {object_name: object} - - Note: This function makes heavy use of the inherent ordering of dicts, - if this is not supported by your python version, it will not work. - """ - pulling_from = None # line number of start of latest object - current_name = None # most recent section title - functions: Dict[str, str] = {} - constants: Dict[str, str] = {} - ssz_objects: Dict[str, str] = {} - function_matcher = re.compile(FUNCTION_REGEX) - is_ssz = False - custom_types: Dict[str, str] = {} - for linenum, line in enumerate(open(file_name).readlines()): - line = line.rstrip() - if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`': - current_name = line[line[:-1].rfind('`') + 1: -1] - if line[:9] == '```python': - assert pulling_from is None - pulling_from = linenum + 1 - elif line[:3] == '```': - pulling_from = None - else: - # Handle function definitions & ssz_objects - if pulling_from is not None: - # SSZ Object - if len(line) > 18 and line[:6] == 'class ' and line[-12:] == '(Container):': - name = line[6:-12] - # Check consistency with markdown header - assert name == current_name - is_ssz = True - # function definition - elif function_matcher.match(line) is not None: - current_name = function_matcher.match(line).group(0) - is_ssz = False - if is_ssz: - ssz_objects[current_name] = ssz_objects.get(current_name, '') + line + '\n' - else: - functions[current_name] = functions.get(current_name, '') + line + '\n' - # Handle constant and custom types table entries - elif pulling_from is None and len(line) > 0 and line[0] == '|': - row = line[1:].split('|') - if len(row) >= 2: - for i in range(2): - row[i] = row[i].strip().strip('`') - if '`' in row[i]: - row[i] = row[i][:row[i].find('`')] - is_constant_def = True - if row[0][0] not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_': - is_constant_def = False - for c in row[0]: - if c not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789': - is_constant_def = False - if is_constant_def: - constants[row[0]] = row[1].replace('**TBD**', '2**32') - elif row[1].startswith('uint') or row[1].startswith('Bytes'): - custom_types[row[0]] = row[1] - return SpecObject((functions, custom_types, constants, ssz_objects)) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000..17851d6a1d --- /dev/null +++ b/setup.py @@ -0,0 +1,465 @@ +from setuptools import setup, find_packages, Command +from setuptools.command.build_py import build_py +from distutils import dir_util +from distutils.util import convert_path +import os +import re +from typing import Dict, NamedTuple, List + +FUNCTION_REGEX = r'^def [\w_]*' + + +class SpecObject(NamedTuple): + functions: Dict[str, str] + custom_types: Dict[str, str] + constants: Dict[str, str] + ssz_objects: Dict[str, str] + + +def get_spec(file_name: str) -> SpecObject: + """ + Takes in the file name of a spec.md file, opens it and returns a parsed spec object. + + Note: This function makes heavy use of the inherent ordering of dicts, + if this is not supported by your python version, it will not work. + """ + pulling_from = None # line number of start of latest object + current_name = None # most recent section title + functions: Dict[str, str] = {} + constants: Dict[str, str] = {} + ssz_objects: Dict[str, str] = {} + function_matcher = re.compile(FUNCTION_REGEX) + is_ssz = False + custom_types: Dict[str, str] = {} + for linenum, line in enumerate(open(file_name).readlines()): + line = line.rstrip() + if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`': + current_name = line[line[:-1].rfind('`') + 1: -1] + if line[:9] == '```python': + assert pulling_from is None + pulling_from = linenum + 1 + elif line[:3] == '```': + pulling_from = None + else: + # Handle function definitions & ssz_objects + if pulling_from is not None: + # SSZ Object + if len(line) > 18 and line[:6] == 'class ' and line[-12:] == '(Container):': + name = line[6:-12] + # Check consistency with markdown header + assert name == current_name + is_ssz = True + # function definition + elif function_matcher.match(line) is not None: + current_name = function_matcher.match(line).group(0) + is_ssz = False + if is_ssz: + ssz_objects[current_name] = ssz_objects.get(current_name, '') + line + '\n' + else: + functions[current_name] = functions.get(current_name, '') + line + '\n' + # Handle constant and custom types table entries + elif pulling_from is None and len(line) > 0 and line[0] == '|': + row = line[1:].split('|') + if len(row) >= 2: + for i in range(2): + row[i] = row[i].strip().strip('`') + if '`' in row[i]: + row[i] = row[i][:row[i].find('`')] + is_constant_def = True + if row[0][0] not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_': + is_constant_def = False + for c in row[0]: + if c not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789': + is_constant_def = False + if is_constant_def: + constants[row[0]] = row[1].replace('**TBD**', '2**32') + elif row[1].startswith('uint') or row[1].startswith('Bytes'): + custom_types[row[0]] = row[1] + return SpecObject(functions, custom_types, constants, ssz_objects) + + +CONFIG_LOADER = ''' +apply_constants_config(globals()) +''' + +PHASE0_IMPORTS = '''from eth2spec.config.config_util import apply_constants_config +from typing import ( + Any, Callable, Dict, Set, Sequence, Tuple, Optional, TypeVar +) + +from dataclasses import ( + dataclass, + field, +) + +from eth2spec.utils.ssz.ssz_impl import hash_tree_root +from eth2spec.utils.ssz.ssz_typing import ( + View, boolean, Container, List, Vector, uint64, + Bytes1, Bytes4, Bytes8, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, +) +from eth2spec.utils import bls + +from eth2spec.utils.hash_function import hash + +SSZObject = TypeVar('SSZObject', bound=View) +''' +PHASE1_IMPORTS = '''from eth2spec.phase0 import spec as phase0 +from eth2spec.config.config_util import apply_constants_config +from typing import ( + Any, Dict, Set, Sequence, NewType, Tuple, TypeVar, Callable, Optional +) + +from dataclasses import ( + dataclass, + field, +) + +from eth2spec.utils.ssz.ssz_impl import hash_tree_root +from eth2spec.utils.ssz.ssz_typing import ( + View, boolean, Container, List, Vector, uint64, uint8, bit, + ByteVector, ByteList, Bytes1, Bytes4, Bytes8, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, +) +from eth2spec.utils import bls + +from eth2spec.utils.hash_function import hash + +# Whenever phase 1 is loaded, make sure we have the latest phase0 +from importlib import reload +reload(phase0) + + +SSZVariableName = str +GeneralizedIndex = NewType('GeneralizedIndex', int) +SSZObject = TypeVar('SSZObject', bound=View) +''' +SUNDRY_CONSTANTS_FUNCTIONS = ''' +def ceillog2(x: uint64) -> int: + return (x - 1).bit_length() +''' +SUNDRY_FUNCTIONS = ''' +# Monkey patch hash cache +_hash = hash +hash_cache: Dict[bytes, Bytes32] = {} + + +def get_eth1_data(distance: uint64) -> Bytes32: + return hash(distance) + + +def hash(x: bytes) -> Bytes32: # type: ignore + if x not in hash_cache: + hash_cache[x] = Bytes32(_hash(x)) + return hash_cache[x] + + +def cache_this(key_fn, value_fn): # type: ignore + cache_dict = {} # type: ignore + + def wrapper(*args, **kw): # type: ignore + key = key_fn(*args, **kw) + nonlocal cache_dict + if key not in cache_dict: + cache_dict[key] = value_fn(*args, **kw) + return cache_dict[key] + return wrapper + + +get_base_reward = cache_this( + lambda state, index: (state.validators.hash_tree_root(), state.slot), + get_base_reward) + +get_committee_count_at_slot = cache_this( + lambda state, epoch: (state.validators.hash_tree_root(), epoch), + get_committee_count_at_slot) + +get_active_validator_indices = cache_this( + lambda state, epoch: (state.validators.hash_tree_root(), epoch), + get_active_validator_indices) + +get_beacon_committee = cache_this( + lambda state, slot, index: (state.validators.hash_tree_root(), state.randao_mixes.hash_tree_root(), slot, index), + get_beacon_committee)''' + + +def objects_to_spec(spec_object: SpecObject, imports: str, version: str) -> str: + """ + Given all the objects that constitute a spec, combine them into a single pyfile. + """ + new_type_definitions = ( + '\n\n'.join( + [ + f"class {key}({value}):\n pass\n" + for key, value in spec_object.custom_types.items() + ] + ) + ) + for k in list(spec_object.functions): + if "ceillog2" in k: + del spec_object.functions[k] + functions_spec = '\n\n'.join(spec_object.functions.values()) + for k in list(spec_object.constants.keys()): + if k == "BLS12_381_Q": + spec_object.constants[k] += " # noqa: E501" + constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, spec_object.constants[x]), spec_object.constants)) + ssz_objects_instantiation_spec = '\n\n'.join(spec_object.ssz_objects.values()) + spec = ( + imports + + '\n\n' + f"version = \'{version}\'\n" + + '\n\n' + new_type_definitions + + '\n' + SUNDRY_CONSTANTS_FUNCTIONS + + '\n\n' + constants_spec + + '\n\n' + CONFIG_LOADER + + '\n\n' + ssz_objects_instantiation_spec + + '\n\n' + functions_spec + + '\n' + SUNDRY_FUNCTIONS + + '\n' + ) + return spec + + +def combine_functions(old_functions: Dict[str, str], new_functions: Dict[str, str]) -> Dict[str, str]: + for key, value in new_functions.items(): + old_functions[key] = value + return old_functions + + +def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, str]) -> Dict[str, str]: + for key, value in new_constants.items(): + old_constants[key] = value + return old_constants + + +ignored_dependencies = [ + 'bit', 'boolean', 'Vector', 'List', 'Container', 'BLSPubkey', 'BLSSignature', + 'Bytes1', 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', 'Bitlist', 'Bitvector', + 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', + 'bytes', 'byte', 'ByteList', 'ByteVector' +] + + +def dependency_order_ssz_objects(objects: Dict[str, str], custom_types: Dict[str, str]) -> None: + """ + Determines which SSZ Object is dependent on which other and orders them appropriately + """ + items = list(objects.items()) + for key, value in items: + dependencies = [] + for line in value.split('\n'): + if not re.match(r'\s+\w+: .+', line): + continue # skip whitespace etc. + line = line[line.index(':') + 1:] # strip of field name + if '#' in line: + line = line[:line.index('#')] # strip of comment + dependencies.extend(re.findall(r'(\w+)', line)) # catch all legible words, potential dependencies + dependencies = filter(lambda x: '_' not in x and x.upper() != x, dependencies) # filter out constants + dependencies = filter(lambda x: x not in ignored_dependencies, dependencies) + dependencies = filter(lambda x: x not in custom_types, dependencies) + for dep in dependencies: + key_list = list(objects.keys()) + for item in [dep, key] + key_list[key_list.index(dep)+1:]: + objects[item] = objects.pop(item) + + +def combine_ssz_objects(old_objects: Dict[str, str], new_objects: Dict[str, str], custom_types) -> Dict[str, str]: + """ + Takes in old spec and new spec ssz objects, combines them, + and returns the newer versions of the objects in dependency order. + """ + for key, value in new_objects.items(): + old_objects[key] = value + return old_objects + + +def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject: + """ + Takes in two spec variants (as tuples of their objects) and combines them using the appropriate combiner function. + """ + functions0, custom_types0, constants0, ssz_objects0 = spec0 + functions1, custom_types1, constants1, ssz_objects1 = spec1 + functions = combine_functions(functions0, functions1) + custom_types = combine_constants(custom_types0, custom_types1) + constants = combine_constants(constants0, constants1) + ssz_objects = combine_ssz_objects(ssz_objects0, ssz_objects1, custom_types) + return SpecObject(functions, custom_types, constants, ssz_objects) + + +def dependency_order_spec(objs: SpecObject): + functions, custom_types, constants, ssz_objects = objs + dependency_order_ssz_objects(ssz_objects, custom_types) + + +version_imports = { + 'phase0': PHASE0_IMPORTS, + 'phase1': PHASE1_IMPORTS, +} + + +def build_spec(version: str, source_files: List[str]) -> str: + all_spescs = [get_spec(spec) for spec in source_files] + + spec_object = all_spescs[0] + for value in all_spescs[1:]: + spec_object = combine_spec_objects(spec_object, value) + + dependency_order_spec(spec_object) + + return objects_to_spec(spec_object, version_imports[version], version) + + +class PySpecCommand(Command): + """Convert spec markdown files to a spec python file""" + + description = "Convert spec markdown files to a spec python file" + + spec_version: str + md_doc_paths: str + parsed_md_doc_paths: List[str] + out_dir: str + + # The format is (long option, short option, description). + user_options = [ + ('spec-version=', None, "Spec version to tag build with. Used to select md-docs defaults."), + ('md-doc-paths=', None, "List of paths of markdown files to build spec with"), + ('out-dir=', None, "Output directory to write spec package to") + ] + + def initialize_options(self): + """Set default values for options.""" + # Each user option must be listed here with their default value. + self.spec_version = 'phase0' + self.md_doc_paths = '' + self.out_dir = 'pyspec_output' + + def finalize_options(self): + """Post-process options.""" + if len(self.md_doc_paths) == 0: + if self.spec_version == "phase0": + self.md_doc_paths = """ + specs/phase0/beacon-chain.md + specs/phase0/fork-choice.md + specs/phase0/validator.md + """ + elif self.spec_version == "phase1": + self.md_doc_paths = """ + specs/phase0/beacon-chain.md + specs/phase0/fork-choice.md + specs/phase1/custody-game.md + specs/phase1/beacon-chain.md + specs/phase1/fraud-proofs.md + specs/phase1/fork-choice.md + specs/phase1/phase1-fork.md + """ + else: + raise Exception('no markdown files specified, and spec version "%s" is unknown', self.spec_version) + print("no paths were specified, using default markdown file paths for pyspec build (spec version: %s)" % self.spec_version) + + self.parsed_md_doc_paths = self.md_doc_paths.split() + + for filename in self.parsed_md_doc_paths: + if not os.path.exists(filename): + raise Exception('Pyspec markdown input file "%s" does not exist.' % filename) + + def run(self): + spec_str = build_spec(self.spec_version, self.parsed_md_doc_paths) + if self.dry_run: + self.announce('dry run successfully prepared contents for spec.' + f' out dir: "{self.out_dir}", spec version: "{self.spec_version}"') + self.debug_print(spec_str) + else: + dir_util.mkpath(self.out_dir) + with open(os.path.join(self.out_dir, 'spec.py'), 'w') as out: + out.write(spec_str) + with open(os.path.join(self.out_dir, '__init__.py'), 'w') as out: + out.write("") + + +class BuildPyCommand(build_py): + """Customize the build command to run the spec-builder on setup.py build""" + + def initialize_options(self): + super(BuildPyCommand, self).initialize_options() + + def run_pyspec_cmd(self, spec_version: str, **opts): + cmd_obj: PySpecCommand = self.distribution.reinitialize_command("pyspec") + cmd_obj.spec_version = spec_version + cmd_obj.out_dir = os.path.join(self.build_lib, 'eth2spec', spec_version) + for k, v in opts.items(): + setattr(cmd_obj, k, v) + self.run_command('pyspec') + + def run(self): + self.run_pyspec_cmd(spec_version="phase0") + self.run_pyspec_cmd(spec_version="phase1") + + super(BuildPyCommand, self).run() + + +class PyspecDevCommand(Command): + """Build the markdown files in-place to their source location for testing.""" + description = "Build the markdown files in-place to their source location for testing." + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run_pyspec_cmd(self, spec_version: str, **opts): + cmd_obj: PySpecCommand = self.distribution.reinitialize_command("pyspec") + cmd_obj.spec_version = spec_version + eth2spec_dir = convert_path(self.distribution.package_dir['eth2spec']) + cmd_obj.out_dir = os.path.join(eth2spec_dir, spec_version) + for k, v in opts.items(): + setattr(cmd_obj, k, v) + self.run_command('pyspec') + + def run(self): + print("running build_py command") + self.run_pyspec_cmd(spec_version="phase0", package_inplace=False) + self.run_pyspec_cmd(spec_version="phase1", package_inplace=False) + + +commands = { + 'pyspec': PySpecCommand, + 'build_py': BuildPyCommand, + 'pyspecdev': PyspecDevCommand, +} + +with open("README.md", "rt", encoding="utf8") as f: + readme = f.read() + +setup( + name='eth2spec', + description="Eth2 spec, provided as Python package for tooling and testing", + long_description=readme, + long_description_content_type="text/markdown", + author="ethereum", + url="https://github.com/ethereum/eth2.0-specs", + include_package_data=False, + package_data={'configs': ['*.yaml'], + 'specs': ['**/*.md']}, + package_dir={ + "eth2spec": "tests/core/pyspec/eth2spec", + "configs": "configs", + "specs": "specs" + }, + packages=find_packages(where='tests/core/pyspec') + ['configs', 'specs'], + py_modules=["eth2spec"], + cmdclass=commands, + python_requires=">=3.8, <4", + tests_require=[], # avoid old style tests require. Enable explicit (re-)installs, e.g. `pip install .[testing]` + extras_require={ + "testing": ["pytest>=4.4", "pytest-cov", "pytest-xdist"], + "linting": ["flake8==3.7.7", "mypy==0.750"], + }, + install_requires=[ + "eth-utils>=1.3.0,<2", + "eth-typing>=2.1.0,<3.0.0", + "pycryptodome==3.9.4", + "py_ecc==2.0.0", + "dataclasses==0.6", + "remerkleable==0.1.10", + "ruamel.yaml==0.16.5" + ] +) diff --git a/tests/core/config_helpers/README.md b/tests/core/config_helpers/README.md deleted file mode 100644 index 85a9304d75..0000000000 --- a/tests/core/config_helpers/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# Eth2 config helpers - -`preset_loader`: A util to load config-presets with. -See [Configs documentation](../../../configs/README.md). - -Usage: - -```python -configs_path = 'configs/' - -... - -import preset_loader -from eth2spec.phase0 import spec -my_presets = preset_loader.load_presets(configs_path, 'mainnet') -spec.apply_constants_preset(my_presets) -``` - -WARNING: this overwrites globals, make sure to prevent accidental collisions with other usage of the same imported specs package. diff --git a/tests/core/config_helpers/preset_loader/__init__.py b/tests/core/config_helpers/preset_loader/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/core/config_helpers/preset_loader/loader.py b/tests/core/config_helpers/preset_loader/loader.py deleted file mode 100644 index 95f147f6e0..0000000000 --- a/tests/core/config_helpers/preset_loader/loader.py +++ /dev/null @@ -1,27 +0,0 @@ -from typing import Dict, Any - -from ruamel.yaml import ( - YAML, -) -from pathlib import Path -from os.path import join - - -def load_presets(configs_dir, presets_name) -> Dict[str, Any]: - """ - Loads the given preset - :param presets_name: The name of the presets. (lowercase snake_case) - :return: Dictionary, mapping of constant-name -> constant-value - """ - path = Path(join(configs_dir, presets_name+'.yaml')) - yaml = YAML(typ='base') - loaded = yaml.load(path) - out = dict() - for k, v in loaded.items(): - if isinstance(v, list): - out[k] = v - elif isinstance(v, str) and v.startswith("0x"): - out[k] = bytes.fromhex(v[2:]) - else: - out[k] = int(v) - return out diff --git a/tests/core/config_helpers/requirements.txt b/tests/core/config_helpers/requirements.txt deleted file mode 100644 index 6c7334268a..0000000000 --- a/tests/core/config_helpers/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -ruamel.yaml==0.16.5 diff --git a/tests/core/config_helpers/setup.py b/tests/core/config_helpers/setup.py deleted file mode 100644 index 3f893f3d46..0000000000 --- a/tests/core/config_helpers/setup.py +++ /dev/null @@ -1,9 +0,0 @@ -from distutils.core import setup - -setup( - name='config_helpers', - packages=['preset_loader'], - install_requires=[ - "ruamel.yaml==0.16.5" - ] -) diff --git a/tests/core/pyspec/README.md b/tests/core/pyspec/README.md index 2e596520c7..4b2269a2b4 100644 --- a/tests/core/pyspec/README.md +++ b/tests/core/pyspec/README.md @@ -7,22 +7,31 @@ With this executable spec, test-generators can easily create test-vectors for client implementations, and the spec itself can be verified to be consistent and coherent through sanity tests implemented with pytest. - ## Building -All the dynamic parts of the spec can be build at once with `make pyspec`. +Building the pyspec is simply: `python setup.py build` + (or `pip install .`, but beware that ignored files will still be copied over to a temporary dir, due to pip issue 2195). +This outputs the build files to the `./build/lib/eth2spec/...` dir, and can't be used for local test running. Instead, use the dev-install as described below. -Alternatively, you can build a sub-set of the pyspec: `make phase0`. +## Dev Install -Or, to build a single file, specify the path, e.g. `make test_libs/pyspec/eth2spec/phase0/spec.py`. +All the dynamic parts of the spec are automatically built with `python setup.py pyspecdev`. +Unlike the regular install, this outputs spec files to their original source location, instead of build output only. +Alternatively, you can build a sub-set of the pyspec with the distutil command: +```bash +python setup.py pyspec --spec-version=phase0 --md-doc-paths="specs/phase0/beacon-chain.md specs/phase0/fork-choice.md" --out-dir=my_spec_dir +``` ## Py-tests -After building, you can install the dependencies for running the `pyspec` tests with `make install_test`. +After installing, you can install the optional dependencies for testing and linting. +With makefile: `make install_test`. +Or manually: run `pip install .[testing]` and `pip install .[linting]`. These tests are not intended for client-consumption. -These tests are sanity tests, to verify if the spec itself is consistent. +These tests are testing the spec itself, to verify consistency and provide feedback on modifications of the spec. +However, most of the tests can be run in generator-mode, to output test vectors for client-consumption. ### How to run tests @@ -32,23 +41,19 @@ Run `make test` from the root of the specs repository (after running `make insta #### Manual -From within the `pyspec` folder: +From the repository root: -Install dependencies: +Install venv and install: ```bash python3 -m venv venv . venv/bin/activate -pip3 install -r requirements-testing.txt +python setup.py pyspecdev ``` -*Note*: Make sure to run `make -B pyspec` from the root of the specs repository, - to build the parts of the pyspec module derived from the markdown specs. -The `-B` flag may be helpful to force-overwrite the `pyspec` output after you made a change to the markdown source files. -Run the tests: +Run the test command from the `tests/core/pyspec` directory: ``` pytest --config=minimal eth2spec ``` -Note the package-name, this is to locate the tests. ### How to view code coverage report diff --git a/tests/core/pyspec/eth2spec/config/README.md b/tests/core/pyspec/eth2spec/config/README.md new file mode 100644 index 0000000000..ea2b2ccd84 --- /dev/null +++ b/tests/core/pyspec/eth2spec/config/README.md @@ -0,0 +1,20 @@ +# Eth2 config util + +For configuration, see [Configs documentation](../../../../../configs/README.md). + +## Usage: + +```python +configs_path = 'configs/' + +... + +from eth2spec.config import config_util +from eth2spec.phase0 import spec +from importlib import reload +my_presets = config_util.prepare_config(configs_path, 'mainnet') +# reload spec to make loaded config effective +reload(spec) +``` + +WARNING: this overwrites globals, make sure to prevent accidental collisions with other usage of the same imported specs package. diff --git a/tests/core/pyspec/eth2spec/config/apply_config.py b/tests/core/pyspec/eth2spec/config/apply_config.py deleted file mode 100644 index 2f0ce59021..0000000000 --- a/tests/core/pyspec/eth2spec/config/apply_config.py +++ /dev/null @@ -1,22 +0,0 @@ -from preset_loader import loader -from typing import Dict, Any - -presets: Dict[str, Any] = {} - - -# Access to overwrite spec constants based on configuration -# This is called by the spec module after declaring its globals, and applies the loaded presets. -def apply_constants_preset(spec_globals: Dict[str, Any]) -> None: - global presets - for k, v in presets.items(): - if k.startswith('DOMAIN_'): - spec_globals[k] = spec_globals['DomainType'](v) # domain types are defined as bytes in the configs - else: - spec_globals[k] = v - - -# Load presets from a file. This does not apply the presets. -# To apply the presets, reload the spec module (it will re-initialize with the presets taken from here). -def load_presets(configs_path, config_name): - global presets - presets = loader.load_presets(configs_path, config_name) diff --git a/tests/core/pyspec/eth2spec/config/config_util.py b/tests/core/pyspec/eth2spec/config/config_util.py new file mode 100644 index 0000000000..42ad76d69d --- /dev/null +++ b/tests/core/pyspec/eth2spec/config/config_util.py @@ -0,0 +1,44 @@ +from ruamel.yaml import YAML +from pathlib import Path +from os.path import join +from typing import Dict, Any + +config: Dict[str, Any] = {} + + +# Access to overwrite spec constants based on configuration +# This is called by the spec module after declaring its globals, and applies the loaded presets. +def apply_constants_config(spec_globals: Dict[str, Any]) -> None: + global config + for k, v in config.items(): + if k.startswith('DOMAIN_'): + spec_globals[k] = spec_globals['DomainType'](v) # domain types are defined as bytes in the configs + else: + spec_globals[k] = v + + +# Load presets from a file, and then prepares the global config setting. This does not apply the config. +# To apply the config, reload the spec module (it will re-initialize with the config taken from here). +def prepare_config(configs_path, config_name): + global config + config = load_config_file(configs_path, config_name) + + +def load_config_file(configs_dir, presets_name) -> Dict[str, Any]: + """ + Loads the given preset + :param presets_name: The name of the presets. (lowercase snake_case) + :return: Dictionary, mapping of constant-name -> constant-value + """ + path = Path(join(configs_dir, presets_name + '.yaml')) + yaml = YAML(typ='base') + loaded = yaml.load(path) + out = dict() + for k, v in loaded.items(): + if isinstance(v, list): + out[k] = v + elif isinstance(v, str) and v.startswith("0x"): + out[k] = bytes.fromhex(v[2:]) + else: + out[k] = int(v) + return out diff --git a/tests/core/pyspec/eth2spec/phase0/__init__.py b/tests/core/pyspec/eth2spec/phase0/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/core/pyspec/eth2spec/phase1/__init__.py b/tests/core/pyspec/eth2spec/phase1/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/core/pyspec/eth2spec/test/conftest.py b/tests/core/pyspec/eth2spec/test/conftest.py index 08cd850abb..01187b05fc 100644 --- a/tests/core/pyspec/eth2spec/test/conftest.py +++ b/tests/core/pyspec/eth2spec/test/conftest.py @@ -1,4 +1,4 @@ -from eth2spec.config import apply_config +from eth2spec.config import config_util from eth2spec.test.context import reload_specs @@ -34,6 +34,6 @@ def pytest_addoption(parser): @fixture(autouse=True) def config(request): config_name = request.config.getoption("--config") - apply_config.load_presets('../../../configs/', config_name) + config_util.prepare_config('../../../configs/', config_name) # now that the presets are loaded, reload the specs to apply them reload_specs() diff --git a/tests/core/pyspec/requirements-testing.txt b/tests/core/pyspec/requirements-testing.txt deleted file mode 100644 index e8ecd12a66..0000000000 --- a/tests/core/pyspec/requirements-testing.txt +++ /dev/null @@ -1,7 +0,0 @@ --r requirements.txt -pytest>=4.4 -../config_helpers -flake8==3.7.7 -mypy==0.750 -pytest-cov -pytest-xdist diff --git a/tests/core/pyspec/requirements.txt b/tests/core/pyspec/requirements.txt deleted file mode 100644 index 01f1caed78..0000000000 --- a/tests/core/pyspec/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -eth-utils>=1.3.0,<2 -eth-typing>=2.1.0,<3.0.0 -pycryptodome==3.9.4 -py_ecc==2.0.0 -dataclasses==0.6 -remerkleable==0.1.10 diff --git a/tests/core/pyspec/setup.py b/tests/core/pyspec/setup.py deleted file mode 100644 index 319b869536..0000000000 --- a/tests/core/pyspec/setup.py +++ /dev/null @@ -1,16 +0,0 @@ -from setuptools import setup, find_packages - -setup( - name='pyspec', - packages=find_packages(), - python_requires=">=3.8, <4", - tests_require=["pytest"], - install_requires=[ - "eth-utils>=1.3.0,<2", - "eth-typing>=2.1.0,<3.0.0", - "pycryptodome==3.9.4", - "py_ecc==2.0.0", - "dataclasses==0.6", - "remerkleable==0.1.10", - ] -) diff --git a/tests/generators/epoch_processing/main.py b/tests/generators/epoch_processing/main.py index b6e3d6c049..8f2a6e94fc 100644 --- a/tests/generators/epoch_processing/main.py +++ b/tests/generators/epoch_processing/main.py @@ -11,15 +11,16 @@ ) from gen_base import gen_runner, gen_typing from gen_from_tests.gen import generate_from_tests -from preset_loader import loader +from importlib import reload +from eth2spec.config import config_util def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: - presets = loader.load_presets(configs_path, config_name) - spec_phase0.apply_constants_preset(presets) - spec_phase1.apply_constants_preset(presets) + config_util.prepare_config(configs_path, config_name) + reload(spec_phase0) + reload(spec_phase1) return config_name def cases_fn() -> Iterable[gen_typing.TestCase]: diff --git a/tests/generators/epoch_processing/requirements.txt b/tests/generators/epoch_processing/requirements.txt index 3c318f56b2..b823142988 100644 --- a/tests/generators/epoch_processing/requirements.txt +++ b/tests/generators/epoch_processing/requirements.txt @@ -1,3 +1,2 @@ ../../core/gen_helpers -../../core/config_helpers -../../core/pyspec \ No newline at end of file +../../../ \ No newline at end of file diff --git a/tests/generators/genesis/main.py b/tests/generators/genesis/main.py index 9a91afbfd7..3563c3fd9d 100644 --- a/tests/generators/genesis/main.py +++ b/tests/generators/genesis/main.py @@ -4,15 +4,16 @@ from gen_base import gen_runner, gen_typing from gen_from_tests.gen import generate_from_tests -from preset_loader import loader from eth2spec.phase0 import spec as spec +from importlib import reload +from eth2spec.config import config_util def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: - presets = loader.load_presets(configs_path, config_name) - spec.apply_constants_preset(presets) + config_util.prepare_config(configs_path, config_name) + reload(spec) return config_name def cases_fn() -> Iterable[gen_typing.TestCase]: diff --git a/tests/generators/genesis/requirements.txt b/tests/generators/genesis/requirements.txt index 3c318f56b2..b823142988 100644 --- a/tests/generators/genesis/requirements.txt +++ b/tests/generators/genesis/requirements.txt @@ -1,3 +1,2 @@ ../../core/gen_helpers -../../core/config_helpers -../../core/pyspec \ No newline at end of file +../../../ \ No newline at end of file diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index de3eb7cf11..6906c9df71 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -11,7 +11,8 @@ from gen_base import gen_runner, gen_typing from gen_from_tests.gen import generate_from_tests -from preset_loader import loader +from importlib import reload +from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 from eth2spec.phase1 import spec as spec_phase1 @@ -19,9 +20,9 @@ def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: - presets = loader.load_presets(configs_path, config_name) - spec_phase0.apply_constants_preset(presets) - spec_phase1.apply_constants_preset(presets) + config_util.prepare_config(configs_path, config_name) + reload(spec_phase0) + reload(spec_phase1) return config_name def cases_fn() -> Iterable[gen_typing.TestCase]: diff --git a/tests/generators/operations/requirements.txt b/tests/generators/operations/requirements.txt index f34243cf4b..a6ea61aea2 100644 --- a/tests/generators/operations/requirements.txt +++ b/tests/generators/operations/requirements.txt @@ -1,4 +1,3 @@ eth-utils==1.6.0 ../../core/gen_helpers -../../core/config_helpers -../../core/pyspec \ No newline at end of file +../../../ \ No newline at end of file diff --git a/tests/generators/sanity/main.py b/tests/generators/sanity/main.py index 712f51c076..051f4877f9 100644 --- a/tests/generators/sanity/main.py +++ b/tests/generators/sanity/main.py @@ -4,7 +4,10 @@ from gen_base import gen_runner, gen_typing from gen_from_tests.gen import generate_from_tests -from preset_loader import loader + +from importlib import reload +from eth2spec.config import config_util + from eth2spec.phase0 import spec as spec_phase0 from eth2spec.phase1 import spec as spec_phase1 @@ -12,9 +15,9 @@ def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: - presets = loader.load_presets(configs_path, config_name) - spec_phase0.apply_constants_preset(presets) - spec_phase1.apply_constants_preset(presets) + config_util.prepare_config(configs_path, config_name) + reload(spec_phase0) + reload(spec_phase1) return config_name def cases_fn() -> Iterable[gen_typing.TestCase]: diff --git a/tests/generators/sanity/requirements.txt b/tests/generators/sanity/requirements.txt index 3c318f56b2..b823142988 100644 --- a/tests/generators/sanity/requirements.txt +++ b/tests/generators/sanity/requirements.txt @@ -1,3 +1,2 @@ ../../core/gen_helpers -../../core/config_helpers -../../core/pyspec \ No newline at end of file +../../../ \ No newline at end of file diff --git a/tests/generators/shuffling/main.py b/tests/generators/shuffling/main.py index 6425c708ac..63284db2c7 100644 --- a/tests/generators/shuffling/main.py +++ b/tests/generators/shuffling/main.py @@ -1,9 +1,11 @@ from eth2spec.phase0 import spec as spec from eth_utils import to_tuple from gen_base import gen_runner, gen_typing -from preset_loader import loader from typing import Iterable +from importlib import reload +from eth2spec.config import config_util + def shuffling_case_fn(seed, count): yield 'mapping', 'data', { @@ -27,8 +29,8 @@ def shuffling_test_cases(): def create_provider(config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: - presets = loader.load_presets(configs_path, config_name) - spec.apply_constants_preset(presets) + config_util.prepare_config(configs_path, config_name) + reload(spec) return config_name def cases_fn() -> Iterable[gen_typing.TestCase]: diff --git a/tests/generators/shuffling/requirements.txt b/tests/generators/shuffling/requirements.txt index f34243cf4b..a6ea61aea2 100644 --- a/tests/generators/shuffling/requirements.txt +++ b/tests/generators/shuffling/requirements.txt @@ -1,4 +1,3 @@ eth-utils==1.6.0 ../../core/gen_helpers -../../core/config_helpers -../../core/pyspec \ No newline at end of file +../../../ \ No newline at end of file diff --git a/tests/generators/ssz_generic/requirements.txt b/tests/generators/ssz_generic/requirements.txt index 6b11d61afb..af061a3b16 100644 --- a/tests/generators/ssz_generic/requirements.txt +++ b/tests/generators/ssz_generic/requirements.txt @@ -1,4 +1,3 @@ eth-utils==1.6.0 ../../core/gen_helpers -../../core/config_helpers -../../core/pyspec +../../../ diff --git a/tests/generators/ssz_static/main.py b/tests/generators/ssz_static/main.py index 334f45fa7e..bae911a0e4 100644 --- a/tests/generators/ssz_static/main.py +++ b/tests/generators/ssz_static/main.py @@ -10,7 +10,11 @@ serialize, ) from gen_base import gen_runner, gen_typing -from preset_loader import loader + + +from importlib import reload +from eth2spec.config import config_util + MAX_BYTES_LENGTH = 100 MAX_LIST_LENGTH = 10 @@ -54,8 +58,8 @@ def create_provider(config_name: str, seed: int, mode: random_value.Randomizatio cases_if_random: int) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: # Apply changes to presets, this affects some of the vector types. - presets = loader.load_presets(configs_path, config_name) - spec.apply_constants_preset(presets) + config_util.prepare_config(configs_path, config_name) + reload(spec) return config_name def cases_fn() -> Iterable[gen_typing.TestCase]: diff --git a/tests/generators/ssz_static/requirements.txt b/tests/generators/ssz_static/requirements.txt index 3c318f56b2..b823142988 100644 --- a/tests/generators/ssz_static/requirements.txt +++ b/tests/generators/ssz_static/requirements.txt @@ -1,3 +1,2 @@ ../../core/gen_helpers -../../core/config_helpers -../../core/pyspec \ No newline at end of file +../../../ \ No newline at end of file