Remerkleable - merkle tree based ssz for better and faster spec

Zkx007 · Jan 24, 2020 · d9f62f9 · d9f62f9
1 parent 8e5c176
commit d9f62f9
Show file tree

Hide file tree

Showing 27 changed files with 200 additions and 1,466 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -35,39 +35,39 @@ commands:
     description: "Restore the cache with pyspec keys"
     steps:
       - restore_cached_venv:
-          venv_name: v9-pyspec
+          venv_name: v17-pyspec
           reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "tests/core/pyspec/requirements-testing.txt" }}
   save_pyspec_cached_venv:
     description: Save a venv into a cache with pyspec keys"
     steps:
       - save_cached_venv:
-          venv_name: v9-pyspec
+          venv_name: v17-pyspec
           reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "tests/core/pyspec/requirements-testing.txt" }}
           venv_path: ./tests/core/pyspec/venv
   restore_deposit_contract_compiler_cached_venv:
     description: "Restore the venv from cache for the deposit contract compiler"
     steps:
       - restore_cached_venv:
-          venv_name: v15-deposit-contract-compiler
+          venv_name: v16-deposit-contract-compiler
           reqs_checksum: cache-{{ checksum "deposit_contract/compiler/requirements.txt" }}
   save_deposit_contract_compiler_cached_venv:
     description: "Save the venv to cache for later use of the deposit contract compiler"
     steps:
       - save_cached_venv:
-          venv_name: v15-deposit-contract-compiler
+          venv_name: v16-deposit-contract-compiler
           reqs_checksum: cache-{{ checksum "deposit_contract/compiler/requirements.txt" }}
           venv_path: ./deposit_contract/compiler/venv
   restore_deposit_contract_tester_cached_venv:
     description: "Restore the venv from cache for the deposit contract tester"
     steps:
       - restore_cached_venv:
-          venv_name: v15-deposit-contract-tester
+          venv_name: v17-deposit-contract-tester
           reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/tester/requirements.txt" }}
   save_deposit_contract_tester_cached_venv:
     description: "Save the venv to cache for later use of the deposit contract tester"
     steps:
       - save_cached_venv:
-          venv_name: v15-deposit-contract-tester
+          venv_name: v17-deposit-contract-tester
           reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/tester/requirements.txt" }}
           venv_path: ./deposit_contract/tester/venv
 jobs:

diff --git a/.gitignore b/.gitignore
@@ -20,6 +20,9 @@ tests/core/pyspec/eth2spec/phase1/spec.py
 # coverage reports
 .htmlcov
 .coverage
+.coverage.*
 
 # local CI testing output
 tests/core/pyspec/test-reports
+
+*.egg-info
diff --git a/deposit_contract/tester/requirements.txt b/deposit_contract/tester/requirements.txt
@@ -2,3 +2,4 @@ eth-tester[py-evm]>=0.3.0b1,<0.4
 web3==5.4.0
 pytest==3.6.1
 ../../tests/core/pyspec
+../../tests/core/config_helpers
diff --git a/scripts/build_spec.py b/scripts/build_spec.py
@@ -25,14 +25,14 @@
 
 from eth2spec.utils.ssz.ssz_impl import hash_tree_root
 from eth2spec.utils.ssz.ssz_typing import (
-    boolean, Container, List, Vector, uint64, SSZType,
+    View, boolean, Container, List, Vector, uint64,
     Bytes1, Bytes4, Bytes8, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector,
 )
 from eth2spec.utils import bls
 
 from eth2spec.utils.hash_function import hash
 
-SSZObject = TypeVar('SSZObject', bound=SSZType)
+SSZObject = TypeVar('SSZObject', bound=View)
 '''
 PHASE1_IMPORTS = '''from eth2spec.phase0 import spec as phase0
 from eth2spec.config.apply_config import apply_constants_preset
@@ -47,9 +47,8 @@
 
 from eth2spec.utils.ssz.ssz_impl import hash_tree_root
 from eth2spec.utils.ssz.ssz_typing import (
-    SSZType, Container, List, Vector, ByteList, ByteVector, Bitlist, Bitvector,
-    Bytes1, Bytes4, Bytes8, Bytes32, Bytes48, Bytes96,
-    uint64, uint8, bit, boolean,
+    View, boolean, Container, List, Vector, uint64, uint8, bit,
+    ByteVector, ByteList, Bytes1, Bytes4, Bytes8, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector,
 )
 from eth2spec.utils import bls
 
@@ -58,7 +57,7 @@
 
 SSZVariableName = str
 GeneralizedIndex = NewType('GeneralizedIndex', int)
-SSZObject = TypeVar('SSZObject', bound=SSZType)
+SSZObject = TypeVar('SSZObject', bound=View)
 '''
 SUNDRY_CONSTANTS_FUNCTIONS = '''
 def ceillog2(x: uint64) -> int:
@@ -80,27 +79,40 @@ def hash(x: bytes) -> Bytes32:  # type: ignore
     return hash_cache[x]
 
 
-# Monkey patch validator compute committee code
-_compute_committee = compute_committee
-committee_cache: Dict[Tuple[Bytes32, Bytes32, int, int], Sequence[ValidatorIndex]] = {}
+def cache_this(key_fn, value_fn):  # type: ignore
+    cache_dict = {}  # type: ignore
 
+    def wrapper(*args, **kw):  # type: ignore
+        key = key_fn(*args, **kw)
+        nonlocal cache_dict
+        if key not in cache_dict:
+            cache_dict[key] = value_fn(*args, **kw)
+        return cache_dict[key]
+    return wrapper
 
-def compute_committee(indices: Sequence[ValidatorIndex],  # type: ignore
-                      seed: Bytes32,
-                      index: int,
-                      count: int) -> Sequence[ValidatorIndex]:
-    param_hash = (hash(b''.join(index.to_bytes(length=4, byteorder='little') for index in indices)), seed, index, count)
 
-    if param_hash not in committee_cache:
-        committee_cache[param_hash] = _compute_committee(indices, seed, index, count)
-    return committee_cache[param_hash]'''
+get_base_reward = cache_this(
+    lambda state, index: (state.validators.hash_tree_root(), state.slot),
+    get_base_reward)
+
+get_committee_count_at_slot = cache_this(
+    lambda state, epoch: (state.validators.hash_tree_root(), epoch),
+    get_committee_count_at_slot)
+
+get_active_validator_indices = cache_this(
+    lambda state, epoch: (state.validators.hash_tree_root(), epoch),
+    get_active_validator_indices)
+
+get_beacon_committee = cache_this(
+    lambda state, slot, index: (state.validators.hash_tree_root(), state.randao_mixes.hash_tree_root(), slot, index),
+    get_beacon_committee)'''
 
 
 def objects_to_spec(functions: Dict[str, str],
                     custom_types: Dict[str, str],
                     constants: Dict[str, str],
                     ssz_objects: Dict[str, str],
-                    imports: Dict[str, str],
+                    imports: str,
                     version: str,
                     ) -> str:
     """
@@ -201,7 +213,7 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject:
     custom_types = combine_constants(custom_types0, custom_types1)
     constants = combine_constants(constants0, constants1)
     ssz_objects = combine_ssz_objects(ssz_objects0, ssz_objects1, custom_types)
-    return functions, custom_types, constants, ssz_objects
+    return SpecObject((functions, custom_types, constants, ssz_objects))
 
 
 def dependency_order_spec(objs: SpecObject):

diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md
@@ -1,81 +1,73 @@
-<!-- START doctoc generated TOC please keep comment here to allow auto update -->
-<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
-**Table of Contents**  *generated with [DocToc](https://github.com/thlorenz/doctoc)*
-
-- [Ethereum 2.0 Phase 1 -- The Beacon Chain for Shards](#ethereum-20-phase-1----the-beacon-chain-for-shards)
-  - [Table of contents](#table-of-contents)
-  - [Introduction](#introduction)
-  - [Custom types](#custom-types)
-  - [Configuration](#configuration)
-    - [Misc](#misc)
-  - [Updated containers](#updated-containers)
-    - [Extended `AttestationData`](#extended-attestationdata)
-    - [Extended `Attestation`](#extended-attestation)
-    - [Extended `PendingAttestation`](#extended-pendingattestation)
-    - [`IndexedAttestation`](#indexedattestation)
-      - [Extended `AttesterSlashing`](#extended-attesterslashing)
-    - [Extended `Validator`](#extended-validator)
-    - [Extended `BeaconBlockBody`](#extended-beaconblockbody)
-    - [Extended `BeaconBlock`](#extended-beaconblock)
-      - [Extended `SignedBeaconBlock`](#extended-signedbeaconblock)
-    - [Extended `BeaconState`](#extended-beaconstate)
-  - [New containers](#new-containers)
-    - [`ShardBlockWrapper`](#shardblockwrapper)
-    - [`ShardSignableHeader`](#shardsignableheader)
-    - [`ShardState`](#shardstate)
-    - [`ShardTransition`](#shardtransition)
-    - [`CompactCommittee`](#compactcommittee)
-    - [`AttestationCustodyBitWrapper`](#attestationcustodybitwrapper)
-  - [Helper functions](#helper-functions)
-    - [Misc](#misc-1)
-      - [`get_previous_slot`](#get_previous_slot)
-      - [`pack_compact_validator`](#pack_compact_validator)
-      - [`committee_to_compact_committee`](#committee_to_compact_committee)
-      - [`chunks_to_body_root`](#chunks_to_body_root)
-      - [`compute_shard_from_committee_index`](#compute_shard_from_committee_index)
-    - [Beacon state accessors](#beacon-state-accessors)
-      - [`get_active_shard_count`](#get_active_shard_count)
-      - [`get_online_validator_indices`](#get_online_validator_indices)
-      - [`get_shard_committee`](#get_shard_committee)
-      - [`get_shard_proposer_index`](#get_shard_proposer_index)
-      - [`get_light_client_committee`](#get_light_client_committee)
-      - [`get_indexed_attestation`](#get_indexed_attestation)
-      - [`get_updated_gasprice`](#get_updated_gasprice)
-      - [`get_start_shard`](#get_start_shard)
-      - [`get_shard`](#get_shard)
-      - [`get_next_slot_for_shard`](#get_next_slot_for_shard)
-      - [`get_offset_slots`](#get_offset_slots)
-    - [Predicates](#predicates)
-      - [Updated `is_valid_indexed_attestation`](#updated-is_valid_indexed_attestation)
-    - [Block processing](#block-processing)
-      - [Operations](#operations)
-        - [New Attestation processing](#new-attestation-processing)
-          - [`validate_attestation`](#validate_attestation)
-          - [`apply_shard_transition`](#apply_shard_transition)
-          - [`process_crosslink_for_shard`](#process_crosslink_for_shard)
-          - [`process_crosslinks`](#process_crosslinks)
-          - [`process_attestations`](#process_attestations)
-        - [New Attester slashing processing](#new-attester-slashing-processing)
-      - [Shard transition false positives](#shard-transition-false-positives)
-      - [Light client processing](#light-client-processing)
-    - [Epoch transition](#epoch-transition)
-      - [Custody game updates](#custody-game-updates)
-      - [Online-tracking](#online-tracking)
-      - [Light client committee updates](#light-client-committee-updates)
-
-<!-- END doctoc generated TOC please keep comment here to allow auto update -->
-
 # Ethereum 2.0 Phase 1 -- The Beacon Chain for Shards
 
 **Notice**: This document is a work-in-progress for researchers and implementers.
 
 ## Table of contents
 
-<!-- TOC -->
-
-TODO
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+**Table of Contents**
+
+- [Introduction](#introduction)
+- [Custom types](#custom-types)
+- [Configuration](#configuration)
+  - [Misc](#misc)
+- [Updated containers](#updated-containers)
+  - [Extended `AttestationData`](#extended-attestationdata)
+  - [Extended `Attestation`](#extended-attestation)
+  - [Extended `PendingAttestation`](#extended-pendingattestation)
+  - [`IndexedAttestation`](#indexedattestation)
+    - [Extended `AttesterSlashing`](#extended-attesterslashing)
+  - [Extended `Validator`](#extended-validator)
+  - [Extended `BeaconBlockBody`](#extended-beaconblockbody)
+  - [Extended `BeaconBlock`](#extended-beaconblock)
+    - [Extended `SignedBeaconBlock`](#extended-signedbeaconblock)
+  - [Extended `BeaconState`](#extended-beaconstate)
+- [New containers](#new-containers)
+  - [`ShardBlockWrapper`](#shardblockwrapper)
+  - [`ShardSignableHeader`](#shardsignableheader)
+  - [`ShardState`](#shardstate)
+  - [`ShardTransition`](#shardtransition)
+  - [`CompactCommittee`](#compactcommittee)
+  - [`AttestationCustodyBitWrapper`](#attestationcustodybitwrapper)
+- [Helper functions](#helper-functions)
+  - [Misc](#misc-1)
+    - [`get_previous_slot`](#get_previous_slot)
+    - [`pack_compact_validator`](#pack_compact_validator)
+    - [`committee_to_compact_committee`](#committee_to_compact_committee)
+    - [`chunks_to_body_root`](#chunks_to_body_root)
+    - [`compute_shard_from_committee_index`](#compute_shard_from_committee_index)
+  - [Beacon state accessors](#beacon-state-accessors)
+    - [`get_active_shard_count`](#get_active_shard_count)
+    - [`get_online_validator_indices`](#get_online_validator_indices)
+    - [`get_shard_committee`](#get_shard_committee)
+    - [`get_shard_proposer_index`](#get_shard_proposer_index)
+    - [`get_light_client_committee`](#get_light_client_committee)
+    - [`get_indexed_attestation`](#get_indexed_attestation)
+    - [`get_updated_gasprice`](#get_updated_gasprice)
+    - [`get_start_shard`](#get_start_shard)
+    - [`get_shard`](#get_shard)
+    - [`get_next_slot_for_shard`](#get_next_slot_for_shard)
+    - [`get_offset_slots`](#get_offset_slots)
+  - [Predicates](#predicates)
+    - [Updated `is_valid_indexed_attestation`](#updated-is_valid_indexed_attestation)
+  - [Block processing](#block-processing)
+    - [Operations](#operations)
+      - [New Attestation processing](#new-attestation-processing)
+        - [`validate_attestation`](#validate_attestation)
+        - [`apply_shard_transition`](#apply_shard_transition)
+        - [`process_crosslink_for_shard`](#process_crosslink_for_shard)
+        - [`process_crosslinks`](#process_crosslinks)
+        - [`process_attestations`](#process_attestations)
+      - [New Attester slashing processing](#new-attester-slashing-processing)
+    - [Shard transition false positives](#shard-transition-false-positives)
+    - [Light client processing](#light-client-processing)
+  - [Epoch transition](#epoch-transition)
+    - [Custody game updates](#custody-game-updates)
+    - [Online-tracking](#online-tracking)
+    - [Light client committee updates](#light-client-committee-updates)
 
-<!-- /TOC -->
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
 
 ## Introduction
 

diff --git a/tests/core/pyspec/eth2spec/debug/decode.py b/tests/core/pyspec/eth2spec/debug/decode.py
@@ -1,21 +1,23 @@
 from typing import Any
 from eth2spec.utils.ssz.ssz_impl import hash_tree_root
 from eth2spec.utils.ssz.ssz_typing import (
-    SSZType, SSZValue, uint, Container, ByteList, List, boolean,
-    Vector, ByteVector
+    uint, Container, List, boolean,
+    Vector, ByteVector, ByteList
 )
 
 
-def decode(data: Any, typ: SSZType) -> SSZValue:
+def decode(data: Any, typ):
     if issubclass(typ, (uint, boolean)):
         return typ(data)
     elif issubclass(typ, (List, Vector)):
-        return typ(decode(element, typ.elem_type) for element in data)
-    elif issubclass(typ, (ByteList, ByteVector)):
+        return typ(decode(element, typ.element_cls()) for element in data)
+    elif issubclass(typ, ByteVector):
+        return typ(bytes.fromhex(data[2:]))
+    elif issubclass(typ, ByteList):
         return typ(bytes.fromhex(data[2:]))
     elif issubclass(typ, Container):
         temp = {}
-        for field_name, field_type in typ.get_fields().items():
+        for field_name, field_type in typ.fields().items():
             temp[field_name] = decode(data[field_name], field_type)
             if field_name + "_hash_tree_root" in data:
                 assert (data[field_name + "_hash_tree_root"][2:] ==

diff --git a/tests/core/pyspec/eth2spec/debug/encode.py b/tests/core/pyspec/eth2spec/debug/encode.py
@@ -1,27 +1,30 @@
 from eth2spec.utils.ssz.ssz_impl import hash_tree_root, serialize
 from eth2spec.utils.ssz.ssz_typing import (
     uint, boolean,
-    Bitlist, Bitvector, Container
+    Bitlist, Bitvector, Container, Vector, List
 )
 
 
 def encode(value, include_hash_tree_roots=False):
     if isinstance(value, uint):
         # Larger uints are boxed and the class declares their byte length
-        if value.type().byte_len > 8:
+        if value.__class__.type_byte_length() > 8:
             return str(int(value))
         return int(value)
     elif isinstance(value, boolean):
         return value == 1
     elif isinstance(value, (Bitlist, Bitvector)):
         return '0x' + serialize(value).hex()
-    elif isinstance(value, list):  # normal python lists, ssz-List, Vector
+    elif isinstance(value, list):  # normal python lists
         return [encode(element, include_hash_tree_roots) for element in value]
-    elif isinstance(value, bytes):  # both bytes and ByteVector
+    elif isinstance(value, (List, Vector)):
+        return [encode(element, include_hash_tree_roots) for element in value]
+    elif isinstance(value, bytes):  # bytes, ByteList, ByteVector
         return '0x' + value.hex()
     elif isinstance(value, Container):
         ret = {}
-        for field_value, field_name in zip(value, value.get_fields().keys()):
+        for field_name in value.fields().keys():
+            field_value = getattr(value, field_name)
             ret[field_name] = encode(field_value, include_hash_tree_roots)
             if include_hash_tree_roots:
                 ret[field_name + "_hash_tree_root"] = '0x' + hash_tree_root(field_value).hex()