Add types to support Generator Back References (Chia-Network#1901)

* Add types to support Generator Back References

* Add test workflows

Co-authored-by: Adam Kelly <[email protected]>

.github/workflows/build-test-macos-generator.yml

@@ -0,0 +1,93 @@
+name: MacOS generator Tests
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+        - '**'
+  pull_request:
+    branches:
+      - '**'
+
+jobs:
+  build:
+    name: MacOS generator Tests
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      max-parallel: 4
+      matrix:
+        python-version: [3.8, 3.9]
+        os: [macOS-latest]
+
+    steps:
+    - name: Cancel previous runs on the same branch
+      if: ${{ github.ref != 'refs/heads/main' }}
+      uses: styfle/[email protected]
+      with:
+        access_token: ${{ github.token }}
+
+    - name: Checkout Code
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Setup Python environment
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Get pip cache dir
+      id: pip-cache
+      run: |
+        echo "::set-output name=dir::$(pip cache dir)"
+
+    - name: Cache pip
+      uses: actions/[email protected]
+      with:
+        # Note that new runners may break this https://github.com/actions/cache/issues/292
+        path: ${{ steps.pip-cache.outputs.dir }}
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+
+    - name: Checkout test blocks and plots
+      uses: actions/checkout@v2
+      with:
+        repository: 'Chia-Network/test-cache'
+        path: '.chia'
+        ref: '0.26.0'
+        fetch-depth: 1
+
+    - name: Link home directory
+      run: |
+        cd $HOME
+        ln -s $GITHUB_WORKSPACE/.chia
+        echo "$HOME/.chia"
+        ls -al $HOME/.chia
+
+    - name: Run install script
+      env:
+        INSTALL_PYTHON_VERSION: ${{ matrix.python-version }}
+        BUILD_VDF_CLIENT: "N"
+      run: |
+        brew install boost
+        sh install.sh
+
+    - name: Install timelord
+      run: |
+        . ./activate
+        sh install-timelord.sh
+        ./vdf_bench square_asm 400000
+
+    - name: Install developer requirements
+      run: |
+        . ./activate
+        venv/bin/python -m pip install pytest pytest-asyncio pytest-xdist
+
+    - name: Test generator code with pytest
+      run: |
+        . ./activate
+        ./venv/bin/py.test tests/generator/test_generator_types.py -s -v --durations 0 

.github/workflows/build-test-ubuntu-generator.yml

@@ -0,0 +1,105 @@
+name: Ubuntu generator Test
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+        - '**'
+  pull_request:
+    branches:
+      - '**'
+
+jobs:
+  build:
+    name: Ubuntu generator Test
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      max-parallel: 4
+      matrix:
+        python-version: [3.7, 3.8, 3.9]
+        os: [ubuntu-latest]
+
+    steps:
+    - name: Cancel previous runs on the same branch
+      if: ${{ github.ref != 'refs/heads/main' }}
+      uses: styfle/[email protected]
+      with:
+        access_token: ${{ github.token }}
+
+    - name: Checkout Code
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Setup Python environment
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Cache npm
+      uses: actions/[email protected]
+      with:
+        path: ~/.npm
+        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+        restore-keys: |
+          ${{ runner.os }}-node-
+
+    - name: Get pip cache dir
+      id: pip-cache
+      run: |
+        echo "::set-output name=dir::$(pip cache dir)"
+
+    - name: Cache pip
+      uses: actions/[email protected]
+      with:
+        path: ${{ steps.pip-cache.outputs.dir }}
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+
+    - name: Checkout test blocks and plots
+      uses: actions/checkout@v2
+      with:
+        repository: 'Chia-Network/test-cache'
+        path: '.chia'
+        ref: '0.26.0'
+        fetch-depth: 1
+
+    - name: Link home directory
+      run: |
+        cd $HOME
+        ln -s $GITHUB_WORKSPACE/.chia
+        echo "$HOME/.chia"
+        ls -al $HOME/.chia
+
+    - name: Install ubuntu dependencies
+      run: |
+        sudo apt-get install software-properties-common
+        sudo add-apt-repository ppa:deadsnakes/ppa
+        sudo apt-get update
+        sudo apt-get install python${{ matrix.python-version }}-venv python${{ matrix.python-version }}-distutils git -y
+
+    - name: Run install script
+      env:
+        INSTALL_PYTHON_VERSION: ${{ matrix.python-version }}
+      run: |
+        sh install.sh
+
+    - name: Install timelord
+      run: |
+        . ./activate
+        sh install-timelord.sh
+        ./vdf_bench square_asm 400000
+
+    - name: Install developer requirements
+      run: |
+        . ./activate
+        venv/bin/python -m pip install pytest pytest-asyncio pytest-xdist
+
+    - name: Test generator code with pytest
+      run: |
+        . ./activate
+        ./venv/bin/py.test tests/generator/test_generator_types.py -s -v --durations 0 

chia/full_node/generator.py

@@ -0,0 +1,32 @@
+import logging
+from typing import List, Optional
+from chia.types.blockchain_format.program import Program, SerializedProgram
+from chia.types.generator_types import BlockGenerator, GeneratorArg, GeneratorBlockCacheInterface
+from chia.util.ints import uint32
+from chia.wallet.puzzles.load_clvm import load_clvm
+
+DESERIALIZE_MOD = load_clvm("chialisp_deserialisation.clvm", package_or_requirement="chia.wallet.puzzles")
+
+log = logging.getLogger(__name__)
+
+
+def create_block_generator(
+    generator: SerializedProgram, block_heights_list: List[uint32], generator_block_cache: GeneratorBlockCacheInterface
+) -> Optional[BlockGenerator]:
+    """ `create_block_generator` will returns None if it fails to look up any referenced block """
+    generator_arg_list: List[GeneratorArg] = []
+    for i in block_heights_list:
+        previous_generator = generator_block_cache.get_generator_for_block_height(i)
+        if previous_generator is None:
+            log.error(f"Failed to look up generator for block {i}. Ref List: {block_heights_list}")
+            return None
+        generator_arg_list.append(GeneratorArg(i, previous_generator))
+    return BlockGenerator(generator, generator_arg_list)
+
+
+def make_generator_args(generator_ref_list: List[SerializedProgram]) -> SerializedProgram:
+    """
+    `make_generator_args`: The format and contents of these arguments affect consensus.
+    """
+    gen_ref_list = [Program.from_bytes(bytes(g)) for g in generator_ref_list]
+    return SerializedProgram.from_bytes(bytes(Program.to([DESERIALIZE_MOD, gen_ref_list])))

chia/types/generator_types.py

@@ -0,0 +1,31 @@
+from dataclasses import dataclass
+from typing import List, Tuple
+
+from chia.types.blockchain_format.program import SerializedProgram
+from chia.util.ints import uint32
+
+# from chia.util.streamable import Streamable, streamable
+
+
+class GeneratorBlockCacheInterface:
+    def get_generator_for_block_height(self, uint32) -> SerializedProgram:
+        # Requested block must be a transaction block
+        pass
+
+
+@dataclass(frozen=True)
+class GeneratorArg:
+    block_height: uint32
+    generator: SerializedProgram
+
+
+@dataclass(frozen=True)
+class BlockGenerator:
+    generator: SerializedProgram
+    generator_args: List[GeneratorArg]
+
+    def make_generator_args(self) -> SerializedProgram:
+        """ `make_generator_args` is consensus-critical """
+
+    def run(self) -> Tuple[int, SerializedProgram]:
+        pass

tests/generator/test_generator_types.py

@@ -0,0 +1,62 @@
+from typing import Dict
+from unittest import TestCase
+
+from chia.types.blockchain_format.program import Program, SerializedProgram
+from chia.types.generator_types import GeneratorBlockCacheInterface
+from chia.full_node.generator import create_block_generator, make_generator_args
+from chia.util.byte_types import hexstr_to_bytes
+from chia.util.ints import uint32
+
+
+gen0 = SerializedProgram.from_bytes(
+    hexstr_to_bytes(
+        "ff01ffffffa00000000000000000000000000000000000000000000000000000000000000000ff830186a080ffffff02ffff01ff02ffff01ff02ffff03ff0bffff01ff02ffff03ffff09ff05ffff1dff0bffff1effff0bff0bffff02ff06ffff04ff02ffff04ff17ff8080808080808080ffff01ff02ff17ff2f80ffff01ff088080ff0180ffff01ff04ffff04ff04ffff04ff05ffff04ffff02ff06ffff04ff02ffff04ff17ff80808080ff80808080ffff02ff17ff2f808080ff0180ffff04ffff01ff32ff02ffff03ffff07ff0580ffff01ff0bffff0102ffff02ff06ffff04ff02ffff04ff09ff80808080ffff02ff06ffff04ff02ffff04ff0dff8080808080ffff01ff0bffff0101ff058080ff0180ff018080ffff04ffff01b081963921826355dcb6c355ccf9c2637c18adf7d38ee44d803ea9ca41587e48c913d8d46896eb830aeadfc13144a8eac3ff018080ffff80ffff01ffff33ffa06b7a83babea1eec790c947db4464ab657dbe9b887fe9acc247062847b8c2a8a9ff830186a08080ff8080808080"  # noqa
+    )
+)
+
+gen1 = SerializedProgram.from_bytes(
+    hexstr_to_bytes(
+        "ff01ffffffa00000000000000000000000000000000000000000000000000000000000000000ff830186a080ffffff02ffff01ff02ffff01ff02ffff03ff0bffff01ff02ffff03ffff09ff05ffff1dff0bffff1effff0bff0bffff02ff06ffff04ff02ffff04ff17ff8080808080808080ffff01ff02ff17ff2f80ffff01ff088080ff0180ffff01ff04ffff04ff04ffff04ff05ffff04ffff02ff06ffff04ff02ffff04ff17ff80808080ff80808080ffff02ff17ff2f808080ff0180ffff04ffff01ff32ff02ffff03ffff07ff0580ffff01ff0bffff0102ffff02ff06ffff04ff02ffff04ff09ff80808080ffff02ff06ffff04ff02ffff04ff0dff8080808080ffff01ff0bffff0101ff058080ff0180ff018080ffff04ffff01b081963921826355dcb6c355ccf9c2637c18adf7d38ee44d803ea9ca41587e48c913d8d46896eb830aeadfc13144a8eac3ff018080ffff80ffff01ffff33ffa06b7a83babea1eec790c947db4464ab657dbe9b887fe9acc247062847b8c2a8a9ff830186a08080ff8080808080"  # noqa
+    )
+)
+
+gen2 = SerializedProgram.from_bytes(
+    hexstr_to_bytes(
+        "ff01ffffffa00000000000000000000000000000000000000000000000000000000000000000ff830186a080ffffff02ffff01ff02ffff01ff02ffff03ff0bffff01ff02ffff03ffff09ff05ffff1dff0bffff1effff0bff0bffff02ff06ffff04ff02ffff04ff17ff8080808080808080ffff01ff02ff17ff2f80ffff01ff088080ff0180ffff01ff04ffff04ff04ffff04ff05ffff04ffff02ff06ffff04ff02ffff04ff17ff80808080ff80808080ffff02ff17ff2f808080ff0180ffff04ffff01ff32ff02ffff03ffff07ff0580ffff01ff0bffff0102ffff02ff06ffff04ff02ffff04ff09ff80808080ffff02ff06ffff04ff02ffff04ff0dff8080808080ffff01ff0bffff0101ff058080ff0180ff018080ffff04ffff01b081963921826355dcb6c355ccf9c2637c18adf7d38ee44d803ea9ca41587e48c913d8d46896eb830aeadfc13144a8eac3ff018080ffff80ffff01ffff33ffa06b7a83babea1eec790c947db4464ab657dbe9b887fe9acc247062847b8c2a8a9ff830186a08080ff8080808080"  # noqa
+    )
+)
+
+
+class BlockDict(GeneratorBlockCacheInterface):
+    def __init__(self, d: Dict[uint32, SerializedProgram]):
+        self.d = d
+
+    def get_generator_for_block_height(self, index: uint32) -> SerializedProgram:
+        return self.d[index]
+
+
+class TestGeneratorTypes(TestCase):
+    def test_make_generator(self):
+        block_dict = BlockDict({1: gen1})
+        gen = create_block_generator(gen2, [1], block_dict)
+        print(gen)
+
+    def test_make_generator_args(self):
+        generator_ref_list = [gen1]
+        gen_args = make_generator_args(generator_ref_list)
+        gen_args_as_program = Program.from_bytes(bytes(gen_args))
+
+        d = gen_args_as_program.first()
+
+        # First arguemnt: clvm deserializer
+
+        b = hexstr_to_bytes("ff8568656c6c6fff86667269656e6480")  # ("hello" "friend")
+        cost, output = d.run_with_cost([b])
+        # print(cost, output)
+        out = Program.to(output)
+        assert out == Program.from_bytes(b)
+
+        # Second Argument
+        arg2 = gen_args_as_program.rest().first().first()
+        print(arg2)
+        assert bytes(arg2) == bytes(gen1)

tests/testconfig.py

@@ -1,6 +1,6 @@
 # Github actions template config
 oses = ["ubuntu", "macos"]
-root_test_dirs = ["blockchain", "clvm", "core", "simulation", "wallet"]
+root_test_dirs = ["blockchain", "clvm", "core", "generator", "simulation", "wallet"]
 
 # Defaults are conservative.
 parallel = False