Add .clvm.hex pre-commit check (Chia-Network#12050)

* Add .clvm.hex pre-commit check

* report all results

* stuff

* maybe pass

* add todo

* explicitly define suffixes

* check for (and remove) extra hex and hash files

* remove garbage failure hex and hash files

* provide runner scripts

* matrix pre-commit checks

* maybe now

* more cross platform

* move it

* oops

* shell:

* maybe

* more configurable

* explicitly specify powershell

* tidy

* remove other stuff

* tidy

* remove existing pytest clvm hex etc checker

* add a trailing newline to decompress_block_spends.clvm.hex.sha256tree

* Update activated.ps1

* git config --global core.autocrlf false

* Update pre-commit.yml

* only check `chia/`

* typing_extensions

* typing_extensions

* removing unused typing import

* move top_level out of the rglob

* catchup dl clvm hex files

* Revert "catchup dl clvm hex files"

This reverts commit cd3d4f7.

* now

.github/workflows/pre-commit.yml

@@ -54,6 +54,10 @@ jobs:
 
     - uses: Chia-Network/actions/git-mark-workspace-safe@main
 
+    - name: disable git autocrlf
+      run: |
+        git config --global core.autocrlf false
+
     - uses: actions/checkout@v3
 
     - uses: Chia-Network/actions/setup-python@main

.pre-commit-config.yaml

@@ -29,6 +29,13 @@ repos:
     -   id: check-merge-conflict
     -   id: check-ast
     -   id: debug-statements
+-   repo: local
+    hooks:
+    -   id: clvm_hex
+        name: .clvm.hex files
+        entry: ./activated.py python tests/check_clvm.py
+        language: python
+        pass_filenames: false
 -   repo: local
     hooks:
     -   id: mypy

activated.ps1

@@ -2,7 +2,11 @@ $ErrorActionPreference = "Stop"
 
 $script_directory = Split-Path $MyInvocation.MyCommand.Path -Parent
 
+$command = $args[0]
+$parameters = [System.Collections.ArrayList]$args
+$parameters.RemoveAt(0)
+
 & $script_directory/venv/Scripts/Activate.ps1
-& @args
+& $command @parameters
 
 exit $LASTEXITCODE

chia/wallet/puzzles/decompress_block_spends.clvm.hex.sha256tree

@@ -1 +1 @@
-f890a7866079009517ae0b652d530268d2531bbac99670aaba461d604bc0ff0c
+f890a7866079009517ae0b652d530268d2531bbac99670aaba461d604bc0ff0c

chia/wallet/puzzles/everything_with_signature.clvm.hex.sha256tree

@@ -1 +1 @@
-1720d13250a7c16988eaf530331cefa9dd57a76b2c82236bec8bbbff91499b89
+1720d13250a7c16988eaf530331cefa9dd57a76b2c82236bec8bbbff91499b89

chia/wallet/puzzles/genesis_by_coin_id.clvm.hex.sha256tree

@@ -1 +1 @@
-493afb89eed93ab86741b2aa61b8f5de495d33ff9b781dfc8919e602b2afa150
+493afb89eed93ab86741b2aa61b8f5de495d33ff9b781dfc8919e602b2afa150

chia/wallet/puzzles/genesis_by_puzzle_hash.clvm.hex.sha256tree

@@ -1 +1 @@
-de5a6e06d41518be97ff6365694f4f89475dda773dede267caa33da63b434e36
+de5a6e06d41518be97ff6365694f4f89475dda773dede267caa33da63b434e36

chia/wallet/puzzles/graftroot_dl_offers.clvm.hex.sha256tree

@@ -1 +1 @@
-0893e36a88c064fddfa6f8abdb42c044584a98cb4273b80cccc83b4867b701a1
+0893e36a88c064fddfa6f8abdb42c044584a98cb4273b80cccc83b4867b701a1

chia/wallet/puzzles/nft_metadata_updater_default.clvm.hex

@@ -1 +1 @@
-ff02ffff01ff04ffff04ffff02ffff03ffff22ff27ff3780ffff01ff02ffff03ffff21ffff09ff27ffff01826d7580ffff09ff27ffff01826c7580ffff09ff27ffff01758080ffff01ff02ff02ffff04ff02ffff04ff05ffff04ff27ffff04ff37ff808080808080ffff010580ff0180ffff010580ff0180ffff04ff0bff808080ffff01ff808080ffff04ffff01ff02ffff03ff05ffff01ff02ffff03ffff09ff11ff0b80ffff01ff04ffff04ff0bffff04ff17ff198080ff0d80ffff01ff04ff09ffff02ff02ffff04ff02ffff04ff0dffff04ff0bffff04ff17ff8080808080808080ff0180ff8080ff0180ff018080
+ff02ffff01ff04ffff04ffff02ffff03ffff22ff27ff3780ffff01ff02ffff03ffff21ffff09ff27ffff01826d7580ffff09ff27ffff01826c7580ffff09ff27ffff01758080ffff01ff02ff02ffff04ff02ffff04ff05ffff04ff27ffff04ff37ff808080808080ffff010580ff0180ffff010580ff0180ffff04ff0bff808080ffff01ff808080ffff04ffff01ff02ffff03ff05ffff01ff02ffff03ffff09ff11ff0b80ffff01ff04ffff04ff0bffff04ff17ff198080ff0d80ffff01ff04ff09ffff02ff02ffff04ff02ffff04ff0dffff04ff0bffff04ff17ff8080808080808080ff0180ff8080ff0180ff018080

chia/wallet/puzzles/nft_metadata_updater_updateable.clvm.hex

@@ -1 +1 @@
-ff02ffff01ff04ffff04ffff02ffff03ff27ffff01ff02ff02ffff04ff02ffff04ff05ffff04ff27ff8080808080ffff010580ff0180ffff04ffff02ffff03ffff09ffff0dff5780ffff012080ffff0157ffff010b80ff0180ff808080ffff01ff808080ffff04ffff01ff02ffff03ff05ffff01ff02ffff03ffff09ff11ffff017580ffff01ff04ffff04ffff0175ffff04ff0bff198080ff0d80ffff01ff04ff09ffff02ff02ffff04ff02ffff04ff0dffff04ff0bff80808080808080ff0180ff8080ff0180ff018080
+ff02ffff01ff04ffff04ffff02ffff03ff27ffff01ff02ff02ffff04ff02ffff04ff05ffff04ff27ff8080808080ffff010580ff0180ffff04ffff02ffff03ffff09ffff0dff5780ffff012080ffff0157ffff010b80ff0180ff808080ffff01ff808080ffff04ffff01ff02ffff03ff05ffff01ff02ffff03ffff09ff11ffff017580ffff01ff04ffff04ffff0175ffff04ff0bff198080ff0d80ffff01ff04ff09ffff02ff02ffff04ff02ffff04ff0dffff04ff0bff80808080808080ff0180ff8080ff0180ff018080

chia/wallet/puzzles/nft_ownership_layer.clvm.hex.sha256tree

@@ -1 +1 @@
-c5abea79afaa001b5427dfa0c8cf42ca6f38f5841b78f9b3c252733eb2de2726
+c5abea79afaa001b5427dfa0c8cf42ca6f38f5841b78f9b3c252733eb2de2726

chia/wallet/puzzles/p2_delegated_conditions.clvm.hex

@@ -1 +1 @@
-ff02ffff01ff04ffff04ff04ffff04ff05ffff04ffff02ff06ffff04ff02ffff04ff0bff80808080ff80808080ff0b80ffff04ffff01ff32ff02ffff03ffff07ff0580ffff01ff0bffff0102ffff02ff06ffff04ff02ffff04ff09ff80808080ffff02ff06ffff04ff02ffff04ff0dff8080808080ffff01ff0bffff0101ff058080ff0180ff018080
+ff02ffff01ff04ffff04ff04ffff04ff05ffff04ffff02ff06ffff04ff02ffff04ff0bff80808080ff80808080ff0b80ffff04ffff01ff32ff02ffff03ffff07ff0580ffff01ff0bffff0102ffff02ff06ffff04ff02ffff04ff09ff80808080ffff02ff06ffff04ff02ffff04ff0dff8080808080ffff01ff0bffff0101ff058080ff0180ff018080

chia/wallet/puzzles/p2_parent.clvm.hex.sha256tree

@@ -1 +1 @@
-b10ce2d0b18dcf8c21ddfaf55d9b9f0adcbf1e0beb55b1a8b9cad9bbff4e5f22
+b10ce2d0b18dcf8c21ddfaf55d9b9f0adcbf1e0beb55b1a8b9cad9bbff4e5f22

chia/wallet/puzzles/rl.clvm.hex

@@ -1 +1 @@
-ff02ffff01ff02ffff03ffff09ff81bfff2480ffff01ff04ffff04ff30ffff04ff5fffff04ffff02ff3effff04ff02ffff04ffff04ff81bfff81ff80ff80808080ff80808080ff81ff80ffff01ff04ffff04ff30ffff04ff05ffff04ffff02ff3effff04ff02ffff04ffff04ff81bfff81ff80ff80808080ff80808080ffff02ffff03ffff09ff81bfff3c80ffff01ff02ff2effff04ff02ffff04ff0bffff04ff17ffff04ff2fffff04ff81ffff80808080808080ffff01ff02ff22ffff04ff02ffff04ff2fffff04ff81ffff808080808080ff01808080ff0180ffff04ffff01ffffffffff02ffff03ffff15ff05ff0b80ffff01ff0101ffff01ff02ffff03ffff09ff05ff0b80ffff01ff0101ff8080ff018080ff018031ff5246ffff0333ff3c01ffffffff02ffff03ffff02ffff03ffff09ffff0bff820bfbff13ff8205fb80ff82017b80ffff01ff0101ffff01ff02ffff03ffff09ff05ff82017b80ffff01ff0101ff8080ff018080ff0180ffff01ff04ffff02ff26ffff04ff02ffff04ff82017bffff04ff13ffff04ff8202fbff808080808080ffff04ffff02ff2affff04ff02ffff04ff2bffff04ff5bffff04ff81bbff808080808080ffff04ffff02ff3affff04ff02ffff04ff13ffff04ffff10ff81bbff8202fb80ff8080808080ff80808080ffff01ffff08808080ff0180ff04ff34ffff04ff05ffff04ffff11ff0bffff10ff17ff2f8080ff80808080ffff04ff2cffff04ffff0bff05ff0bff1780ff808080ff04ff34ffff04ff05ffff04ff0bff80808080ffffff04ff38ffff04ffff0bff05ff0bff1780ff808080ff02ffff03ffff02ff20ffff04ff02ffff04ffff12ff05ff1780ffff04ffff12ff0bff2f80ff8080808080ffff01ff04ff28ffff04ff05ff808080ffff01ffff08808080ff0180ffff02ffff03ffff02ffff03ffff09ffff0bff8217efff81afff822fef80ff4f80ffff01ff0101ffff01ff02ffff03ffff09ff17ff4f80ffff01ff0101ff8080ff018080ff0180ffff01ff04ffff02ff36ffff04ff02ffff04ff820befffff04ff8205efffff04ff05ffff04ff0bff80808080808080ffff04ffff02ff32ffff04ff02ffff04ff81afffff04ff82016fffff04ff8205efffff04ff825fefff80808080808080ffff04ffff02ff26ffff04ff02ffff04ff4fffff04ff81afffff04ff82016fff808080808080ffff04ffff02ff3affff04ff02ffff04ff8202efffff04ff8205efff8080808080ff8080808080ffff01ffff08808080ff0180ff02ffff03ffff07ff0580ffff01ff0bffff0102ffff02ff3effff04ff02ffff04ff09ff80808080ffff02ff3effff04ff02ffff04ff0dff8080808080ffff01ff0bffff0101ff058080ff0180ff018080
+ff02ffff01ff02ffff03ffff09ff81bfff2480ffff01ff04ffff04ff30ffff04ff5fffff04ffff02ff3effff04ff02ffff04ffff04ff81bfff81ff80ff80808080ff80808080ff81ff80ffff01ff04ffff04ff30ffff04ff05ffff04ffff02ff3effff04ff02ffff04ffff04ff81bfff81ff80ff80808080ff80808080ffff02ffff03ffff09ff81bfff3c80ffff01ff02ff2effff04ff02ffff04ff0bffff04ff17ffff04ff2fffff04ff81ffff80808080808080ffff01ff02ff22ffff04ff02ffff04ff2fffff04ff81ffff808080808080ff01808080ff0180ffff04ffff01ffffffffff02ffff03ffff15ff05ff0b80ffff01ff0101ffff01ff02ffff03ffff09ff05ff0b80ffff01ff0101ff8080ff018080ff018031ff5246ffff0333ff3c01ffffffff02ffff03ffff02ffff03ffff09ffff0bff820bfbff13ff8205fb80ff82017b80ffff01ff0101ffff01ff02ffff03ffff09ff05ff82017b80ffff01ff0101ff8080ff018080ff0180ffff01ff04ffff02ff26ffff04ff02ffff04ff82017bffff04ff13ffff04ff8202fbff808080808080ffff04ffff02ff2affff04ff02ffff04ff2bffff04ff5bffff04ff81bbff808080808080ffff04ffff02ff3affff04ff02ffff04ff13ffff04ffff10ff81bbff8202fb80ff8080808080ff80808080ffff01ffff08808080ff0180ff04ff34ffff04ff05ffff04ffff11ff0bffff10ff17ff2f8080ff80808080ffff04ff2cffff04ffff0bff05ff0bff1780ff808080ff04ff34ffff04ff05ffff04ff0bff80808080ffffff04ff38ffff04ffff0bff05ff0bff1780ff808080ff02ffff03ffff02ff20ffff04ff02ffff04ffff12ff05ff1780ffff04ffff12ff0bff2f80ff8080808080ffff01ff04ff28ffff04ff05ff808080ffff01ffff08808080ff0180ffff02ffff03ffff02ffff03ffff09ffff0bff8217efff81afff822fef80ff4f80ffff01ff0101ffff01ff02ffff03ffff09ff17ff4f80ffff01ff0101ff8080ff018080ff0180ffff01ff04ffff02ff36ffff04ff02ffff04ff820befffff04ff8205efffff04ff05ffff04ff0bff80808080808080ffff04ffff02ff32ffff04ff02ffff04ff81afffff04ff82016fffff04ff8205efffff04ff825fefff80808080808080ffff04ffff02ff26ffff04ff02ffff04ff4fffff04ff81afffff04ff82016fff808080808080ffff04ffff02ff3affff04ff02ffff04ff8202efffff04ff8205efff8080808080ff8080808080ffff01ffff08808080ff0180ff02ffff03ffff07ff0580ffff01ff0bffff0102ffff02ff3effff04ff02ffff04ff09ff80808080ffff02ff3effff04ff02ffff04ff0dff8080808080ffff01ff0bffff0101ff058080ff0180ff018080

tests/check_clvm.py

@@ -0,0 +1,167 @@
+from __future__ import annotations
+
+import dataclasses
+import os
+import pathlib
+import sys
+import tempfile
+import traceback
+
+import typing_extensions
+
+here = pathlib.Path(__file__).parent.resolve()
+root = here.parent
+
+# This is a work-around for fixing imports so they get the appropriate top level
+# packages instead of those of the same name in the same directory as this program.
+# This undoes the Python mis-feature meant to support 'scripts' that have not been
+# installed by adding the script's directory to the import search path.  This is why
+# it is simpler to just have all code get installed and all things you run be
+# accessible via entry points.
+sys.path = [path for path in sys.path if path != os.fspath(here)]
+
+from clvm_tools_rs import compile_clvm  # noqa: E402
+
+from chia.types.blockchain_format.program import SerializedProgram  # noqa: E402
+
+clvm_suffix = ".clvm"
+hex_suffix = ".clvm.hex"
+hash_suffix = ".clvm.hex.sha256tree"
+
+
+def generate_hash_bytes(hex_bytes: bytes) -> bytes:
+    cleaned_blob = bytes.fromhex(hex_bytes.decode("utf-8"))
+    serialize_program = SerializedProgram.from_bytes(cleaned_blob)
+    result = serialize_program.get_tree_hash().hex()
+    return (result + "\n").encode("utf-8")
+
+
+@typing_extensions.final
+@dataclasses.dataclass(frozen=True)
+class ClvmPaths:
+    clvm: pathlib.Path
+    hex: pathlib.Path
+    hash: pathlib.Path
+
+    @classmethod
+    def from_clvm(cls, clvm: pathlib.Path) -> ClvmPaths:
+        return cls(
+            clvm=clvm,
+            hex=clvm.with_name(clvm.name[: -len(clvm_suffix)] + hex_suffix),
+            hash=clvm.with_name(clvm.name[: -len(clvm_suffix)] + hash_suffix),
+        )
+
+
+@typing_extensions.final
+@dataclasses.dataclass(frozen=True)
+class ClvmBytes:
+    hex: bytes
+    hash: bytes
+
+    @classmethod
+    def from_clvm_paths(cls, paths: ClvmPaths) -> ClvmBytes:
+        return cls(
+            hex=paths.hex.read_bytes(),
+            hash=paths.hash.read_bytes(),
+        )
+
+    @classmethod
+    def from_hex_bytes(cls, hex_bytes: bytes) -> ClvmBytes:
+        return cls(
+            hex=hex_bytes,
+            hash=generate_hash_bytes(hex_bytes=hex_bytes),
+        )
+
+
+# These files have the wrong extension for now so we'll just manually exclude them
+excludes = {"condition_codes.clvm", "create-lock-puzzlehash.clvm"}
+
+
+def main() -> int:
+    used_excludes = set()
+    overall_fail = False
+
+    suffixes = {"clvm": clvm_suffix, "hex": hex_suffix, "hash": hash_suffix}
+    top_levels = {"chia"}
+    found_stems = {
+        name: {
+            path.with_name(path.name[: -len(suffix)])
+            for top_level in top_levels
+            for path in root.joinpath(top_level).rglob(f"**/*{suffix}")
+        }
+        for name, suffix in suffixes.items()
+    }
+    for name in ["hex", "hash"]:
+        found = found_stems[name]
+        suffix = suffixes[name]
+        extra = found - found_stems["clvm"]
+
+        print()
+        print(f"Extra {suffix} files:")
+
+        if len(extra) == 0:
+            print("    -")
+        else:
+            overall_fail = True
+            for stem in extra:
+                print(f"    {stem.with_name(stem.name + suffix)}")
+
+    print()
+    print("Checking that all existing .clvm files compile to .clvm.hex that match existing caches:")
+    for stem_path in sorted(found_stems["clvm"]):
+        clvm_path = stem_path.with_name(stem_path.name + clvm_suffix)
+        if clvm_path.name in excludes:
+            used_excludes.add(clvm_path.name)
+            continue
+
+        file_fail = False
+        error = None
+
+        try:
+            reference_paths = ClvmPaths.from_clvm(clvm=clvm_path)
+            reference_bytes = ClvmBytes.from_clvm_paths(paths=reference_paths)
+
+            with tempfile.TemporaryDirectory() as temporary_directory:
+                generated_paths = ClvmPaths.from_clvm(
+                    clvm=pathlib.Path(temporary_directory).joinpath(f"generated{clvm_suffix}")
+                )
+
+                compile_clvm(
+                    input_path=os.fspath(reference_paths.clvm),
+                    output_path=os.fspath(generated_paths.hex),
+                    search_paths=[os.fspath(reference_paths.clvm.parent)],
+                )
+
+                generated_bytes = ClvmBytes.from_hex_bytes(hex_bytes=generated_paths.hex.read_bytes())
+
+            if generated_bytes != reference_bytes:
+                file_fail = True
+                error = f"        reference: {reference_bytes!r}\n"
+                error += f"        generated: {generated_bytes!r}"
+        except Exception:
+            file_fail = True
+            error = traceback.format_exc()
+
+        if file_fail:
+            print(f"FAIL    : {clvm_path}")
+            if error is not None:
+                print(error)
+        else:
+            print(f"    pass: {clvm_path}")
+
+        if file_fail:
+            overall_fail = True
+
+    unused_excludes = sorted(excludes - used_excludes)
+    if len(unused_excludes) > 0:
+        overall_fail = True
+        print()
+        print("Unused excludes:")
+
+        for exclude in unused_excludes:
+            print(f"    {exclude}")
+
+    return 1 if overall_fail else 0
+
+
+sys.exit(main())