From ac9dcfda873e9022d240cb0a6ebd90ccdd16440d Mon Sep 17 00:00:00 2001 From: Lucas Clemente Vella Date: Thu, 18 Apr 2024 10:16:09 +0100 Subject: [PATCH] zkEVM prover backend (#1211) Similar to #1193, but in here I am just interested in having it working end-to-end, at least for a few cases, so that everybody can try it and build upon. --------- Co-authored-by: Leo --- .github/workflows/build-cache.yml | 2 + .github/workflows/nightly-tests.yml | 2 + .github/workflows/pr-tests.yml | 55 ++++- README.md | 2 +- backend/Cargo.toml | 5 +- .../json_exporter/expression_counter.rs | 0 .../{pilstark => estark}/json_exporter/mod.rs | 0 backend/src/estark/mod.rs | 231 ++++++++++++++++++ backend/src/estark/polygon_wrapper.rs | 83 +++++++ .../estark.rs => estark/starky_wrapper.rs} | 77 +----- backend/src/lib.rs | 27 +- backend/src/pilstark/mod.rs | 72 ------ book/src/installation.md | 13 + cli/Cargo.toml | 4 + cli/src/main.rs | 3 +- number/src/serialize.rs | 15 +- pipeline/Cargo.toml | 1 + pipeline/src/pipeline.rs | 24 +- pipeline/src/test_util.rs | 19 +- pipeline/src/util.rs | 5 +- pipeline/src/verify.rs | 11 +- pipeline/tests/asm.rs | 11 +- powdr/Cargo.toml | 3 +- riscv/Cargo.toml | 1 + riscv/tests/common/mod.rs | 4 +- riscv/tests/instructions.rs | 9 +- riscv/tests/riscv.rs | 39 ++- 27 files changed, 507 insertions(+), 211 deletions(-) rename backend/src/{pilstark => estark}/json_exporter/expression_counter.rs (100%) rename backend/src/{pilstark => estark}/json_exporter/mod.rs (100%) create mode 100644 backend/src/estark/mod.rs create mode 100644 backend/src/estark/polygon_wrapper.rs rename backend/src/{pilstark/estark.rs => estark/starky_wrapper.rs} (71%) delete mode 100644 backend/src/pilstark/mod.rs diff --git a/.github/workflows/build-cache.yml b/.github/workflows/build-cache.yml index e611f09116..91b30bcdc3 100644 --- a/.github/workflows/build-cache.yml +++ b/.github/workflows/build-cache.yml @@ -17,6 +17,8 @@ jobs: submodules: recursive - name: Install Rust toolchain 1.77 (with clippy and rustfmt) run: rustup toolchain install 1.77-x86_64-unknown-linux-gnu && rustup component add clippy --toolchain 1.77-x86_64-unknown-linux-gnu && rustup component add rustfmt --toolchain 1.77-x86_64-unknown-linux-gnu + - name: Install EStarkPolygon prover dependencies + run: sudo apt-get install -y nlohmann-json3-dev libpqxx-dev nasm - name: Lint run: cargo clippy --all --all-targets --all-features --profile pr-tests -- -D warnings - name: Lint diff --git a/.github/workflows/nightly-tests.yml b/.github/workflows/nightly-tests.yml index fa02ed4926..3ad6d78cae 100644 --- a/.github/workflows/nightly-tests.yml +++ b/.github/workflows/nightly-tests.yml @@ -72,6 +72,8 @@ jobs: run: rustup target add riscv32imac-unknown-none-elf --toolchain nightly-2024-02-01-x86_64-unknown-linux-gnu - name: Install stdlib run: rustup component add rust-src --toolchain nightly-2024-02-01-x86_64-unknown-linux-gnu + - name: Install EStarkPolygon prover dependencies + run: sudo apt-get install -y nlohmann-json3-dev libpqxx-dev nasm - name: Install pilcom run: git clone https://github.com/0xPolygonHermez/pilcom.git && cd pilcom && npm install - name: Check without Halo2 diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 0e4ba3d9e0..8ebc7ba27b 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -34,6 +34,8 @@ jobs: ${{ runner.os }}-cargo-pr-tests- - name: Install Rust toolchain 1.77 (with clippy and rustfmt) run: rustup toolchain install 1.77-x86_64-unknown-linux-gnu && rustup component add clippy --toolchain 1.77-x86_64-unknown-linux-gnu && rustup component add rustfmt --toolchain 1.77-x86_64-unknown-linux-gnu + - name: Install EStarkPolygon prover dependencies + run: sudo apt-get install -y nlohmann-json3-dev libpqxx-dev nasm - name: Lint no default features run: cargo clippy --all --all-targets --no-default-features --profile pr-tests -- -D warnings - name: Lint all features @@ -43,13 +45,17 @@ jobs: - name: Build run: cargo build --all-targets --all --all-features --profile pr-tests - uses: taiki-e/install-action@nextest + - name: Archive EStarkPolygon prover built dependencies + run: tar --zstd -cf pil-stark-prover-deps.tar.zst target/pr-tests/build/pil-stark-prover-*/out - name: Create tests archive run: cargo nextest archive --archive-file tests.tar.zst --cargo-profile pr-tests --workspace --all-features - - name: Upload tests archive + - name: Upload build artifacts uses: actions/upload-artifact@v2 with: name: tests_archive - path: tests.tar.zst + path: | + tests.tar.zst + pil-stark-prover-deps.tar.zst test_quick: needs: build @@ -81,10 +87,49 @@ jobs: run: git clone https://github.com/0xPolygonHermez/pilcom.git && cd pilcom && npm install - uses: taiki-e/install-action@nextest - name: Run default tests - run: PILCOM=$(pwd)/pilcom/ cargo nextest run --archive-file tests.tar.zst --verbose + run: cargo nextest run --archive-file tests.tar.zst --verbose + env: + PILCOM: ${{ github.workspace }}/pilcom/ - name: Run examples run: cargo run --example hello_world + test_estark_polygon: + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + - name: Download build artifacts + uses: actions/download-artifact@v2 + with: + name: tests_archive + - name: ⚡ Cache nodejs + uses: actions/cache@v3 + with: + path: | + ~/pilcom/node_modules + key: ${{ runner.os }}-pilcom-node-modules + - name: Install Rust toolchain 1.77 (with clippy and rustfmt) + run: rustup toolchain install 1.77-x86_64-unknown-linux-gnu && rustup component add clippy --toolchain 1.77-x86_64-unknown-linux-gnu && rustup component add rustfmt --toolchain 1.77-x86_64-unknown-linux-gnu + - name: Install nightly + run: rustup toolchain install nightly-2024-02-01-x86_64-unknown-linux-gnu + - name: Install riscv target + run: rustup target add riscv32imac-unknown-none-elf --toolchain nightly-2024-02-01-x86_64-unknown-linux-gnu + - name: Install stdlib + run: rustup component add rust-src --toolchain nightly-2024-02-01-x86_64-unknown-linux-gnu + - name: Install pilcom + run: git clone https://github.com/0xPolygonHermez/pilcom.git && cd pilcom && npm install + - name: Install EStarkPolygon prover system dependency + run: sudo apt-get install -y nlohmann-json3-dev + - uses: taiki-e/install-action@nextest + - name: Unpack EStarkPolygon built dependencies + run: tar --zstd -xf pil-stark-prover-deps.tar.zst + - name: Run EStark Polygon test + run: cargo nextest run --archive-file tests.tar.zst --verbose --run-ignored=ignored-only --no-capture -E "test(=test_vec_median_estark_polygon)" + env: + PILCOM: ${{ github.workspace }}/pilcom/ + test_slow: strategy: matrix: @@ -127,5 +172,7 @@ jobs: elif [[ "${{ matrix.test }}" == "subset2" ]]; then TESTS="test(=keccak) | test(=vec_median) | test(=instruction_tests::addi) | test(=many_chunks) | test(=sum_serde)" fi - PILCOM=$(pwd)/pilcom/ cargo nextest run --archive-file tests.tar.zst --verbose --run-ignored=ignored-only --no-capture -E "$TESTS" + cargo nextest run --archive-file tests.tar.zst --verbose --run-ignored=ignored-only --no-capture -E "$TESTS" shell: bash + env: + PILCOM: ${{ github.workspace }}/pilcom/ diff --git a/README.md b/README.md index 95ee4233d2..365e4c1956 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ which is compiled to RISCV, then to powdr-asm and finally to powdr-PIL. *powdr*-pil can be used to generate proofs using multiple backends, such as: - Halo2: via [polyexen](https://github.com/Dhole/polyexen) and [snark-verifer](https://github.com/privacy-scaling-explorations/snark-verifier/) -- eSTARK: via [Eigen's starky](https://github.com/0xEigenLabs/eigen-zkvm/) +- eSTARK: via [Eigen's starky](https://github.com/0xEigenLabs/eigen-zkvm/) or via [our fork of Polygon's zkevm-prover](https://github.com/powdr-labs/zkevm-prover). - SuperNova: ongoing work (https://github.com/powdr-labs/powdr/pull/453) All stages are fully automatic, which means you do not need to write any diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 13c5c19eb0..02bf89861e 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -9,6 +9,7 @@ repository = { workspace = true } [features] halo2 = ["dep:powdr-halo2"] +estark-polygon = ["dep:pil-stark-prover"] [dependencies] powdr-ast = { path = "../ast" } @@ -19,12 +20,14 @@ powdr-executor = { path = "../executor" } strum = { version = "0.24.1", features = ["derive"] } log = "0.4.17" +serde = "1.0" serde_json = "1.0" thiserror = "1.0.43" starky = { git = "https://github.com/0xEigenLabs/eigen-zkvm.git", rev = "59d2152" } +pil-stark-prover = { git = "https://github.com/powdr-labs/pil-stark-prover.git", rev = "d119f0b402c37be160", optional = true } +mktemp = "0.5.0" [dev-dependencies] -mktemp = "0.5.0" test-log = "0.2.12" env_logger = "0.10.0" pretty_assertions = "1.4.0" diff --git a/backend/src/pilstark/json_exporter/expression_counter.rs b/backend/src/estark/json_exporter/expression_counter.rs similarity index 100% rename from backend/src/pilstark/json_exporter/expression_counter.rs rename to backend/src/estark/json_exporter/expression_counter.rs diff --git a/backend/src/pilstark/json_exporter/mod.rs b/backend/src/estark/json_exporter/mod.rs similarity index 100% rename from backend/src/pilstark/json_exporter/mod.rs rename to backend/src/estark/json_exporter/mod.rs diff --git a/backend/src/estark/mod.rs b/backend/src/estark/mod.rs new file mode 100644 index 0000000000..21f0c97101 --- /dev/null +++ b/backend/src/estark/mod.rs @@ -0,0 +1,231 @@ +mod json_exporter; +#[cfg(feature = "estark-polygon")] +pub mod polygon_wrapper; +pub mod starky_wrapper; + +use std::{ + fs::File, + io::{self, BufWriter, Write}, + iter::{once, repeat}, + path::{Path, PathBuf}, +}; + +use crate::{Backend, BackendFactory, Error, Proof}; +use powdr_ast::analyzed::Analyzed; + +use powdr_executor::witgen::WitgenCallback; +use powdr_number::{write_polys_file, DegreeType, FieldElement}; +use serde::Serialize; +use starky::types::{StarkStruct, Step, PIL}; + +fn create_stark_struct(degree: DegreeType) -> StarkStruct { + assert!(degree > 1); + let n_bits = (DegreeType::BITS - (degree - 1).leading_zeros()) as usize; + let n_bits_ext = n_bits + 1; + + let steps = (2..=n_bits_ext) + .rev() + .step_by(4) + .map(|b| Step { nBits: b }) + .collect(); + + StarkStruct { + nBits: n_bits, + nBitsExt: n_bits_ext, + nQueries: 2, + verificationHashType: "GL".to_owned(), + steps, + } +} + +/// eStark provers require a fixed column with the equivalent semantics to +/// Polygon zkEVM's `L1` column. Powdr generated PIL will always have +/// `main.first_step`, but directly given PIL may not have it. This is a fixup +/// to inject such column if it doesn't exist. +/// +/// TODO Improve how this is done. +fn first_step_fixup<'a, F: FieldElement>( + pil: &'a Analyzed, + fixed: &'a [(String, Vec)], +) -> (PIL, Vec<(String, Vec)>) { + let degree = pil.degree(); + + let mut pil: PIL = json_exporter::export(pil); + + let mut fixed = fixed.to_vec(); + if !fixed.iter().any(|(k, _)| k == "main.first_step") { + use starky::types::Reference; + pil.nConstants += 1; + pil.references.insert( + "main.first_step".to_string(), + Reference { + polType: None, + type_: "constP".to_string(), + id: fixed.len(), + polDeg: degree as usize, + isArray: false, + elementType: None, + len: None, + }, + ); + fixed.push(( + "main.first_step".to_string(), + once(F::one()) + .chain(repeat(F::zero())) + .take(degree as usize) + .collect(), + )); + } + + (pil, fixed) +} + +struct EStarkFilesCommon<'a, F: FieldElement> { + degree: DegreeType, + pil: PIL, + fixed: Vec<(String, Vec)>, + output_dir: Option<&'a Path>, +} + +fn buffered_write_file( + path: &Path, + do_write: impl FnOnce(&mut BufWriter) -> R, +) -> Result { + let mut writer = BufWriter::new(File::create(path)?); + let result = do_write(&mut writer); + writer.flush()?; + + Ok(result) +} + +fn write_json_file(path: &Path, data: &T) -> Result<(), Error> { + buffered_write_file(path, |writer| { + serde_json::to_writer(writer, data).map_err(|e| e.to_string()) + })??; + + Ok(()) +} + +fn write_polys_bin( + path: &Path, + constants: &[(String, Vec)], +) -> Result<(), Error> { + buffered_write_file(path, |writer| write_polys_file(writer, constants))??; + + Ok(()) +} + +impl<'a, F: FieldElement> EStarkFilesCommon<'a, F> { + fn create( + analyzed: &'a Analyzed, + fixed: &'a [(String, Vec)], + output_dir: Option<&'a Path>, + setup: Option<&mut dyn std::io::Read>, + verification_key: Option<&mut dyn std::io::Read>, + ) -> Result { + if setup.is_some() { + return Err(Error::NoSetupAvailable); + } + if verification_key.is_some() { + return Err(Error::NoVerificationAvailable); + } + + // Pre-process the PIL and fixed columns. + let (pil, fixed) = first_step_fixup(analyzed, fixed); + + Ok(EStarkFilesCommon { + degree: analyzed.degree(), + pil, + fixed, + output_dir, + }) + } +} + +struct ProverInputFilePaths { + constants: PathBuf, + commits: PathBuf, + stark_struct: PathBuf, + contraints: PathBuf, +} + +impl<'a, F: FieldElement> EStarkFilesCommon<'a, F> { + /// Write the files in the EStark Polygon format. + fn write_files( + &self, + output_dir: &Path, + witness: &[(String, Vec)], + ) -> Result { + let paths = ProverInputFilePaths { + constants: output_dir.join("constants.bin"), + commits: output_dir.join("commits.bin"), + stark_struct: output_dir.join("starkstruct.json"), + contraints: output_dir.join("constraints.json"), + }; + + // Write the constants. + log::info!("Writing {}.", paths.constants.to_string_lossy()); + write_polys_bin(&paths.constants, &self.fixed)?; + + // Write the commits. + log::info!("Writing {}.", paths.commits.to_string_lossy()); + write_polys_bin(&paths.commits, witness)?; + + // Write the stark struct JSON. + log::info!("Writing {}.", paths.stark_struct.to_string_lossy()); + write_json_file(&paths.stark_struct, &create_stark_struct(self.degree))?; + + // Write the constraints in JSON. + log::info!("Writing {}.", paths.contraints.to_string_lossy()); + write_json_file(&paths.contraints, &self.pil)?; + + Ok(paths) + } +} + +pub struct DumpFactory; + +impl BackendFactory for DumpFactory { + fn create<'a>( + &self, + analyzed: &'a Analyzed, + fixed: &'a [(String, Vec)], + output_dir: Option<&'a Path>, + setup: Option<&mut dyn std::io::Read>, + verification_key: Option<&mut dyn std::io::Read>, + ) -> Result + 'a>, Error> { + Ok(Box::new(DumpBackend(EStarkFilesCommon::create( + analyzed, + fixed, + output_dir, + setup, + verification_key, + )?))) + } +} + +/// A backend that just dumps the files to the output directory. +struct DumpBackend<'a, F: FieldElement>(EStarkFilesCommon<'a, F>); + +impl<'a, F: FieldElement> Backend<'a, F> for DumpBackend<'a, F> { + fn prove( + &self, + witness: &[(String, Vec)], + prev_proof: Option, + // TODO: Implement challenges + _witgen_callback: WitgenCallback, + ) -> Result { + if prev_proof.is_some() { + return Err(Error::NoAggregationAvailable); + } + + let output_dir = self + .0 + .output_dir + .ok_or(Error::BackendError("output_dir is None".to_owned()))?; + + self.0.write_files(output_dir, witness)?; + + Ok(Vec::new()) + } +} diff --git a/backend/src/estark/polygon_wrapper.rs b/backend/src/estark/polygon_wrapper.rs new file mode 100644 index 0000000000..716cd39794 --- /dev/null +++ b/backend/src/estark/polygon_wrapper.rs @@ -0,0 +1,83 @@ +use std::{fs, path::Path}; + +use powdr_ast::analyzed::Analyzed; +use powdr_executor::witgen::WitgenCallback; +use powdr_number::FieldElement; + +use crate::{Backend, BackendFactory, Error, Proof}; + +use super::EStarkFilesCommon; + +pub struct Factory; + +impl BackendFactory for Factory { + fn create<'a>( + &self, + analyzed: &'a Analyzed, + fixed: &'a [(String, Vec)], + output_dir: Option<&'a Path>, + setup: Option<&mut dyn std::io::Read>, + verification_key: Option<&mut dyn std::io::Read>, + ) -> Result + 'a>, Error> { + Ok(Box::new(PolygonBackend(EStarkFilesCommon::create( + analyzed, + fixed, + output_dir, + setup, + verification_key, + )?))) + } +} + +struct PolygonBackend<'a, F: FieldElement>(EStarkFilesCommon<'a, F>); + +// TODO: make both eStark backends interchangeable, from user perspective. +// TODO: implement the other Backend trait methods. +impl<'a, F: FieldElement> Backend<'a, F> for PolygonBackend<'a, F> { + fn prove( + &self, + witness: &[(String, Vec)], + prev_proof: Option, + // TODO: Implement challenges + _witgen_callback: WitgenCallback, + ) -> Result { + if prev_proof.is_some() { + return Err(Error::NoAggregationAvailable); + } + + let tmp_dir; + let output_dir = if let Some(output_dir) = self.0.output_dir { + output_dir + } else { + tmp_dir = mktemp::Temp::new_dir()?; + tmp_dir.as_path() + }; + + let input_paths = self.0.write_files(output_dir, witness)?; + + // Generate the proof. + let proof_paths = pil_stark_prover::generate_proof( + &input_paths.contraints, + &input_paths.stark_struct, + &input_paths.constants, + &input_paths.commits, + output_dir, + ) + .map_err(|e| Error::BackendError(e.to_string()))?; + + // Sanity check: verify the proof. + let publics_path = output_dir.join("publics.json"); + // TODO: properly handle publics + fs::write(&publics_path, "[]")?; + pil_stark_prover::verify_proof( + &proof_paths.verification_key_json, + &proof_paths.starkinfo_json, + &proof_paths.proof_json, + &publics_path, + ) + .map_err(|e| Error::BackendError(e.to_string()))?; + + // Read the proof. + Ok(fs::read(&proof_paths.proof_json)?) + } +} diff --git a/backend/src/pilstark/estark.rs b/backend/src/estark/starky_wrapper.rs similarity index 71% rename from backend/src/pilstark/estark.rs rename to backend/src/estark/starky_wrapper.rs index 93d1477142..600c0370e9 100644 --- a/backend/src/pilstark/estark.rs +++ b/backend/src/estark/starky_wrapper.rs @@ -1,11 +1,10 @@ use std::io; -use std::iter::{once, repeat}; use std::time::Instant; -use crate::{pilstark, Backend, BackendFactory, Error}; +use crate::{Backend, BackendFactory, Error}; use powdr_ast::analyzed::Analyzed; use powdr_executor::witgen::WitgenCallback; -use powdr_number::{DegreeType, FieldElement, GoldilocksField, LargeInt}; +use powdr_number::{FieldElement, GoldilocksField, LargeInt}; use starky::{ merklehash::MerkleTreeGL, @@ -15,12 +14,14 @@ use starky::{ stark_verify::stark_verify, traits::FieldExtension, transcript::TranscriptGL, - types::{StarkStruct, Step, PIL}, + types::{StarkStruct, PIL}, }; -pub struct EStarkFactory; +use super::{create_stark_struct, first_step_fixup}; -impl BackendFactory for EStarkFactory { +pub struct Factory; + +impl BackendFactory for Factory { fn create<'a>( &self, pil: &'a Analyzed, @@ -38,26 +39,9 @@ impl BackendFactory for EStarkFactory { return Err(Error::NoSetupAvailable); } - let degree = pil.degree(); - assert!(degree > 1); - let n_bits = (DegreeType::BITS - (degree - 1).leading_zeros()) as usize; - let n_bits_ext = n_bits + 1; - - let steps = (2..=n_bits_ext) - .rev() - .step_by(4) - .map(|b| Step { nBits: b }) - .collect(); - - let params = StarkStruct { - nBits: n_bits, - nBitsExt: n_bits_ext, - nQueries: 2, - verificationHashType: "GL".to_owned(), - steps, - }; + let params = create_stark_struct(pil.degree()); - let (pil_json, fixed) = pil_json(pil, fixed); + let (pil_json, fixed) = first_step_fixup(pil, fixed); let const_pols = to_starky_pols_array(&fixed, &pil_json, PolKind::Constant); let setup = if let Some(vkey) = verification_key { @@ -75,49 +59,6 @@ impl BackendFactory for EStarkFactory { } } -fn pil_json<'a, F: FieldElement>( - pil: &'a Analyzed, - fixed: &'a [(String, Vec)], -) -> (PIL, Vec<(String, Vec)>) { - let degree = pil.degree(); - - let mut pil: PIL = pilstark::json_exporter::export(pil); - - // TODO starky requires a fixed column with the equivalent - // semantics to Polygon zkEVM's `L1` column. - // It takes the name of that column via the API. - // Powdr generated PIL will always have `main.first_step`, - // but directly given PIL may not have it. - // This is a hack to inject such column if it doesn't exist. - // It should be eventually improved. - let mut fixed = fixed.to_vec(); - if !fixed.iter().any(|(k, _)| k == "main.first_step") { - use starky::types::Reference; - pil.nConstants += 1; - pil.references.insert( - "main.first_step".to_string(), - Reference { - polType: None, - type_: "constP".to_string(), - id: fixed.len(), - polDeg: degree as usize, - isArray: false, - elementType: None, - len: None, - }, - ); - fixed.push(( - "main.first_step".to_string(), - once(F::one()) - .chain(repeat(F::zero())) - .take(degree as usize) - .collect(), - )); - } - - (pil, fixed) -} - fn create_stark_setup( mut pil: PIL, const_pols: &PolsArray, diff --git a/backend/src/lib.rs b/backend/src/lib.rs index fd3315ae5a..8fe2c5b31a 100644 --- a/backend/src/lib.rs +++ b/backend/src/lib.rs @@ -1,8 +1,8 @@ #![deny(clippy::print_stdout)] +mod estark; #[cfg(feature = "halo2")] mod halo2_impl; -mod pilstark; use powdr_ast::analyzed::Analyzed; use powdr_executor::witgen::WitgenCallback; @@ -18,10 +18,13 @@ pub enum BackendType { #[cfg(feature = "halo2")] #[strum(serialize = "halo2-mock")] Halo2Mock, - #[strum(serialize = "estark")] - EStark, - #[strum(serialize = "pil-stark-cli")] - PilStarkCli, + #[cfg(feature = "estark-polygon")] + #[strum(serialize = "estark-polygon")] + EStarkPolygon, + #[strum(serialize = "estark-starky")] + EStarkStarky, + #[strum(serialize = "estark-dump")] + EStarkDump, } impl BackendType { @@ -30,16 +33,22 @@ impl BackendType { const HALO2_FACTORY: halo2_impl::Halo2ProverFactory = halo2_impl::Halo2ProverFactory; #[cfg(feature = "halo2")] const HALO2_MOCK_FACTORY: halo2_impl::Halo2MockFactory = halo2_impl::Halo2MockFactory; - const ESTARK_FACTORY: pilstark::estark::EStarkFactory = pilstark::estark::EStarkFactory; - const PIL_STARK_CLI_FACTORY: pilstark::PilStarkCliFactory = pilstark::PilStarkCliFactory; + #[cfg(feature = "estark-polygon")] + const ESTARK_POLYGON_FACTORY: estark::polygon_wrapper::Factory = + estark::polygon_wrapper::Factory; + const ESTARK_STARKY_FACTORY: estark::starky_wrapper::Factory = + estark::starky_wrapper::Factory; + const ESTARK_DUMP_FACTORY: estark::DumpFactory = estark::DumpFactory; match self { #[cfg(feature = "halo2")] BackendType::Halo2 => &HALO2_FACTORY, #[cfg(feature = "halo2")] BackendType::Halo2Mock => &HALO2_MOCK_FACTORY, - BackendType::EStark => &ESTARK_FACTORY, - BackendType::PilStarkCli => &PIL_STARK_CLI_FACTORY, + #[cfg(feature = "estark-polygon")] + BackendType::EStarkPolygon => &ESTARK_POLYGON_FACTORY, + BackendType::EStarkStarky => &ESTARK_STARKY_FACTORY, + BackendType::EStarkDump => &ESTARK_DUMP_FACTORY, } } } diff --git a/backend/src/pilstark/mod.rs b/backend/src/pilstark/mod.rs deleted file mode 100644 index be908ab2f2..0000000000 --- a/backend/src/pilstark/mod.rs +++ /dev/null @@ -1,72 +0,0 @@ -pub mod estark; -mod json_exporter; - -use std::{ - fs::File, - io::{BufWriter, Write}, - path::Path, -}; - -use crate::{Backend, BackendFactory, Error, Proof}; -use powdr_ast::analyzed::Analyzed; -use powdr_executor::witgen::WitgenCallback; -use powdr_number::FieldElement; - -pub struct PilStarkCliFactory; - -impl BackendFactory for PilStarkCliFactory { - fn create<'a>( - &self, - analyzed: &'a Analyzed, - _fixed: &'a [(String, Vec)], - output_dir: Option<&'a Path>, - setup: Option<&mut dyn std::io::Read>, - verification_key: Option<&mut dyn std::io::Read>, - ) -> Result + 'a>, Error> { - if setup.is_some() { - return Err(Error::NoSetupAvailable); - } - if verification_key.is_some() { - return Err(Error::NoVerificationAvailable); - } - Ok(Box::new(PilStarkCli { - analyzed, - output_dir, - })) - } -} - -pub struct PilStarkCli<'a, F: FieldElement> { - analyzed: &'a Analyzed, - output_dir: Option<&'a Path>, -} - -impl<'a, F: FieldElement> Backend<'a, F> for PilStarkCli<'a, F> { - fn prove( - &self, - _witness: &[(String, Vec)], - prev_proof: Option, - // TODO: Implement challenges - _witgen_callback: WitgenCallback, - ) -> Result { - if prev_proof.is_some() { - return Err(Error::NoAggregationAvailable); - } - - // Write the constraints in the format expected by the prover-cpp - if let Some(output_dir) = self.output_dir { - let path = output_dir.join("constraints.json"); - let mut writer = BufWriter::new(File::create(path)?); - serde_json::to_writer(&mut writer, &json_exporter::export(self.analyzed)) - .map_err(|e| e.to_string())?; - writer.flush()?; - } else { - // If we were going to call the prover-cpp, we could write the - // constraints.json to a temporary directory in case no output_dir - // is provided. - } - - // TODO: actually use prover-cpp to generate the proof - Ok(Vec::new()) - } -} diff --git a/book/src/installation.md b/book/src/installation.md index 19b1d7db56..28019f46af 100644 --- a/book/src/installation.md +++ b/book/src/installation.md @@ -10,6 +10,19 @@ The easiest way to install both is with [`rustup.rs`](https://rustup.rs/). On Windows, you will also need a recent version of [Visual Studio](https://visualstudio.microsoft.com/downloads/), installed with the "Desktop Development With C++" Workloads option. +If you want to enable `estark-polygon` feature, you also need the following +runtime dependencies installed on the system: +- `gcc` +- `nlohmann-json3-dev` + +Yes, gcc and headers at runtime, this is not a typo. + +You will also need the following build time dependencies: +- `make` +- `pkg-config` +- `libpqxx-dev` +- `nasm` + ## Building Using a single Cargo command (enable the halo2 backend to use it with the cli): diff --git a/cli/Cargo.toml b/cli/Cargo.toml index e62cfed8df..5851c112f4 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -11,6 +11,7 @@ default-run = "powdr" [features] default = [] # halo2 is disabled by default halo2 = ["powdr-backend/halo2", "powdr-pipeline/halo2"] +estark-polygon = ["powdr-backend/estark-polygon", "powdr-pipeline/estark-polygon", "powdr-riscv/estark-polygon"] [dependencies] powdr-backend = { path = "../backend" } @@ -30,6 +31,9 @@ clap-markdown = "0.1.3" [dev-dependencies] tempfile = "3.6" +test-log = "0.2.12" +env_logger = "0.10.0" + [[bin]] name = "powdr" path = "src/main.rs" diff --git a/cli/src/main.rs b/cli/src/main.rs index 570097a795..534c22ecfd 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -912,6 +912,7 @@ fn optimize_and_output(file: &str) { mod test { use crate::{run_command, Commands, CsvRenderModeCLI, FieldArgument}; use powdr_backend::BackendType; + use test_log::test; #[test] fn simple_sum() { @@ -930,7 +931,7 @@ mod test { inputs: "3,2,1,2".into(), force: false, pilo: false, - prove_with: Some(BackendType::PilStarkCli), + prove_with: Some(BackendType::EStarkDump), export_csv: true, csv_mode: CsvRenderModeCLI::Hex, just_execute: false, diff --git a/number/src/serialize.rs b/number/src/serialize.rs index 394f57c55e..f463d3db2c 100644 --- a/number/src/serialize.rs +++ b/number/src/serialize.rs @@ -1,4 +1,4 @@ -use std::io::{Read, Write}; +use std::io::{self, Read, Write}; use ark_serialize::{CanonicalDeserialize, CanonicalSerialize, Compress, Validate}; use csv::{Reader, Writer}; @@ -89,11 +89,14 @@ fn ceil_div(num: usize, div: usize) -> usize { (num + div - 1) / div } -pub fn write_polys_file(file: &mut impl Write, polys: &[(String, Vec)]) { +pub fn write_polys_file( + file: &mut impl Write, + polys: &[(String, Vec)], +) -> Result<(), io::Error> { let width = ceil_div(T::BITS as usize, 64) * 8; if polys.is_empty() { - return; + return Ok(()); } // TODO maybe the witness should have a proper type that @@ -107,9 +110,11 @@ pub fn write_polys_file(file: &mut impl Write, polys: &[(String for (_name, constant) in polys { let bytes = constant[i].to_bytes_le(); assert_eq!(bytes.len(), width); - file.write_all(&bytes).unwrap(); + file.write_all(&bytes)?; } } + + Ok(()) } pub fn read_polys_file( @@ -188,7 +193,7 @@ mod tests { let (polys, degree) = test_polys(); - write_polys_file(&mut buf, &polys); + write_polys_file(&mut buf, &polys).unwrap(); let (read_polys, read_degree) = read_polys_file::( &mut Cursor::new(buf), &["a".to_string(), "b".to_string()], diff --git a/pipeline/Cargo.toml b/pipeline/Cargo.toml index 0b6dd69e2e..64dcf7252f 100644 --- a/pipeline/Cargo.toml +++ b/pipeline/Cargo.toml @@ -9,6 +9,7 @@ repository = { workspace = true } [features] halo2 = ["powdr-backend/halo2"] +estark-polygon = ["powdr-backend/estark-polygon"] [dependencies] powdr-airgen = { path = "../airgen" } diff --git a/pipeline/src/pipeline.rs b/pipeline/src/pipeline.rs index 8b4c3c1b15..ba7e040fd0 100644 --- a/pipeline/src/pipeline.rs +++ b/pipeline/src/pipeline.rs @@ -2,7 +2,7 @@ use std::{ borrow::Borrow, fmt::Display, fs, - io::{self, BufReader, BufWriter}, + io::{self, BufReader}, marker::Send, path::{Path, PathBuf}, rc::Rc, @@ -24,12 +24,12 @@ use powdr_executor::{ chain_callbacks, unused_query_callback, QueryCallback, WitgenCallback, WitnessGenerator, }, }; -use powdr_number::{write_polys_csv_file, write_polys_file, CsvRenderMode, FieldElement}; +use powdr_number::{write_polys_csv_file, CsvRenderMode, FieldElement}; use powdr_schemas::SerializedAnalyzed; use crate::{ inputs_to_query_callback, serde_data_to_query_callback, - util::{try_read_poly_set, write_or_panic, FixedPolySet, WitnessPolySet}, + util::{try_read_poly_set, FixedPolySet, WitnessPolySet}, }; type Columns = Vec<(String, Vec)>; @@ -373,7 +373,7 @@ impl Pipeline { pub fn read_constants(mut self, directory: &Path) -> Self { let pil = self.compute_optimized_pil().unwrap(); - let fixed = try_read_poly_set::(&pil, directory, self.name()) + let fixed = try_read_poly_set::(&pil, directory) .map(|(fixed, degree_fixed)| { assert_eq!(pil.degree.unwrap(), degree_fixed); fixed @@ -393,7 +393,7 @@ impl Pipeline { pub fn read_witness(mut self, directory: &Path) -> Self { let pil = self.compute_optimized_pil().unwrap(); - let witness = try_read_poly_set::(&pil, directory, self.name()) + let witness = try_read_poly_set::(&pil, directory) .map(|(witness, degree_witness)| { assert_eq!(pil.degree.unwrap(), degree_witness); witness @@ -487,24 +487,11 @@ impl Pipeline { Ok(()) } - fn maybe_write_constants(&self, constants: &[(String, Vec)]) -> Result<(), Vec> { - if let Some(path) = self.path_if_should_write(|name| format!("{name}_constants.bin"))? { - let writer = BufWriter::new(fs::File::create(path).unwrap()); - write_or_panic(writer, |writer| write_polys_file(writer, constants)); - } - Ok(()) - } - fn maybe_write_witness( &self, fixed: &[(String, Vec)], witness: &[(String, Vec)], ) -> Result<(), Vec> { - if let Some(path) = self.path_if_should_write(|name| format!("{name}_commits.bin"))? { - let file = BufWriter::new(fs::File::create(path).unwrap()); - write_or_panic(file, |file| write_polys_file(file, witness)); - } - if self.arguments.export_witness_csv { if let Some(path) = self.path_if_should_write(|name| format!("{name}_columns.csv"))? { let columns = fixed.iter().chain(witness.iter()).collect::>(); @@ -792,7 +779,6 @@ impl Pipeline { let start = Instant::now(); let fixed_cols = constant_evaluator::generate(&pil); - self.maybe_write_constants(&fixed_cols)?; self.log(&format!("Took {}", start.elapsed().as_secs_f32())); self.artifact.fixed_cols = Some(Rc::new(fixed_cols)); diff --git a/pipeline/src/test_util.rs b/pipeline/src/test_util.rs index 1bce5f5c12..07fdd34ef2 100644 --- a/pipeline/src/test_util.rs +++ b/pipeline/src/test_util.rs @@ -28,7 +28,7 @@ pub fn verify_test_file( .from_file(resolve_test_file(file_name)) .with_prover_inputs(inputs) .add_external_witness_values(external_witness_values); - verify_pipeline(pipeline) + verify_pipeline(pipeline, BackendType::EStarkDump) } pub fn verify_asm_string( @@ -47,11 +47,14 @@ pub fn verify_asm_string( pipeline = pipeline.add_data_vec(&data); } - verify_pipeline(pipeline).unwrap(); + verify_pipeline(pipeline, BackendType::EStarkDump).unwrap(); } -pub fn verify_pipeline(pipeline: Pipeline) -> Result<(), String> { - let mut pipeline = pipeline.with_backend(BackendType::PilStarkCli); +pub fn verify_pipeline( + pipeline: Pipeline, + backend: BackendType, +) -> Result<(), String> { + let mut pipeline = pipeline.with_backend(backend); let tmp_dir = mktemp::Temp::new_dir().unwrap(); if pipeline.output_dir().is_none() { @@ -60,7 +63,7 @@ pub fn verify_pipeline(pipeline: Pipeline) -> Result<(), String pipeline.compute_proof().unwrap(); - verify(pipeline.output_dir().unwrap(), pipeline.name(), None) + verify(pipeline.output_dir().unwrap()) } pub fn gen_estark_proof(file_name: &str, inputs: Vec) { @@ -69,7 +72,7 @@ pub fn gen_estark_proof(file_name: &str, inputs: Vec) { .with_tmp_output(&tmp_dir) .from_file(resolve_test_file(file_name)) .with_prover_inputs(inputs) - .with_backend(powdr_backend::BackendType::EStark); + .with_backend(powdr_backend::BackendType::EStarkStarky); pipeline.clone().compute_proof().unwrap(); @@ -228,7 +231,7 @@ pub fn assert_proofs_fail_for_invalid_witnesses_pilcom( .from_file(resolve_test_file(file_name)) .set_witness(convert_witness(witness)); - assert!(verify_pipeline(pipeline.clone()).is_err()); + assert!(verify_pipeline(pipeline.clone(), BackendType::EStarkDump).is_err()); } pub fn assert_proofs_fail_for_invalid_witnesses_estark( @@ -241,7 +244,7 @@ pub fn assert_proofs_fail_for_invalid_witnesses_estark( assert!(pipeline .clone() - .with_backend(powdr_backend::BackendType::EStark) + .with_backend(powdr_backend::BackendType::EStarkStarky) .compute_proof() .is_err()); } diff --git a/pipeline/src/util.rs b/pipeline/src/util.rs index f3d26e7da3..ec17b83815 100644 --- a/pipeline/src/util.rs +++ b/pipeline/src/util.rs @@ -39,7 +39,6 @@ impl PolySet for WitnessPolySet { pub fn try_read_poly_set( pil: &Analyzed, dir: &Path, - name: &str, ) -> Option<(Vec<(String, Vec)>, DegreeType)> { let column_names: Vec = P::get_polys(pil) .iter() @@ -48,9 +47,9 @@ pub fn try_read_poly_set( .collect(); (!column_names.is_empty()).then(|| { - let fname = format!("{name}_{}", P::FILE_NAME); + let path = dir.join(P::FILE_NAME); read_polys_file( - &mut BufReader::new(File::open(dir.join(fname)).unwrap()), + &mut BufReader::new(File::open(path).unwrap()), &column_names, ) }) diff --git a/pipeline/src/verify.rs b/pipeline/src/verify.rs index 149947da6d..617d9845b0 100644 --- a/pipeline/src/verify.rs +++ b/pipeline/src/verify.rs @@ -1,16 +1,11 @@ use std::{path::Path, process::Command}; -pub fn verify(temp_dir: &Path, name: &str, constants_name: Option<&str>) -> Result<(), String> { +pub fn verify(temp_dir: &Path) -> Result<(), String> { let pilcom = std::env::var("PILCOM") .expect("Please set the PILCOM environment variable to the path to the pilcom repository."); - let constants_name = constants_name.unwrap_or(name); - - let constants_file = format!( - "{}/{constants_name}_constants.bin", - temp_dir.to_str().unwrap() - ); - let commits_file = format!("{}/{name}_commits.bin", temp_dir.to_str().unwrap()); + let constants_file = format!("{}/constants.bin", temp_dir.to_str().unwrap()); + let commits_file = format!("{}/commits.bin", temp_dir.to_str().unwrap()); let constraints_file = format!("{}/constraints.json", temp_dir.to_str().unwrap()); let verifier_output = Command::new("node") diff --git a/pipeline/tests/asm.rs b/pipeline/tests/asm.rs index 1cd62cf5f7..40131bb943 100644 --- a/pipeline/tests/asm.rs +++ b/pipeline/tests/asm.rs @@ -1,3 +1,4 @@ +use powdr_backend::BackendType; use powdr_number::{Bn254Field, FieldElement, GoldilocksField}; use powdr_pipeline::{ test_util::{gen_estark_proof, resolve_test_file, test_halo2, verify_test_file}, @@ -341,14 +342,14 @@ fn read_poly_files() { // generate poly files let mut pipeline = Pipeline::::default() .from_file(resolve_test_file(f)) - .with_output(tmp_dir.to_path_buf(), true); + .with_output(tmp_dir.to_path_buf(), true) + .with_backend(BackendType::EStarkDump); pipeline.compute_witness().unwrap(); - let name = pipeline.name().to_string(); let pil = pipeline.compute_optimized_pil().unwrap(); + pipeline.compute_proof().unwrap(); // check fixed cols (may have no fixed cols) - if let Some((fixed, degree)) = - try_read_poly_set::(&pil, tmp_dir.as_path(), &name) + if let Some((fixed, degree)) = try_read_poly_set::(&pil, tmp_dir.as_path()) { assert_eq!(pil.degree(), degree); assert_eq!(pil.degree(), fixed[0].1.len() as u64); @@ -356,7 +357,7 @@ fn read_poly_files() { // check witness cols (examples assumed to have at least one witness col) let (witness, degree) = - try_read_poly_set::(&pil, tmp_dir.as_path(), &name).unwrap(); + try_read_poly_set::(&pil, tmp_dir.as_path()).unwrap(); assert_eq!(pil.degree(), degree); assert_eq!(pil.degree(), witness[0].1.len() as u64); } diff --git a/powdr/Cargo.toml b/powdr/Cargo.toml index 32c1cb1eac..422cec638c 100644 --- a/powdr/Cargo.toml +++ b/powdr/Cargo.toml @@ -18,5 +18,6 @@ powdr-riscv = { path = "../riscv" } powdr-riscv-executor = { path = "../riscv-executor" } [features] -default = ["halo2"] # halo2 is enabled by default +default = ["halo2"] halo2 = ["powdr-backend/halo2", "powdr-pipeline/halo2"] +estark-polygon = ["powdr-backend/estark-polygon", "powdr-pipeline/estark-polygon", "powdr-riscv/estark-polygon"] diff --git a/riscv/Cargo.toml b/riscv/Cargo.toml index 5046018d0d..3e9af3323c 100644 --- a/riscv/Cargo.toml +++ b/riscv/Cargo.toml @@ -11,6 +11,7 @@ repository = { workspace = true } [features] default = [] # complex-tests is disabled by default complex-tests = [] +estark-polygon = ["powdr-pipeline/estark-polygon"] [dependencies] powdr-ast = { path = "../ast" } diff --git a/riscv/tests/common/mod.rs b/riscv/tests/common/mod.rs index b312c0555a..96560d0c40 100644 --- a/riscv/tests/common/mod.rs +++ b/riscv/tests/common/mod.rs @@ -1,3 +1,4 @@ +use powdr_backend::BackendType; use powdr_number::GoldilocksField; use powdr_pipeline::{test_util::verify_pipeline, Pipeline}; use std::path::PathBuf; @@ -8,6 +9,7 @@ pub fn verify_riscv_asm_string( contents: &str, inputs: Vec, data: Option>, + backend: BackendType, ) { let temp_dir = mktemp::Temp::new_dir().unwrap().release(); @@ -30,5 +32,5 @@ pub fn verify_riscv_asm_string( usize::MAX, powdr_riscv_executor::ExecMode::Fast, ); - verify_pipeline(pipeline).unwrap(); + verify_pipeline(pipeline, backend).unwrap(); } diff --git a/riscv/tests/instructions.rs b/riscv/tests/instructions.rs index 1e18edab20..1476631a7e 100644 --- a/riscv/tests/instructions.rs +++ b/riscv/tests/instructions.rs @@ -2,6 +2,7 @@ mod common; mod instruction_tests { use crate::common::verify_riscv_asm_string; + use powdr_backend::BackendType; use powdr_number::GoldilocksField; use powdr_riscv::compiler::compile; use powdr_riscv::Runtime; @@ -15,7 +16,13 @@ mod instruction_tests { false, ); - verify_riscv_asm_string::<()>(&format!("{name}.asm"), &powdr_asm, Default::default(), None); + verify_riscv_asm_string::<()>( + &format!("{name}.asm"), + &powdr_asm, + Default::default(), + None, + BackendType::EStarkDump, + ); } include!(concat!(env!("OUT_DIR"), "/instruction_tests.rs")); diff --git a/riscv/tests/riscv.rs b/riscv/tests/riscv.rs index 1932560b5a..99c4d0c3ea 100644 --- a/riscv/tests/riscv.rs +++ b/riscv/tests/riscv.rs @@ -34,9 +34,9 @@ pub fn test_continuations(case: &str) { let pipeline_callback = |pipeline: Pipeline| -> Result<(), ()> { // Can't use `verify_pipeline`, because the pipeline was renamed in the middle of after // computing the constants file. - let mut pipeline = pipeline.with_backend(BackendType::PilStarkCli); + let mut pipeline = pipeline.with_backend(BackendType::EStarkDump); pipeline.compute_proof().unwrap(); - verify(pipeline.output_dir().unwrap(), pipeline.name(), Some(case)).unwrap(); + verify(pipeline.output_dir().unwrap()).unwrap(); Ok(()) }; let bootloader_inputs = rust_continuations_dry_run(&mut pipeline); @@ -126,6 +126,22 @@ fn keccak() { verify_riscv_crate(case, Default::default(), &Runtime::base()); } +#[cfg(feature = "estark-polygon")] +#[test] +#[ignore = "Too slow"] +fn test_vec_median_estark_polygon() { + let case = "vec_median"; + verify_riscv_crate_with_backend( + case, + [5, 11, 15, 75, 6, 5, 1, 4, 7, 3, 2, 9, 2] + .into_iter() + .map(|x| x.into()) + .collect(), + &Runtime::base(), + BackendType::EStarkPolygon, + ); +} + #[test] #[ignore = "Too slow"] fn vec_median() { @@ -234,8 +250,17 @@ fn many_chunks_memory() { } fn verify_riscv_crate(case: &str, inputs: Vec, runtime: &Runtime) { + verify_riscv_crate_with_backend(case, inputs, runtime, BackendType::EStarkDump) +} + +fn verify_riscv_crate_with_backend( + case: &str, + inputs: Vec, + runtime: &Runtime, + backend: BackendType, +) { let powdr_asm = compile_riscv_crate::(case, runtime); - verify_riscv_asm_string::<()>(&format!("{case}.asm"), &powdr_asm, inputs, None); + verify_riscv_asm_string::<()>(&format!("{case}.asm"), &powdr_asm, inputs, None, backend); } fn verify_riscv_crate_with_data( @@ -246,7 +271,13 @@ fn verify_riscv_crate_with_data( ) { let powdr_asm = compile_riscv_crate::(case, runtime); - verify_riscv_asm_string(&format!("{case}.asm"), &powdr_asm, inputs, Some(data)); + verify_riscv_asm_string( + &format!("{case}.asm"), + &powdr_asm, + inputs, + Some(data), + BackendType::EStarkDump, + ); } fn compile_riscv_crate(case: &str, runtime: &Runtime) -> String {