feat(rollup-sync-service): support codecv2 (#726)

* feat(codecv1): support zstd compression

* revert .gitignore changes

* change dependency

* add zstd compression

* bump golang version to 1.21.x

* use codecv2

* remove an invalid command

* tweak

* tweak

* remove libzstd, use .so files in da-codec release instead

* fix CI

* fix CI

* try fix CI

* fix CI

* tweak

* fix

* fix CI

* tweak

* tweak

* address comments

* tweak

* remove cp to /usr/local/lib

* fix CI

* remove cp to /usr/local/lib in CI

* try fix

* fix

* fix

* fix

* remove env GO111MODULE=on

* add .so files to Dockerfiles

* revert make geth changes

* fix Dockerfile changes

* tweak

* revert changes

* tweak

* trigger ci

* update Dockerfiles

* fix

* add codecv2 in decode block ranges from encoded chunks

* address comment

* nit

* update dependencies

* use /scroll/lib/ in testing

* trigger ci

* add unit tests

* chore: auto version bump [bot]

* remove zkp-builder in Dockerfile.mockccc

* fix Dockerfile

* Update Makefile

Co-authored-by: Péter Garamvölgyi <[email protected]>

* use -Wl,-rpath,/scroll/lib/

* change some static values to env var

* update dependency

* change pwd to scroll_lib_path

* fix

* add sudo in mkdir

* add sudo

* update da-codec dependency

* chore: auto version bump [bot]

* chore: auto version bump [bot]

* chore: auto version bump [bot]

* update zstd dependency

* update libscroll_zstd.so and remove libzktrie.so related to zstd

* chore: auto version bump [bot]

* uncomment ./rollup tests

* fix CI

* fix CI

* upgrade libzkp to v0.11.0rc2 (#773)

* upgrade libzkp

* remove zktrie.so

* done

* fix build

---------

Co-authored-by: colin <[email protected]>

---------

Co-authored-by: colinlyguo <[email protected]>
Co-authored-by: Péter Garamvölgyi <[email protected]>
Co-authored-by: Zhang Zhuo <[email protected]>

.github/workflows/l2geth_ci.yml

@@ -20,7 +20,7 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.20.x
+        go-version: 1.21.x
     - name: Checkout code
       uses: actions/checkout@v2
     - name: Build
@@ -33,7 +33,7 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.20.x
+        go-version: 1.21.x
     - name: Install rust
       uses: actions-rs/toolchain@v1
       with:
@@ -46,7 +46,6 @@ jobs:
       run: |
         make libzkp
         sudo cp ./rollup/circuitcapacitychecker/libzkp/libzkp.so /usr/local/lib/
-        sudo cp ./rollup/circuitcapacitychecker/libzkp/libzktrie.so /usr/local/lib/
         make geth
   check:
     if: github.event.pull_request.draft == false
@@ -55,7 +54,7 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.20.x
+        go-version: 1.21.x
     - name: Checkout code
       uses: actions/checkout@v2
     - name: Lint
@@ -88,7 +87,7 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.20.x
+        go-version: 1.21.x
     - name: Checkout code
       uses: actions/checkout@v2
     - run: go mod tidy
@@ -105,12 +104,11 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.20.x
+        go-version: 1.21.x
     - name: Checkout code
       uses: actions/checkout@v2
     - name: Test
       run: |
-        go get ./...
         make test
     - name: Upload coverage report
       run: bash <(curl -s https://codecov.io/bash)

Dockerfile

@@ -2,6 +2,8 @@
 ARG COMMIT=""
 ARG VERSION=""
 ARG BUILDNUM=""
+ARG LIBSCROLL_ZSTD_VERSION=v0.1.0-rc0-ubuntu20.04
+ARG SCROLL_LIB_PATH=/scroll/lib
 
 # Build libzkp dependency
 FROM scrolltech/go-rust-builder:go-1.20-rust-nightly-2022-12-10 as chef
@@ -19,27 +21,43 @@ RUN cargo chef cook --release --recipe-path recipe.json
 COPY ./rollup/circuitcapacitychecker/libzkp .
 RUN cargo clean
 RUN cargo build --release
-RUN find ./ | grep libzktrie.so | xargs -I{} cp {} /app/target/release/
 
 # Build Geth in a stock Go builder container
 FROM scrolltech/go-rust-builder:go-1.20-rust-nightly-2022-12-10 as builder
 
 ADD . /go-ethereum
-COPY --from=zkp-builder /app/target/release/libzkp.so /usr/local/lib/
-COPY --from=zkp-builder /app/target/release/libzktrie.so /usr/local/lib/
-ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/
+
+ARG LIBSCROLL_ZSTD_VERSION
+ARG SCROLL_LIB_PATH
+
+RUN mkdir -p $SCROLL_LIB_PATH
+
+COPY --from=zkp-builder /app/target/release/libzkp.so $SCROLL_LIB_PATH
+RUN wget -O $SCROLL_LIB_PATH/libscroll_zstd.so https://github.com/scroll-tech/da-codec/releases/download/$LIBSCROLL_ZSTD_VERSION/libscroll_zstd.so
+
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SCROLL_LIB_PATH
+ENV CGO_LDFLAGS="-L$SCROLL_LIB_PATH -Wl,-rpath,$SCROLL_LIB_PATH"
+
 RUN cd /go-ethereum && env GO111MODULE=on go run build/ci.go install -buildtags circuit_capacity_checker ./cmd/geth
 
 # Pull Geth into a second stage deploy alpine container
 FROM ubuntu:20.04
 
 RUN apt-get -qq update \
-    && apt-get -qq install -y --no-install-recommends ca-certificates
+    && apt-get -qq install -y --no-install-recommends ca-certificates wget
 
 COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/
-COPY --from=zkp-builder /app/target/release/libzkp.so /usr/local/lib/
-COPY --from=zkp-builder /app/target/release/libzktrie.so /usr/local/lib/
-ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/
+
+ARG LIBSCROLL_ZSTD_VERSION
+ARG SCROLL_LIB_PATH
+
+RUN mkdir -p $SCROLL_LIB_PATH
+
+COPY --from=zkp-builder /app/target/release/libzkp.so $SCROLL_LIB_PATH
+RUN wget -O $SCROLL_LIB_PATH/libscroll_zstd.so https://github.com/scroll-tech/da-codec/releases/download/$LIBSCROLL_ZSTD_VERSION/libscroll_zstd.so
+
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SCROLL_LIB_PATH
+ENV CGO_LDFLAGS="-L$SCROLL_LIB_PATH -Wl,-rpath,$SCROLL_LIB_PATH"
 
 EXPOSE 8545 8546 30303 30303/udp
 ENTRYPOINT ["geth"]

Dockerfile.mockccc

@@ -2,18 +2,41 @@
 ARG COMMIT=""
 ARG VERSION=""
 ARG BUILDNUM=""
+ARG LIBSCROLL_ZSTD_VERSION=v0.1.0-rc0-ubuntu20.04
+ARG SCROLL_LIB_PATH=/scroll/lib
 
 # Build Geth in a stock Go builder container
 FROM scrolltech/go-rust-builder:go-1.20-rust-nightly-2022-12-10 as builder
 
 ADD . /go-ethereum
-RUN cd /go-ethereum && env GO111MODULE=on go run build/ci.go install ./cmd/geth
+
+ARG LIBSCROLL_ZSTD_VERSION
+ARG SCROLL_LIB_PATH
+
+RUN mkdir -p $SCROLL_LIB_PATH
+
+RUN apt-get -qq update && apt-get -qq install -y wget
+RUN wget -O $SCROLL_LIB_PATH/libscroll_zstd.so https://github.com/scroll-tech/da-codec/releases/download/$LIBSCROLL_ZSTD_VERSION/libscroll_zstd.so
+
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SCROLL_LIB_PATH
+ENV CGO_LDFLAGS="-L$SCROLL_LIB_PATH -Wl,-rpath,$SCROLL_LIB_PATH"
 
 # Pull Geth into a second stage deploy alpine container
 FROM ubuntu:20.04
 
 COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/
 
+ARG LIBSCROLL_ZSTD_VERSION
+ARG SCROLL_LIB_PATH
+
+RUN mkdir -p $SCROLL_LIB_PATH
+
+RUN apt-get -qq update && apt-get -qq install -y wget
+RUN wget -O $SCROLL_LIB_PATH/libscroll_zstd.so https://github.com/scroll-tech/da-codec/releases/download/$LIBSCROLL_ZSTD_VERSION/libscroll_zstd.so
+
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SCROLL_LIB_PATH
+ENV CGO_LDFLAGS="-L$SCROLL_LIB_PATH -Wl,-rpath,$SCROLL_LIB_PATH"
+
 EXPOSE 8545 8546 30303 30303/udp
 ENTRYPOINT ["geth"]
 

Makefile

@@ -2,21 +2,27 @@
 # with Go source code. If you know what GOPATH is then you probably
 # don't need to bother with make.
 
-.PHONY: geth android ios evm all test clean libzkp
+.PHONY: geth android ios evm all test clean libzkp libzstd
 
 GOBIN = ./build/bin
 GO ?= latest
-GORUN = env GO111MODULE=on go run
+GORUN = go run
+LIBSCROLL_ZSTD_VERSION = v0.1.0-rc0-ubuntu20.04
+SCROLL_LIB_PATH = /scroll/lib
 
 libzkp:
 	cd $(PWD)/rollup/circuitcapacitychecker/libzkp && make libzkp
 
-nccc_geth: ## geth without circuit capacity checker
-	$(GORUN) build/ci.go install ./cmd/geth
+libzstd:
+	@sudo mkdir -p $(SCROLL_LIB_PATH)
+	@sudo wget -O $(SCROLL_LIB_PATH)/libscroll_zstd.so https://github.com/scroll-tech/da-codec/releases/download/$(LIBSCROLL_ZSTD_VERSION)/libscroll_zstd.so
+
+nccc_geth: libzstd ## geth without circuit capacity checker
+	@LD_LIBRARY_PATH="$(LD_LIBRARY_PATH):$(SCROLL_LIB_PATH)" CGO_LDFLAGS="-L$(SCROLL_LIB_PATH) -Wl,-rpath,$(SCROLL_LIB_PATH)" $(GORUN) build/ci.go install ./cmd/geth
 	@echo "Done building."
 	@echo "Run \"$(GOBIN)/geth\" to launch geth."
 
-geth: libzkp
+geth: libzkp libzstd
 	$(GORUN) build/ci.go install -buildtags circuit_capacity_checker ./cmd/geth
 	@echo "Done building."
 	@echo "Run \"$(GOBIN)/geth\" to launch geth."
@@ -36,17 +42,14 @@ ios:
 	@echo "Done building."
 	@echo "Import \"$(GOBIN)/Geth.framework\" to use the library."
 
-test: all
-	# genesis test
-	cd ${PWD}/cmd/geth; go test -test.run TestCustomGenesis
-	# module test
-	$(GORUN) build/ci.go test ./consensus ./core ./eth ./miner ./node ./trie ./rollup/fees
+test:
+	./run_test.sh
 
 lint: ## Run linters.
 	$(GORUN) build/ci.go lint
 
 clean:
-	env GO111MODULE=on go clean -cache
+	go clean -cache
 	rm -fr build/_workspace/pkg/ $(GOBIN)/*
 
 # The devtools target installs tools required for 'go generate'.

go.mod

@@ -1,18 +1,18 @@
 module github.com/scroll-tech/go-ethereum
 
-go 1.20
+go 1.21
 
 require (
 	github.com/Azure/azure-storage-blob-go v0.7.0
-	github.com/VictoriaMetrics/fastcache v1.6.0
+	github.com/VictoriaMetrics/fastcache v1.12.1
 	github.com/aws/aws-sdk-go-v2 v1.2.0
 	github.com/aws/aws-sdk-go-v2/config v1.1.1
 	github.com/aws/aws-sdk-go-v2/credentials v1.1.1
 	github.com/aws/aws-sdk-go-v2/service/route53 v1.1.1
 	github.com/btcsuite/btcd v0.20.1-beta
 	github.com/cespare/cp v0.1.0
 	github.com/cloudflare/cloudflare-go v0.14.0
-	github.com/consensys/gnark-crypto v0.10.0
+	github.com/consensys/gnark-crypto v0.12.1
 	github.com/crate-crypto/go-kzg-4844 v1.0.0
 	github.com/davecgh/go-spew v1.1.1
 	github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea
@@ -23,9 +23,9 @@ require (
 	github.com/fatih/color v1.7.0
 	github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5
 	github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff
-	github.com/go-stack/stack v1.8.0
+	github.com/go-stack/stack v1.8.1
 	github.com/golang/protobuf v1.4.3
-	github.com/golang/snappy v0.0.4
+	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb
 	github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa
 	github.com/google/uuid v1.1.5
 	github.com/gorilla/websocket v1.4.2
@@ -35,7 +35,7 @@ require (
 	github.com/holiman/bloomfilter/v2 v2.0.3
 	github.com/holiman/uint256 v1.2.4
 	github.com/huin/goupnp v1.0.2
-	github.com/iden3/go-iden3-crypto v0.0.12
+	github.com/iden3/go-iden3-crypto v0.0.15
 	github.com/influxdata/influxdb v1.8.3
 	github.com/influxdata/influxdb-client-go/v2 v2.4.0
 	github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458
@@ -50,16 +50,17 @@ require (
 	github.com/prometheus/tsdb v0.7.1
 	github.com/rjeczalik/notify v0.9.1
 	github.com/rs/cors v1.7.0
+	github.com/scroll-tech/da-codec v0.0.0-20240516115958-db04f5e6772c
 	github.com/scroll-tech/zktrie v0.8.4
-	github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible
+	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4
-	github.com/stretchr/testify v1.8.2
+	github.com/stretchr/testify v1.9.0
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
 	github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef
-	golang.org/x/crypto v0.14.0
-	golang.org/x/sync v0.4.0
-	golang.org/x/sys v0.13.0
-	golang.org/x/text v0.13.0
+	golang.org/x/crypto v0.17.0
+	golang.org/x/sync v0.6.0
+	golang.org/x/sys v0.17.0
+	golang.org/x/text v0.14.0
 	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce
 	gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6
@@ -69,36 +70,37 @@ require (
 require (
 	github.com/Azure/azure-pipeline-go v0.2.2 // indirect
 	github.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect
-	github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.2 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.2 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.1.1 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.1.1 // indirect
 	github.com/aws/smithy-go v1.1.0 // indirect
-	github.com/bits-and-blooms/bitset v1.5.0 // indirect
-	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/bits-and-blooms/bitset v1.12.0 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/consensys/bavard v0.1.13 // indirect
 	github.com/deepmap/oapi-codegen v1.8.2 // indirect
 	github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 // indirect
-	github.com/go-ole/go-ole v1.2.1 // indirect
+	github.com/go-ole/go-ole v1.3.0 // indirect
 	github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
 	github.com/gotestyourself/gotestyourself v1.4.0 // indirect
 	github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d // indirect
-	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/mitchellh/mapstructure v1.4.1 // indirect
 	github.com/mitchellh/pointerstructure v1.2.0 // indirect
 	github.com/mmcloughlin/addchain v0.4.0 // indirect
 	github.com/naoina/go-stringutil v0.1.0 // indirect
 	github.com/opentracing/opentracing-go v1.1.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/supranational/blst v0.3.11-0.20230124161941-ca03e11a3ff2 // indirect
-	github.com/tklauser/go-sysconf v0.3.10 // indirect
-	github.com/tklauser/numcpus v0.4.0 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
+	github.com/yusufpapurcu/wmi v1.2.3 // indirect
 	golang.org/x/net v0.16.0 // indirect
-	golang.org/x/term v0.13.0 // indirect
+	golang.org/x/term v0.15.0 // indirect
 	google.golang.org/protobuf v1.23.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect