forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-08-19 10:22:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

update ccc (#906)

Browse source 
* upgrade ccc

* upgrade Dockerfile
This commit is contained in:
HAOYUatHZ 2024-07-17 21:36:40 +08:00 committed by GitHub
parent 74578fa06e
commit ac2f573c4f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 1271 additions and 1006 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
Dockerfile
View file

@ -19,7 +19,6 @@ RUN cargo chef cook --release --recipe-path recipe.json
COPY ./rollup/circuitcapacitychecker/libzkp .
RUN cargo clean
RUN cargo build --release
RUN find ./ | grep libzktrie.so | xargs -I{} cp {} /app/target/release/
# Build Geth in a stock Go builder container
FROM scrolltech/go-rust-builder:go-1.20-rust-nightly-2022-12-10 as builder
@ -31,7 +30,6 @@ RUN cd /go-ethereum && go mod download
ADD . /go-ethereum
COPY --from=zkp-builder /app/target/release/libzkp.so /usr/local/lib/
COPY --from=zkp-builder /app/target/release/libzktrie.so /usr/local/lib/
ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/
RUN cd /go-ethereum && go run build/ci.go install -tags circuit_capacity_checker ./cmd/geth
@ -43,7 +41,6 @@ RUN apt-get -qq update \
COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/
COPY --from=zkp-builder /app/target/release/libzkp.so /usr/local/lib/
COPY --from=zkp-builder /app/target/release/libzktrie.so /usr/local/lib/
ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/
EXPOSE 8545 8546 30303 30303/udp

87
rollup/circuitcapacitychecker/impl.go
View file

@ -3,7 +3,7 @@
package circuitcapacitychecker
/*
#cgo LDFLAGS: -lm -ldl -lzkp -lzktrie
#cgo LDFLAGS: -lm -ldl -lzkp
#include <stdlib.h>
#include "./libzkp/libzkp.h"
*/
@ -14,14 +14,19 @@ import (
"encoding/json"
"fmt"
"sync"
"time"
"unsafe"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
)
// mutex for concurrent CircuitCapacityChecker creations
var creationMu sync.Mutex
var (
creationMu sync.Mutex
encodeTimer = metrics.NewRegisteredTimer("ccc/encode", nil)
)
func init() {
C.init()
@ -67,39 +72,43 @@ func (ccc *CircuitCapacityChecker) ApplyTransaction(traces *types.BlockTrace) (*
return nil, ErrUnknown
}
ccc.jsonBuffer.Reset()
err := json.NewEncoder(&ccc.jsonBuffer).Encode(traces)
if err != nil {
log.Error("fail to json marshal traces in ApplyTransaction", "id", ccc.ID, "TxHash", traces.Transactions[0].TxHash, "err", err)
encodeStart := time.Now()
rustTrace := MakeRustTrace(traces, &ccc.jsonBuffer)
if rustTrace == nil {
log.Error("fail to parse json in to rust trace", "id", ccc.ID, "TxHash", traces.Transactions[0].TxHash)
return nil, ErrUnknown
}
tracesStr := C.CString(string(ccc.jsonBuffer.Bytes()))
defer func() {
C.free(unsafe.Pointer(tracesStr))
}()
encodeTimer.UpdateSince(encodeStart)
log.Debug("start to check circuit capacity for tx", "id", ccc.ID, "TxHash", traces.Transactions[0].TxHash)
rawResult := C.apply_tx(C.uint64_t(ccc.ID), tracesStr)
return ccc.applyTransactionRustTrace(rustTrace)
}
func (ccc *CircuitCapacityChecker) ApplyTransactionRustTrace(rustTrace unsafe.Pointer) (*types.RowConsumption, error) {
ccc.Lock()
defer ccc.Unlock()
return ccc.applyTransactionRustTrace(rustTrace)
}
func (ccc *CircuitCapacityChecker) applyTransactionRustTrace(rustTrace unsafe.Pointer) (*types.RowConsumption, error) {
rawResult := C.apply_tx(C.uint64_t(ccc.ID), rustTrace)
defer func() {
C.free_c_chars(rawResult)
}()
log.Debug("check circuit capacity for tx done", "id", ccc.ID, "TxHash", traces.Transactions[0].TxHash)
result := &WrappedRowUsage{}
if err = json.Unmarshal([]byte(C.GoString(rawResult)), result); err != nil {
log.Error("fail to json unmarshal apply_tx result", "id", ccc.ID, "TxHash", traces.Transactions[0].TxHash, "err", err)
if err := json.Unmarshal([]byte(C.GoString(rawResult)), result); err != nil {
log.Error("fail to json unmarshal apply_tx result", "id", ccc.ID, "err", err)
return nil, ErrUnknown
}
if result.Error != "" {
log.Error("fail to apply_tx in CircuitCapacityChecker", "id", ccc.ID, "TxHash", traces.Transactions[0].TxHash, "err", result.Error)
log.Error("fail to apply_tx in CircuitCapacityChecker", "id", ccc.ID, "err", result.Error)
return nil, ErrUnknown
}
if result.AccRowUsage == nil {
log.Error("fail to apply_tx in CircuitCapacityChecker",
"id", ccc.ID, "TxHash", traces.Transactions[0].TxHash,
"result.AccRowUsage == nil", result.AccRowUsage == nil,
"id", ccc.ID, "result.AccRowUsage == nil", result.AccRowUsage == nil,
"err", "AccRowUsage is empty unexpectedly")
return nil, ErrUnknown
}
@ -114,27 +123,23 @@ func (ccc *CircuitCapacityChecker) ApplyBlock(traces *types.BlockTrace) (*types.
ccc.Lock()
defer ccc.Unlock()
ccc.jsonBuffer.Reset()
err := json.NewEncoder(&ccc.jsonBuffer).Encode(traces)
if err != nil {
log.Error("fail to json marshal traces in ApplyBlock", "id", ccc.ID, "blockNumber", traces.Header.Number, "blockHash", traces.Header.Hash(), "err", err)
encodeStart := time.Now()
rustTrace := MakeRustTrace(traces, &ccc.jsonBuffer)
if rustTrace == nil {
log.Error("fail to parse json in to rust trace", "id", ccc.ID, "TxHash", traces.Transactions[0].TxHash)
return nil, ErrUnknown
}
tracesStr := C.CString(string(ccc.jsonBuffer.Bytes()))
defer func() {
C.free(unsafe.Pointer(tracesStr))
}()
encodeTimer.UpdateSince(encodeStart)
log.Debug("start to check circuit capacity for block", "id", ccc.ID, "blockNumber", traces.Header.Number, "blockHash", traces.Header.Hash())
rawResult := C.apply_block(C.uint64_t(ccc.ID), tracesStr)
rawResult := C.apply_block(C.uint64_t(ccc.ID), rustTrace)
defer func() {
C.free_c_chars(rawResult)
}()
log.Debug("check circuit capacity for block done", "id", ccc.ID, "blockNumber", traces.Header.Number, "blockHash", traces.Header.Hash())
result := &WrappedRowUsage{}
if err = json.Unmarshal([]byte(C.GoString(rawResult)), result); err != nil {
if err := json.Unmarshal([]byte(C.GoString(rawResult)), result); err != nil {
log.Error("fail to json unmarshal apply_block result", "id", ccc.ID, "blockNumber", traces.Header.Number, "blockHash", traces.Header.Hash(), "err", err)
return nil, ErrUnknown
}
@ -198,3 +203,27 @@ func (ccc *CircuitCapacityChecker) SetLightMode(lightMode bool) error {
return nil
}
func MakeRustTrace(trace *types.BlockTrace, buffer *bytes.Buffer) unsafe.Pointer {
if buffer == nil {
buffer = new(bytes.Buffer)
}
buffer.Reset()
err := json.NewEncoder(buffer).Encode(trace)
if err != nil {
log.Error("fail to json marshal traces in MakeRustTrace", "err", err)
return nil
}
tracesStr := C.CString(string(buffer.Bytes()))
defer func() {
C.free(unsafe.Pointer(tracesStr))
}()
return C.parse_json_to_rust_trace(tracesStr)
}
func FreeRustTrace(ptr unsafe.Pointer) {
C.free_rust_trace(ptr)
}

2066
rollup/circuitcapacitychecker/libzkp/Cargo.lock generated
View file

File diff suppressed because it is too large Load diff

28
rollup/circuitcapacitychecker/libzkp/Cargo.toml
View file

@ -8,33 +8,35 @@ edition = "2021"
crate-type = ["cdylib"]
[patch.crates-io]
gobuild = { git = "https://github.com/scroll-tech/gobuild.git" }
halo2curves = { git = "https://github.com/scroll-tech/halo2curves", branch = "v0.1.0" }
ethers-core = { git = "https://github.com/scroll-tech/ethers-rs.git", branch = "v2.0.7" }
ethers-providers = { git = "https://github.com/scroll-tech/ethers-rs.git", branch = "v2.0.7" }
ethers-signers = { git = "https://github.com/scroll-tech/ethers-rs.git", branch = "v2.0.7" }
#ethers-etherscan = { git = "https://github.com/scroll-tech/ethers-rs.git", branch = "v2.0.7" }
#ethers = { git = "https://github.com/scroll-tech/ethers-rs.git", branch = "v2.0.7" }
[patch."https://github.com/privacy-scaling-explorations/halo2.git"]
halo2_proofs = { git = "https://github.com/scroll-tech/halo2.git", branch = "develop" }
halo2_proofs = { git = "https://github.com/scroll-tech/halo2.git", branch = "v1.1" }
[patch."https://github.com/privacy-scaling-explorations/poseidon.git"]
poseidon = { git = "https://github.com/scroll-tech/poseidon.git", branch = "scroll-dev-0220" }
[patch."https://github.com/privacy-scaling-explorations/halo2wrong.git"]
halo2wrong = { git = "https://github.com/scroll-tech/halo2wrong.git", branch = "halo2-ecc-snark-verifier-0323" }
maingate = { git = "https://github.com/scroll-tech/halo2wrong", branch = "halo2-ecc-snark-verifier-0323" }
[patch."https://github.com/privacy-scaling-explorations/halo2curves.git"]
halo2curves = { git = "https://github.com/scroll-tech/halo2curves.git", branch = "0.3.1-derive-serde" }
poseidon = { git = "https://github.com/scroll-tech/poseidon.git", branch = "main" }
[patch."https://github.com/privacy-scaling-explorations/bls12_381"]
bls12_381 = { git = "https://github.com/scroll-tech/bls12_381", branch = "feat/impl_scalar_field" }
[dependencies]
prover = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.9.7", default-features = false, features = ["parallel_syn", "scroll", "shanghai"] }
prover = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.11.4", default-features = false, features = ["parallel_syn", "scroll", "strict-ccc"] }
anyhow = "1.0"
log = "0.4"
base64 = "0.13.0"
env_logger = "0.9.0"
libc = "0.2"
log = "0.4"
once_cell = "1.19"
serde = "1.0"
serde_derive = "1.0"
serde_json = "1.0.66"
libc = "0.2"
once_cell = "1.8.0"
[profile.test]
opt-level = 3
debug-assertions = true
[profile.release]
opt-level = 3

1
rollup/circuitcapacitychecker/libzkp/Makefile
View file

@ -7,4 +7,3 @@ clean:
libzkp:
cargo build --release
cp $(PWD)/target/release/libzkp.so $(PWD)/
find $(PWD)/target | grep libzktrie.so | xargs -I{} cp {} $(PWD)/

6
rollup/circuitcapacitychecker/libzkp/libzkp.h
View file

@ -4,8 +4,10 @@
void init();
uint64_t new_circuit_capacity_checker();
void reset_circuit_capacity_checker(uint64_t id);
char* apply_tx(uint64_t id, char *tx_traces);
char* apply_block(uint64_t id, char *block_trace);
char* apply_tx(uint64_t id, void* tx_traces);
char* apply_block(uint64_t id, void* block_trace);
char* get_tx_num(uint64_t id);
char* set_light_mode(uint64_t id, bool light_mode);
void free_c_chars(char* ptr);
void* parse_json_to_rust_trace(char* trace_json_ptr);
void free_rust_trace(void* trace_ptr);

2
rollup/circuitcapacitychecker/libzkp/rust-toolchain
View file

@ -1 +1 @@
nightly-2022-12-10
nightly-2023-12-03

52
rollup/circuitcapacitychecker/libzkp/src/lib.rs
View file

@ -1,7 +1,5 @@
#![feature(once_cell)]
pub mod checker {
use crate::utils::{c_char_to_str, c_char_to_vec, vec_to_c_char};
use crate::utils::vec_to_c_char;
use anyhow::{anyhow, bail, Error};
use libc::c_char;
use prover::{
@ -9,10 +7,11 @@ pub mod checker {
BlockTrace,
};
use serde_derive::{Deserialize, Serialize};
use std::cell::OnceCell;
use std::{cell::OnceCell, ptr::null_mut};
use std::collections::HashMap;
use std::panic;
use std::ptr::null;
use std::ffi::CStr;
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct CommonResult {
@ -45,6 +44,17 @@ pub mod checker {
.expect("circuit capacity checker initialized twice");
}
/// # Safety
#[no_mangle]
pub unsafe extern "C" fn parse_json_to_rust_trace(trace_json_ptr: *const c_char) -> *mut BlockTrace {
let trace_json_cstr = unsafe { CStr::from_ptr(trace_json_ptr) };
let trace = serde_json::from_slice::<BlockTrace>(trace_json_cstr.to_bytes());
match trace {
Err(_) => return null_mut(),
Ok(t) => return Box::into_raw(Box::new(t))
}
}
/// # Safety
#[no_mangle]
pub unsafe extern "C" fn new_circuit_capacity_checker() -> u64 {
@ -70,8 +80,9 @@ pub mod checker {
/// # Safety
#[no_mangle]
pub unsafe extern "C" fn apply_tx(id: u64, tx_traces: *const c_char) -> *const c_char {
let result = apply_tx_inner(id, tx_traces);
pub unsafe extern "C" fn apply_tx(id: u64, trace_ptr: *mut BlockTrace) -> *const c_char {
let trace = Box::from_raw(trace_ptr);
let result = apply_tx_inner(id, *trace);
let r = match result {
Ok(acc_row_usage) => {
log::debug!(
@ -92,14 +103,12 @@ pub mod checker {
serde_json::to_vec(&r).map_or(null(), vec_to_c_char)
}
unsafe fn apply_tx_inner(id: u64, tx_traces: *const c_char) -> Result<RowUsage, Error> {
unsafe fn apply_tx_inner(id: u64, traces: BlockTrace) -> Result<RowUsage, Error> {
log::debug!(
"ccc apply_tx raw input, id: {:?}, tx_traces: {:?}",
id,
c_char_to_str(tx_traces)?
traces
);
let tx_traces_vec = c_char_to_vec(tx_traces);
let traces = serde_json::from_slice::<BlockTrace>(&tx_traces_vec)?;
if traces.transactions.len() != 1 {
bail!("traces.transactions.len() != 1");
@ -121,7 +130,7 @@ pub mod checker {
.ok_or(anyhow!(
"fail to get circuit capacity checker (id: {id:?}) in apply_tx"
))?
.estimate_circuit_capacity(&[traces])
.estimate_circuit_capacity(traces)
});
match r {
Ok(result) => result,
@ -133,8 +142,9 @@ pub mod checker {
/// # Safety
#[no_mangle]
pub unsafe extern "C" fn apply_block(id: u64, block_trace: *const c_char) -> *const c_char {
let result = apply_block_inner(id, block_trace);
pub unsafe extern "C" fn apply_block(id: u64, trace_ptr: *mut BlockTrace) -> *const c_char {
let trace = Box::from_raw(trace_ptr);
let result = apply_block_inner(id, *trace);
let r = match result {
Ok(acc_row_usage) => {
log::debug!(
@ -155,14 +165,12 @@ pub mod checker {
serde_json::to_vec(&r).map_or(null(), vec_to_c_char)
}
unsafe fn apply_block_inner(id: u64, block_trace: *const c_char) -> Result<RowUsage, Error> {
unsafe fn apply_block_inner(id: u64, traces: BlockTrace) -> Result<RowUsage, Error> {
log::debug!(
"ccc apply_block raw input, id: {:?}, block_trace: {:?}",
id,
c_char_to_str(block_trace)?
traces
);
let block_trace = c_char_to_vec(block_trace);
let traces = serde_json::from_slice::<BlockTrace>(&block_trace)?;
let r = panic::catch_unwind(|| {
CHECKERS
@ -174,7 +182,7 @@ pub mod checker {
.ok_or(anyhow!(
"fail to get circuit capacity checker (id: {id:?}) in apply_block"
))?
.estimate_circuit_capacity(&[traces])
.estimate_circuit_capacity(traces)
});
match r {
Ok(result) => result,
@ -263,6 +271,7 @@ pub mod utils {
use std::ffi::{CStr, CString};
use std::os::raw::c_char;
use std::str::Utf8Error;
use prover::BlockTrace;
/// # Safety
#[no_mangle]
@ -275,6 +284,13 @@ pub mod utils {
let _ = CString::from_raw(ptr);
}
/// # Safety
#[no_mangle]
pub unsafe extern "C" fn free_rust_trace(trace_ptr: *mut BlockTrace) {
let _ = Box::from_raw(trace_ptr);
}
#[allow(dead_code)]
pub(crate) fn c_char_to_str(c: *const c_char) -> Result<&'static str, Utf8Error> {
let cstr = unsafe { CStr::from_ptr(c) };

32
rollup/circuitcapacitychecker/mock.go
View file

@ -3,8 +3,11 @@
package circuitcapacitychecker
import (
"bytes"
"math/rand"
"unsafe"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
)
@ -12,6 +15,9 @@ type CircuitCapacityChecker struct {
ID uint64
countdown int
nextError *error
skipHash string
skipError error
}
// NewCircuitCapacityChecker creates a new CircuitCapacityChecker
@ -36,12 +42,21 @@ func (ccc *CircuitCapacityChecker) ApplyTransaction(traces *types.BlockTrace) (*
return nil, err
}
}
if ccc.skipError != nil {
if traces.Transactions[0].TxHash == ccc.skipHash {
return nil, ccc.skipError
}
}
return &types.RowConsumption{types.SubCircuitRowUsage{
Name: "mock",
RowNumber: 1,
}}, nil
}
func (ccc *CircuitCapacityChecker) ApplyTransactionRustTrace(rustTrace unsafe.Pointer) (*types.RowConsumption, error) {
return ccc.ApplyTransaction(goTraces[rustTrace])
}
// ApplyBlock gets a block's RowConsumption.
// Will only return a dummy value in mock_ccc.
func (ccc *CircuitCapacityChecker) ApplyBlock(traces *types.BlockTrace) (*types.RowConsumption, error) {
@ -67,3 +82,20 @@ func (ccc *CircuitCapacityChecker) ScheduleError(cnt int, err error) {
ccc.countdown = cnt
ccc.nextError = &err
}
// Skip forced CCC to return always an error for a given txn
func (ccc *CircuitCapacityChecker) Skip(txnHash common.Hash, err error) {
ccc.skipHash = txnHash.String()
ccc.skipError = err
}
var goTraces = make(map[unsafe.Pointer]*types.BlockTrace)
func MakeRustTrace(trace *types.BlockTrace, buffer *bytes.Buffer) unsafe.Pointer {
rustTrace := new(struct{})
goTraces[unsafe.Pointer(rustTrace)] = trace
return unsafe.Pointer(rustTrace)
}
func FreeRustTrace(ptr unsafe.Pointer) {
}