rollup: add rollup/circuitcapacitychecker package (#544)

This commit is contained in:
HAOYUatHZ 2023-10-25 15:55:54 +08:00 committed by GitHub
parent a6ac896a2e
commit f4743ad29b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 4937 additions and 0 deletions

View file

@ -0,0 +1,196 @@
//go:build circuit_capacity_checker
package circuitcapacitychecker
/*
#cgo LDFLAGS: -lm -ldl -lzkp -lzktrie
#include <stdlib.h>
#include "./libzkp/libzkp.h"
*/
import "C" //nolint:typecheck
import (
"encoding/json"
"fmt"
"sync"
"unsafe"
"github.com/scroll-tech/go-ethereum/core/types"
"github.com/scroll-tech/go-ethereum/log"
)
// mutex for concurrent CircuitCapacityChecker creations
var creationMu sync.Mutex
func init() {
C.init()
}
type CircuitCapacityChecker struct {
// mutex for each CircuitCapacityChecker itself
sync.Mutex
ID uint64
}
// NewCircuitCapacityChecker creates a new CircuitCapacityChecker
func NewCircuitCapacityChecker(lightMode bool) *CircuitCapacityChecker {
creationMu.Lock()
defer creationMu.Unlock()
id := C.new_circuit_capacity_checker()
ccc := &CircuitCapacityChecker{ID: uint64(id)}
ccc.SetLightMode(lightMode)
return ccc
}
// Reset resets a CircuitCapacityChecker
func (ccc *CircuitCapacityChecker) Reset() {
ccc.Lock()
defer ccc.Unlock()
C.reset_circuit_capacity_checker(C.uint64_t(ccc.ID))
}
// ApplyTransaction appends a tx's wrapped BlockTrace into the ccc, and return the accumulated RowConsumption
func (ccc *CircuitCapacityChecker) ApplyTransaction(traces *types.BlockTrace) (*types.RowConsumption, error) {
ccc.Lock()
defer ccc.Unlock()
if len(traces.Transactions) != 1 || len(traces.ExecutionResults) != 1 || len(traces.TxStorageTraces) != 1 {
log.Error("malformatted BlockTrace in ApplyTransaction", "id", ccc.ID,
"len(traces.Transactions)", len(traces.Transactions),
"len(traces.ExecutionResults)", len(traces.ExecutionResults),
"len(traces.TxStorageTraces)", len(traces.TxStorageTraces),
"err", "length of Transactions, or ExecutionResults, or TxStorageTraces, is not equal to 1")
return nil, ErrUnknown
}
tracesByt, err := json.Marshal(traces)
if err != nil {
log.Error("fail to json marshal traces in ApplyTransaction", "id", ccc.ID, "TxHash", traces.Transactions[0].TxHash, "err", err)
return nil, ErrUnknown
}
tracesStr := C.CString(string(tracesByt))
defer func() {
C.free(unsafe.Pointer(tracesStr))
}()
log.Debug("start to check circuit capacity for tx", "id", ccc.ID, "TxHash", traces.Transactions[0].TxHash)
rawResult := C.apply_tx(C.uint64_t(ccc.ID), tracesStr)
defer func() {
C.free_c_chars(rawResult)
}()
log.Debug("check circuit capacity for tx done", "id", ccc.ID, "TxHash", traces.Transactions[0].TxHash)
result := &WrappedRowUsage{}
if err = json.Unmarshal([]byte(C.GoString(rawResult)), result); err != nil {
log.Error("fail to json unmarshal apply_tx result", "id", ccc.ID, "TxHash", traces.Transactions[0].TxHash, "err", err)
return nil, ErrUnknown
}
if result.Error != "" {
log.Error("fail to apply_tx in CircuitCapacityChecker", "id", ccc.ID, "TxHash", traces.Transactions[0].TxHash, "err", result.Error)
return nil, ErrUnknown
}
if result.AccRowUsage == nil {
log.Error("fail to apply_tx in CircuitCapacityChecker",
"id", ccc.ID, "TxHash", traces.Transactions[0].TxHash,
"result.AccRowUsage == nil", result.AccRowUsage == nil,
"err", "AccRowUsage is empty unexpectedly")
return nil, ErrUnknown
}
if !result.AccRowUsage.IsOk {
return nil, ErrBlockRowConsumptionOverflow
}
return (*types.RowConsumption)(&result.AccRowUsage.RowUsageDetails), nil
}
// ApplyBlock gets a block's RowConsumption
func (ccc *CircuitCapacityChecker) ApplyBlock(traces *types.BlockTrace) (*types.RowConsumption, error) {
ccc.Lock()
defer ccc.Unlock()
tracesByt, err := json.Marshal(traces)
if err != nil {
log.Error("fail to json marshal traces in ApplyBlock", "id", ccc.ID, "blockNumber", traces.Header.Number, "blockHash", traces.Header.Hash(), "err", err)
return nil, ErrUnknown
}
tracesStr := C.CString(string(tracesByt))
defer func() {
C.free(unsafe.Pointer(tracesStr))
}()
log.Debug("start to check circuit capacity for block", "id", ccc.ID, "blockNumber", traces.Header.Number, "blockHash", traces.Header.Hash())
rawResult := C.apply_block(C.uint64_t(ccc.ID), tracesStr)
defer func() {
C.free_c_chars(rawResult)
}()
log.Debug("check circuit capacity for block done", "id", ccc.ID, "blockNumber", traces.Header.Number, "blockHash", traces.Header.Hash())
result := &WrappedRowUsage{}
if err = json.Unmarshal([]byte(C.GoString(rawResult)), result); err != nil {
log.Error("fail to json unmarshal apply_block result", "id", ccc.ID, "blockNumber", traces.Header.Number, "blockHash", traces.Header.Hash(), "err", err)
return nil, ErrUnknown
}
if result.Error != "" {
log.Error("fail to apply_block in CircuitCapacityChecker", "id", ccc.ID, "blockNumber", traces.Header.Number, "blockHash", traces.Header.Hash(), "err", result.Error)
return nil, ErrUnknown
}
if result.AccRowUsage == nil {
log.Error("fail to apply_block in CircuitCapacityChecker", "id", ccc.ID, "blockNumber", traces.Header.Number, "blockHash", traces.Header.Hash(), "err", "AccRowUsage is empty unexpectedly")
return nil, ErrUnknown
}
if !result.AccRowUsage.IsOk {
return nil, ErrBlockRowConsumptionOverflow
}
return (*types.RowConsumption)(&result.AccRowUsage.RowUsageDetails), nil
}
// CheckTxNum compares whether the tx_count in ccc match the expected
func (ccc *CircuitCapacityChecker) CheckTxNum(expected int) (bool, uint64, error) {
ccc.Lock()
defer ccc.Unlock()
log.Debug("ccc get_tx_num start", "id", ccc.ID)
rawResult := C.get_tx_num(C.uint64_t(ccc.ID))
defer func() {
C.free_c_chars(rawResult)
}()
log.Debug("ccc get_tx_num end", "id", ccc.ID)
result := &WrappedTxNum{}
if err := json.Unmarshal([]byte(C.GoString(rawResult)), result); err != nil {
return false, 0, fmt.Errorf("fail to json unmarshal get_tx_num result, id: %d, err: %w", ccc.ID, err)
}
if result.Error != "" {
return false, 0, fmt.Errorf("fail to get_tx_num in CircuitCapacityChecker, id: %d, err: %w", ccc.ID, result.Error)
}
return result.TxNum == uint64(expected), result.TxNum, nil
}
// SetLightMode sets to ccc light mode
func (ccc *CircuitCapacityChecker) SetLightMode(lightMode bool) error {
ccc.Lock()
defer ccc.Unlock()
log.Debug("ccc set_light_mode start", "id", ccc.ID)
rawResult := C.set_light_mode(C.uint64_t(ccc.ID), C.bool(lightMode))
defer func() {
C.free_c_chars(rawResult)
}()
log.Debug("ccc set_light_mode end", "id", ccc.ID)
result := &WrappedCommonResult{}
if err := json.Unmarshal([]byte(C.GoString(rawResult)), result); err != nil {
return fmt.Errorf("fail to json unmarshal set_light_mode result, id: %d, err: %w", ccc.ID, err)
}
if result.Error != "" {
return fmt.Errorf("fail to set_light_mode in CircuitCapacityChecker, id: %d, err: %w", ccc.ID, result.Error)
}
return nil
}

View file

@ -0,0 +1,3 @@
target/
*.a
*.so

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,40 @@
[package]
name = "zkp"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[lib]
crate-type = ["cdylib"]
[patch.crates-io]
ethers-core = { git = "https://github.com/scroll-tech/ethers-rs.git", branch = "v2.0.7" }
[patch."https://github.com/privacy-scaling-explorations/halo2.git"]
halo2_proofs = { git = "https://github.com/scroll-tech/halo2.git", branch = "develop" }
[patch."https://github.com/privacy-scaling-explorations/poseidon.git"]
poseidon = { git = "https://github.com/scroll-tech/poseidon.git", branch = "scroll-dev-0220" }
[patch."https://github.com/privacy-scaling-explorations/halo2wrong.git"]
halo2wrong = { git = "https://github.com/scroll-tech/halo2wrong.git", branch = "halo2-ecc-snark-verifier-0323" }
maingate = { git = "https://github.com/scroll-tech/halo2wrong", branch = "halo2-ecc-snark-verifier-0323" }
[patch."https://github.com/privacy-scaling-explorations/halo2curves.git"]
halo2curves = { git = "https://github.com/scroll-tech/halo2curves.git", branch = "0.3.1-derive-serde" }
[dependencies]
prover = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.9.7", default-features = false, features = ["parallel_syn", "scroll", "shanghai"] }
anyhow = "1.0"
log = "0.4"
env_logger = "0.9.0"
serde = "1.0"
serde_derive = "1.0"
serde_json = "1.0.66"
libc = "0.2"
once_cell = "1.8.0"
[profile.test]
opt-level = 3
debug-assertions = true
[profile.release]
opt-level = 3

View file

@ -0,0 +1,10 @@
.PHONY: clean libzkp
clean:
rm -f *.a *.so
cargo clean
libzkp:
cargo build --release
cp $(PWD)/target/release/libzkp.so $(PWD)/
find $(PWD)/target | grep libzktrie.so | xargs -I{} cp {} $(PWD)/

View file

@ -0,0 +1,11 @@
#include <stdbool.h>
#include<stdint.h>
void init();
uint64_t new_circuit_capacity_checker();
void reset_circuit_capacity_checker(uint64_t id);
char* apply_tx(uint64_t id, char *tx_traces);
char* apply_block(uint64_t id, char *block_trace);
char* get_tx_num(uint64_t id);
char* set_light_mode(uint64_t id, bool light_mode);
void free_c_chars(char* ptr);

View file

@ -0,0 +1 @@
nightly-2022-12-10

View file

@ -0,0 +1,304 @@
#![feature(once_cell)]
pub mod checker {
use crate::utils::{c_char_to_str, c_char_to_vec, vec_to_c_char};
use anyhow::{anyhow, bail, Error};
use libc::c_char;
use prover::{
zkevm::{CircuitCapacityChecker, RowUsage},
BlockTrace,
};
use serde_derive::{Deserialize, Serialize};
use std::cell::OnceCell;
use std::collections::HashMap;
use std::panic;
use std::ptr::null;
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct CommonResult {
pub error: Option<String>,
}
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct RowUsageResult {
pub acc_row_usage: Option<RowUsage>,
pub error: Option<String>,
}
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct TxNumResult {
pub tx_num: u64,
pub error: Option<String>,
}
static mut CHECKERS: OnceCell<HashMap<u64, CircuitCapacityChecker>> = OnceCell::new();
/// # Safety
#[no_mangle]
pub unsafe extern "C" fn init() {
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("debug"))
.format_timestamp_millis()
.init();
let checkers = HashMap::new();
CHECKERS
.set(checkers)
.expect("circuit capacity checker initialized twice");
}
/// # Safety
#[no_mangle]
pub unsafe extern "C" fn new_circuit_capacity_checker() -> u64 {
let checkers = CHECKERS
.get_mut()
.expect("fail to get circuit capacity checkers map in new_circuit_capacity_checker");
let id = checkers.len() as u64;
let checker = CircuitCapacityChecker::new();
checkers.insert(id, checker);
id
}
/// # Safety
#[no_mangle]
pub unsafe extern "C" fn reset_circuit_capacity_checker(id: u64) {
CHECKERS
.get_mut()
.expect("fail to get circuit capacity checkers map in reset_circuit_capacity_checker")
.get_mut(&id)
.unwrap_or_else(|| panic!("fail to get circuit capacity checker (id: {id:?}) in reset_circuit_capacity_checker"))
.reset()
}
/// # Safety
#[no_mangle]
pub unsafe extern "C" fn apply_tx(id: u64, tx_traces: *const c_char) -> *const c_char {
let result = apply_tx_inner(id, tx_traces);
let r = match result {
Ok(acc_row_usage) => {
log::debug!(
"id: {:?}, acc_row_usage: {:?}",
id,
acc_row_usage.row_number,
);
RowUsageResult {
acc_row_usage: Some(acc_row_usage),
error: None,
}
}
Err(e) => RowUsageResult {
acc_row_usage: None,
error: Some(format!("{e:?}")),
},
};
serde_json::to_vec(&r).map_or(null(), vec_to_c_char)
}
unsafe fn apply_tx_inner(id: u64, tx_traces: *const c_char) -> Result<RowUsage, Error> {
log::debug!(
"ccc apply_tx raw input, id: {:?}, tx_traces: {:?}",
id,
c_char_to_str(tx_traces)?
);
let tx_traces_vec = c_char_to_vec(tx_traces);
let traces = serde_json::from_slice::<BlockTrace>(&tx_traces_vec)?;
if traces.transactions.len() != 1 {
bail!("traces.transactions.len() != 1");
}
if traces.execution_results.len() != 1 {
bail!("traces.execution_results.len() != 1");
}
if traces.tx_storage_trace.len() != 1 {
bail!("traces.tx_storage_trace.len() != 1");
}
let r = panic::catch_unwind(|| {
CHECKERS
.get_mut()
.ok_or(anyhow!(
"fail to get circuit capacity checkers map in apply_tx"
))?
.get_mut(&id)
.ok_or(anyhow!(
"fail to get circuit capacity checker (id: {id:?}) in apply_tx"
))?
.estimate_circuit_capacity(&[traces])
});
match r {
Ok(result) => result,
Err(e) => {
bail!("estimate_circuit_capacity (id: {id:?}) error in apply_tx, error: {e:?}")
}
}
}
/// # Safety
#[no_mangle]
pub unsafe extern "C" fn apply_block(id: u64, block_trace: *const c_char) -> *const c_char {
let result = apply_block_inner(id, block_trace);
let r = match result {
Ok(acc_row_usage) => {
log::debug!(
"id: {:?}, acc_row_usage: {:?}",
id,
acc_row_usage.row_number,
);
RowUsageResult {
acc_row_usage: Some(acc_row_usage),
error: None,
}
}
Err(e) => RowUsageResult {
acc_row_usage: None,
error: Some(format!("{e:?}")),
},
};
serde_json::to_vec(&r).map_or(null(), vec_to_c_char)
}
unsafe fn apply_block_inner(id: u64, block_trace: *const c_char) -> Result<RowUsage, Error> {
log::debug!(
"ccc apply_block raw input, id: {:?}, block_trace: {:?}",
id,
c_char_to_str(block_trace)?
);
let block_trace = c_char_to_vec(block_trace);
let traces = serde_json::from_slice::<BlockTrace>(&block_trace)?;
let r = panic::catch_unwind(|| {
CHECKERS
.get_mut()
.ok_or(anyhow!(
"fail to get circuit capacity checkers map in apply_block"
))?
.get_mut(&id)
.ok_or(anyhow!(
"fail to get circuit capacity checker (id: {id:?}) in apply_block"
))?
.estimate_circuit_capacity(&[traces])
});
match r {
Ok(result) => result,
Err(e) => {
bail!("estimate_circuit_capacity (id: {id:?}) error in apply_block, error: {e:?}")
}
}
}
/// # Safety
#[no_mangle]
pub unsafe extern "C" fn get_tx_num(id: u64) -> *const c_char {
let result = get_tx_num_inner(id);
let r = match result {
Ok(tx_num) => {
log::debug!("id: {id}, tx_num: {tx_num}");
TxNumResult {
tx_num,
error: None,
}
}
Err(e) => TxNumResult {
tx_num: 0,
error: Some(format!("{e:?}")),
},
};
serde_json::to_vec(&r).map_or(null(), vec_to_c_char)
}
unsafe fn get_tx_num_inner(id: u64) -> Result<u64, Error> {
log::debug!("ccc get_tx_num raw input, id: {id}");
panic::catch_unwind(|| {
Ok(CHECKERS
.get_mut()
.ok_or(anyhow!(
"fail to get circuit capacity checkers map in get_tx_num"
))?
.get_mut(&id)
.ok_or(anyhow!(
"fail to get circuit capacity checker (id: {id}) in get_tx_num"
))?
.get_tx_num() as u64)
})
.map_or_else(
|e| bail!("circuit capacity checker (id: {id}) error in get_tx_num: {e:?}"),
|result| result,
)
}
/// # Safety
#[no_mangle]
pub unsafe extern "C" fn set_light_mode(id: u64, light_mode: bool) -> *const c_char {
let result = set_light_mode_inner(id, light_mode);
let r = match result {
Ok(()) => CommonResult { error: None },
Err(e) => CommonResult {
error: Some(format!("{e:?}")),
},
};
serde_json::to_vec(&r).map_or(null(), vec_to_c_char)
}
unsafe fn set_light_mode_inner(id: u64, light_mode: bool) -> Result<(), Error> {
log::debug!("ccc set_light_mode raw input, id: {id}");
panic::catch_unwind(|| {
CHECKERS
.get_mut()
.ok_or(anyhow!(
"fail to get circuit capacity checkers map in set_light_mode"
))?
.get_mut(&id)
.ok_or(anyhow!(
"fail to get circuit capacity checker (id: {id}) in set_light_mode"
))?
.set_light_mode(light_mode);
Ok(())
})
.map_or_else(
|e| bail!("circuit capacity checker (id: {id}) error in set_light_mode: {e:?}"),
|result| result,
)
}
}
pub mod utils {
use std::ffi::{CStr, CString};
use std::os::raw::c_char;
use std::str::Utf8Error;
/// # Safety
#[no_mangle]
pub unsafe extern "C" fn free_c_chars(ptr: *mut c_char) {
if ptr.is_null() {
log::warn!("Try to free an empty pointer!");
return;
}
let _ = CString::from_raw(ptr);
}
#[allow(dead_code)]
pub(crate) fn c_char_to_str(c: *const c_char) -> Result<&'static str, Utf8Error> {
let cstr = unsafe { CStr::from_ptr(c) };
cstr.to_str()
}
#[allow(dead_code)]
pub(crate) fn c_char_to_vec(c: *const c_char) -> Vec<u8> {
let cstr = unsafe { CStr::from_ptr(c) };
cstr.to_bytes().to_vec()
}
#[allow(dead_code)]
pub(crate) fn vec_to_c_char(bytes: Vec<u8>) -> *const c_char {
CString::new(bytes)
.expect("fail to create new CString from bytes")
.into_raw()
}
#[allow(dead_code)]
pub(crate) fn bool_to_int(b: bool) -> u8 {
match b {
true => 1,
false => 0,
}
}
}

View file

@ -0,0 +1,69 @@
//go:build !circuit_capacity_checker
package circuitcapacitychecker
import (
"math/rand"
"github.com/scroll-tech/go-ethereum/core/types"
)
type CircuitCapacityChecker struct {
ID uint64
countdown int
nextError *error
}
// NewCircuitCapacityChecker creates a new CircuitCapacityChecker
func NewCircuitCapacityChecker(lightMode bool) *CircuitCapacityChecker {
ccc := &CircuitCapacityChecker{ID: rand.Uint64()}
ccc.SetLightMode(lightMode)
return ccc
}
// Reset resets a ccc, but need to do nothing in mock_ccc.
func (ccc *CircuitCapacityChecker) Reset() {
}
// ApplyTransaction appends a tx's wrapped BlockTrace into the ccc, and return the accumulated RowConsumption.
// Will only return a dummy value in mock_ccc.
func (ccc *CircuitCapacityChecker) ApplyTransaction(traces *types.BlockTrace) (*types.RowConsumption, error) {
if ccc.nextError != nil {
ccc.countdown--
if ccc.countdown == 0 {
err := *ccc.nextError
ccc.nextError = nil
return nil, err
}
}
return &types.RowConsumption{types.SubCircuitRowUsage{
Name: "mock",
RowNumber: 1,
}}, nil
}
// ApplyBlock gets a block's RowConsumption.
// Will only return a dummy value in mock_ccc.
func (ccc *CircuitCapacityChecker) ApplyBlock(traces *types.BlockTrace) (*types.RowConsumption, error) {
return &types.RowConsumption{types.SubCircuitRowUsage{
Name: "mock",
RowNumber: 2,
}}, nil
}
// CheckTxNum compares whether the tx_count in ccc match the expected.
// Will alway return true in mock_ccc.
func (ccc *CircuitCapacityChecker) CheckTxNum(expected int) (bool, uint64, error) {
return true, uint64(expected), nil
}
// SetLightMode sets to ccc light mode
func (ccc *CircuitCapacityChecker) SetLightMode(lightMode bool) error {
return nil
}
// ScheduleError schedules an error for a tx (see `ApplyTransaction`), only used in tests.
func (ccc *CircuitCapacityChecker) ScheduleError(cnt int, err error) {
ccc.countdown = cnt
ccc.nextError = &err
}

View file

@ -0,0 +1,26 @@
package circuitcapacitychecker
import (
"errors"
"github.com/scroll-tech/go-ethereum/core/types"
)
var (
ErrUnknown = errors.New("unknown circuit capacity checker error")
ErrBlockRowConsumptionOverflow = errors.New("block row consumption overflow")
)
type WrappedCommonResult struct {
Error string `json:"error,omitempty"`
}
type WrappedRowUsage struct {
AccRowUsage *types.RowUsage `json:"acc_row_usage,omitempty"`
Error string `json:"error,omitempty"`
}
type WrappedTxNum struct {
TxNum uint64 `json:"tx_num"`
Error string `json:"error,omitempty"`
}