From 83790b072997ab368f1040555ef6cfc3c0dfbf72 Mon Sep 17 00:00:00 2001 From: bitcoin-lightning <153181187+AtomicInnovation321@users.noreply.github.com> Date: Tue, 19 Nov 2024 14:26:39 +0800 Subject: [PATCH 01/21] core: fix typos (#30767) --- core/state/statedb_hooked_test.go | 2 +- core/verkle_witness_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/state/statedb_hooked_test.go b/core/state/statedb_hooked_test.go index 9abd76b02d..5f82ed06d0 100644 --- a/core/state/statedb_hooked_test.go +++ b/core/state/statedb_hooked_test.go @@ -35,7 +35,7 @@ func TestBurn(t *testing.T) { // the following occur: // 1. contract B creates contract A // 2. contract A is destructed - // 3. constract B sends ether to A + // 3. contract B sends ether to A var burned = new(uint256.Int) s, _ := New(types.EmptyRootHash, NewDatabaseForTesting()) diff --git a/core/verkle_witness_test.go b/core/verkle_witness_test.go index 5a4210cdab..b3088f0f90 100644 --- a/core/verkle_witness_test.go +++ b/core/verkle_witness_test.go @@ -338,7 +338,7 @@ func TestProcessVerkleInvalidContractCreation(t *testing.T) { } } } else if bytes.Equal(stemStateDiff.Stem[:], tx1ContractStem) { - // For this contract creation, check that only the accound header and storage slot 41 + // For this contract creation, check that only the account header and storage slot 41 // are found in the witness. for _, suffixDiff := range stemStateDiff.SuffixDiffs { if suffixDiff.Suffix != 105 && suffixDiff.Suffix != 0 && suffixDiff.Suffix != 1 { From 61ff3a1186640e7bed70edb2109721a1ab1a3764 Mon Sep 17 00:00:00 2001 From: jwasinger Date: Tue, 19 Nov 2024 19:29:23 +0700 Subject: [PATCH 02/21] all: remove kilic dependency from bls12381 fuzzers (#30296) The [kilic](https://github.com/kilic/bls12-381) bls12381 implementation has been archived. It shouldn't be necessary to include it as a fuzzing target any longer. This also adds fuzzers for G1/G2 mul that use inputs that are guaranteed to be valid. Previously, we just did random input fuzzing for these precompiles. --- go.mod | 2 +- oss-fuzz.sh | 12 + tests/fuzzers/bls12381/bls12381_fuzz.go | 288 ++++++++++++++---------- tests/fuzzers/bls12381/bls12381_test.go | 26 ++- 4 files changed, 209 insertions(+), 119 deletions(-) diff --git a/go.mod b/go.mod index 1a26321def..c467cef169 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,6 @@ require ( github.com/jackpal/go-nat-pmp v1.0.2 github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267 github.com/karalabe/hid v1.0.1-0.20240306101548-573246063e52 - github.com/kilic/bls12-381 v0.1.0 github.com/kylelemons/godebug v1.1.0 github.com/mattn/go-colorable v0.1.13 github.com/mattn/go-isatty v0.0.20 @@ -116,6 +115,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.4 // indirect github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/kilic/bls12-381 v0.1.0 // indirect github.com/klauspost/compress v1.16.0 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/kr/pretty v0.3.1 // indirect diff --git a/oss-fuzz.sh b/oss-fuzz.sh index 50491b9155..5e4aa1c253 100644 --- a/oss-fuzz.sh +++ b/oss-fuzz.sh @@ -160,6 +160,10 @@ compile_fuzzer github.com/ethereum/go-ethereum/tests/fuzzers/bls12381 \ FuzzG1Add fuzz_g1_add\ $repo/tests/fuzzers/bls12381/bls12381_test.go +compile_fuzzer github.com/ethereum/go-ethereum/tests/fuzzers/bls12381 \ + FuzzCrossG1Mul fuzz_cross_g1_mul\ + $repo/tests/fuzzers/bls12381/bls12381_test.go + compile_fuzzer github.com/ethereum/go-ethereum/tests/fuzzers/bls12381 \ FuzzG1Mul fuzz_g1_mul\ $repo/tests/fuzzers/bls12381/bls12381_test.go @@ -172,6 +176,10 @@ compile_fuzzer github.com/ethereum/go-ethereum/tests/fuzzers/bls12381 \ FuzzG2Add fuzz_g2_add \ $repo/tests/fuzzers/bls12381/bls12381_test.go +compile_fuzzer github.com/ethereum/go-ethereum/tests/fuzzers/bls12381 \ + FuzzCrossG2Mul fuzz_cross_g2_mul\ + $repo/tests/fuzzers/bls12381/bls12381_test.go + compile_fuzzer github.com/ethereum/go-ethereum/tests/fuzzers/bls12381 \ FuzzG2Mul fuzz_g2_mul\ $repo/tests/fuzzers/bls12381/bls12381_test.go @@ -204,6 +212,10 @@ compile_fuzzer github.com/ethereum/go-ethereum/tests/fuzzers/bls12381 \ FuzzCrossG2Add fuzz_cross_g2_add \ $repo/tests/fuzzers/bls12381/bls12381_test.go +compile_fuzzer github.com/ethereum/go-ethereum/tests/fuzzers/bls12381 \ + FuzzCrossG2MultiExp fuzz_cross_g2_multiexp \ + $repo/tests/fuzzers/bls12381/bls12381_test.go + compile_fuzzer github.com/ethereum/go-ethereum/tests/fuzzers/bls12381 \ FuzzCrossPairing fuzz_cross_pairing\ $repo/tests/fuzzers/bls12381/bls12381_test.go diff --git a/tests/fuzzers/bls12381/bls12381_fuzz.go b/tests/fuzzers/bls12381/bls12381_fuzz.go index 74ea6f52a7..a3e0e9f72b 100644 --- a/tests/fuzzers/bls12381/bls12381_fuzz.go +++ b/tests/fuzzers/bls12381/bls12381_fuzz.go @@ -31,42 +31,33 @@ import ( "github.com/consensys/gnark-crypto/ecc/bls12-381/fp" "github.com/consensys/gnark-crypto/ecc/bls12-381/fr" "github.com/ethereum/go-ethereum/common" - bls12381 "github.com/kilic/bls12-381" blst "github.com/supranational/blst/bindings/go" ) func fuzzG1SubgroupChecks(data []byte) int { input := bytes.NewReader(data) - kpG1, cpG1, blG1, err := getG1Points(input) + cpG1, blG1, err := getG1Points(input) if err != nil { return 0 } - inSubGroupKilic := bls12381.NewG1().InCorrectSubgroup(kpG1) inSubGroupGnark := cpG1.IsInSubGroup() inSubGroupBLST := blG1.InG1() - if inSubGroupKilic != inSubGroupGnark { - panic(fmt.Sprintf("differing subgroup check, kilic %v, gnark %v", inSubGroupKilic, inSubGroupGnark)) - } - if inSubGroupKilic != inSubGroupBLST { - panic(fmt.Sprintf("differing subgroup check, kilic %v, blst %v", inSubGroupKilic, inSubGroupBLST)) + if inSubGroupGnark != inSubGroupBLST { + panic(fmt.Sprintf("differing subgroup check, gnark %v, blst %v", inSubGroupGnark, inSubGroupBLST)) } return 1 } func fuzzG2SubgroupChecks(data []byte) int { input := bytes.NewReader(data) - kpG2, cpG2, blG2, err := getG2Points(input) + gpG2, blG2, err := getG2Points(input) if err != nil { return 0 } - inSubGroupKilic := bls12381.NewG2().InCorrectSubgroup(kpG2) - inSubGroupGnark := cpG2.IsInSubGroup() + inSubGroupGnark := gpG2.IsInSubGroup() inSubGroupBLST := blG2.InG2() - if inSubGroupKilic != inSubGroupGnark { - panic(fmt.Sprintf("differing subgroup check, kilic %v, gnark %v", inSubGroupKilic, inSubGroupGnark)) - } - if inSubGroupKilic != inSubGroupBLST { - panic(fmt.Sprintf("differing subgroup check, kilic %v, blst %v", inSubGroupKilic, inSubGroupBLST)) + if inSubGroupGnark != inSubGroupBLST { + panic(fmt.Sprintf("differing subgroup check, gnark %v, blst %v", inSubGroupGnark, inSubGroupBLST)) } return 1 } @@ -75,38 +66,28 @@ func fuzzCrossPairing(data []byte) int { input := bytes.NewReader(data) // get random G1 points - kpG1, cpG1, blG1, err := getG1Points(input) + cpG1, blG1, err := getG1Points(input) if err != nil { return 0 } // get random G2 points - kpG2, cpG2, blG2, err := getG2Points(input) + cpG2, blG2, err := getG2Points(input) if err != nil { return 0 } - // compute pairing using geth - engine := bls12381.NewEngine() - engine.AddPair(kpG1, kpG2) - kResult := engine.Result() - // compute pairing using gnark cResult, err := gnark.Pair([]gnark.G1Affine{*cpG1}, []gnark.G2Affine{*cpG2}) if err != nil { panic(fmt.Sprintf("gnark/bls12381 encountered error: %v", err)) } - // compare result - if !(bytes.Equal(cResult.Marshal(), bls12381.NewGT().ToBytes(kResult))) { - panic("pairing mismatch gnark / geth ") - } - // compute pairing using blst blstResult := blst.Fp12MillerLoop(blG2, blG1) blstResult.FinalExp() res := massageBLST(blstResult.ToBendian()) - if !(bytes.Equal(res, bls12381.NewGT().ToBytes(kResult))) { + if !(bytes.Equal(res, cResult.Marshal())) { panic("pairing mismatch blst / geth") } @@ -141,32 +122,22 @@ func fuzzCrossG1Add(data []byte) int { input := bytes.NewReader(data) // get random G1 points - kp1, cp1, bl1, err := getG1Points(input) + cp1, bl1, err := getG1Points(input) if err != nil { return 0 } // get random G1 points - kp2, cp2, bl2, err := getG1Points(input) + cp2, bl2, err := getG1Points(input) if err != nil { return 0 } - // compute kp = kp1 + kp2 - g1 := bls12381.NewG1() - kp := bls12381.PointG1{} - g1.Add(&kp, kp1, kp2) - // compute cp = cp1 + cp2 _cp1 := new(gnark.G1Jac).FromAffine(cp1) _cp2 := new(gnark.G1Jac).FromAffine(cp2) cp := new(gnark.G1Affine).FromJacobian(_cp1.AddAssign(_cp2)) - // compare result - if !(bytes.Equal(cp.Marshal(), g1.ToBytes(&kp))) { - panic("G1 point addition mismatch gnark / geth ") - } - bl3 := blst.P1AffinesAdd([]*blst.P1Affine{bl1, bl2}) if !(bytes.Equal(cp.Marshal(), bl3.Serialize())) { panic("G1 point addition mismatch blst / geth ") @@ -179,34 +150,24 @@ func fuzzCrossG2Add(data []byte) int { input := bytes.NewReader(data) // get random G2 points - kp1, cp1, bl1, err := getG2Points(input) + gp1, bl1, err := getG2Points(input) if err != nil { return 0 } // get random G2 points - kp2, cp2, bl2, err := getG2Points(input) + gp2, bl2, err := getG2Points(input) if err != nil { return 0 } - // compute kp = kp1 + kp2 - g2 := bls12381.NewG2() - kp := bls12381.PointG2{} - g2.Add(&kp, kp1, kp2) - // compute cp = cp1 + cp2 - _cp1 := new(gnark.G2Jac).FromAffine(cp1) - _cp2 := new(gnark.G2Jac).FromAffine(cp2) - cp := new(gnark.G2Affine).FromJacobian(_cp1.AddAssign(_cp2)) - - // compare result - if !(bytes.Equal(cp.Marshal(), g2.ToBytes(&kp))) { - panic("G2 point addition mismatch gnark / geth ") - } + _gp1 := new(gnark.G2Jac).FromAffine(gp1) + _gp2 := new(gnark.G2Jac).FromAffine(gp2) + gp := new(gnark.G2Affine).FromJacobian(_gp1.AddAssign(_gp2)) bl3 := blst.P2AffinesAdd([]*blst.P2Affine{bl1, bl2}) - if !(bytes.Equal(cp.Marshal(), bl3.Serialize())) { + if !(bytes.Equal(gp.Marshal(), bl3.Serialize())) { panic("G1 point addition mismatch blst / geth ") } @@ -216,10 +177,10 @@ func fuzzCrossG2Add(data []byte) int { func fuzzCrossG1MultiExp(data []byte) int { var ( input = bytes.NewReader(data) - gethScalars []*bls12381.Fr gnarkScalars []fr.Element - gethPoints []*bls12381.PointG1 gnarkPoints []gnark.G1Affine + blstScalars []*blst.Scalar + blstPoints []*blst.P1Affine ) // n random scalars (max 17) for i := 0; i < 17; i++ { @@ -229,50 +190,147 @@ func fuzzCrossG1MultiExp(data []byte) int { break } // get a random G1 point as basis - kp1, cp1, _, err := getG1Points(input) + cp1, bl1, err := getG1Points(input) if err != nil { break } - gethScalars = append(gethScalars, bls12381.NewFr().FromBytes(s.Bytes())) - var gnarkScalar = &fr.Element{} - gnarkScalar = gnarkScalar.SetBigInt(s) - gnarkScalars = append(gnarkScalars, *gnarkScalar) - gethPoints = append(gethPoints, new(bls12381.PointG1).Set(kp1)) + gnarkScalar := new(fr.Element).SetBigInt(s) + gnarkScalars = append(gnarkScalars, *gnarkScalar) gnarkPoints = append(gnarkPoints, *cp1) + + blstScalar := new(blst.Scalar).FromBEndian(common.LeftPadBytes(s.Bytes(), 32)) + blstScalars = append(blstScalars, blstScalar) + blstPoints = append(blstPoints, bl1) } - if len(gethScalars) == 0 { + + if len(gnarkScalars) == 0 || len(gnarkScalars) != len(gnarkPoints) { return 0 } - // compute multi exponentiation - g1 := bls12381.NewG1() - kp := bls12381.PointG1{} - if _, err := g1.MultiExp(&kp, gethPoints, gethScalars); err != nil { - panic(fmt.Sprintf("G1 multi exponentiation errored (geth): %v", err)) - } - // note that geth/crypto/bls12381.MultiExp mutates the scalars slice (and sets all the scalars to zero) // gnark multi exp cp := new(gnark.G1Affine) cp.MultiExp(gnarkPoints, gnarkScalars, ecc.MultiExpConfig{}) - // compare result - gnarkRes := cp.Marshal() - gethRes := g1.ToBytes(&kp) - if !bytes.Equal(gnarkRes, gethRes) { - msg := fmt.Sprintf("G1 multi exponentiation mismatch gnark/geth.\ngnark: %x\ngeth: %x\ninput: %x\n ", - gnarkRes, gethRes, data) - panic(msg) + expectedGnark := multiExpG1Gnark(gnarkPoints, gnarkScalars) + if !bytes.Equal(cp.Marshal(), expectedGnark.Marshal()) { + panic("g1 multi exponentiation mismatch") } + // blst multi exp + expectedBlst := blst.P1AffinesMult(blstPoints, blstScalars, 256).ToAffine() + if !bytes.Equal(cp.Marshal(), expectedBlst.Serialize()) { + panic("g1 multi exponentiation mismatch, gnark/blst") + } return 1 } -func getG1Points(input io.Reader) (*bls12381.PointG1, *gnark.G1Affine, *blst.P1Affine, error) { +func fuzzCrossG1Mul(data []byte) int { + input := bytes.NewReader(data) + gp, blpAffine, err := getG1Points(input) + if err != nil { + return 0 + } + scalar, err := randomScalar(input, fp.Modulus()) + if err != nil { + return 0 + } + + blScalar := new(blst.Scalar).FromBEndian(common.LeftPadBytes(scalar.Bytes(), 32)) + + blp := new(blst.P1) + blp.FromAffine(blpAffine) + + resBl := blp.Mult(blScalar) + resGeth := (new(gnark.G1Affine)).ScalarMultiplication(gp, scalar) + + if !bytes.Equal(resGeth.Marshal(), resBl.Serialize()) { + panic("bytes(blst.G1) != bytes(geth.G1)") + } + return 1 +} + +func fuzzCrossG2Mul(data []byte) int { + input := bytes.NewReader(data) + gp, blpAffine, err := getG2Points(input) + if err != nil { + return 0 + } + scalar, err := randomScalar(input, fp.Modulus()) + if err != nil { + return 0 + } + + blScalar := new(blst.Scalar).FromBEndian(common.LeftPadBytes(scalar.Bytes(), 32)) + + blp := new(blst.P2) + blp.FromAffine(blpAffine) + + resBl := blp.Mult(blScalar) + resGeth := (new(gnark.G2Affine)).ScalarMultiplication(gp, scalar) + + if !bytes.Equal(resGeth.Marshal(), resBl.Serialize()) { + panic("bytes(blst.G1) != bytes(geth.G1)") + } + return 1 +} + +func fuzzCrossG2MultiExp(data []byte) int { + var ( + input = bytes.NewReader(data) + gnarkScalars []fr.Element + gnarkPoints []gnark.G2Affine + blstScalars []*blst.Scalar + blstPoints []*blst.P2Affine + ) + // n random scalars (max 17) + for i := 0; i < 17; i++ { + // note that geth/crypto/bls12381 works only with scalars <= 32bytes + s, err := randomScalar(input, fr.Modulus()) + if err != nil { + break + } + // get a random G1 point as basis + cp1, bl1, err := getG2Points(input) + if err != nil { + break + } + + gnarkScalar := new(fr.Element).SetBigInt(s) + gnarkScalars = append(gnarkScalars, *gnarkScalar) + gnarkPoints = append(gnarkPoints, *cp1) + + blstScalar := new(blst.Scalar).FromBEndian(common.LeftPadBytes(s.Bytes(), 32)) + blstScalars = append(blstScalars, blstScalar) + blstPoints = append(blstPoints, bl1) + } + + if len(gnarkScalars) == 0 || len(gnarkScalars) != len(gnarkPoints) { + return 0 + } + + // gnark multi exp + cp := new(gnark.G2Affine) + cp.MultiExp(gnarkPoints, gnarkScalars, ecc.MultiExpConfig{}) + + expectedGnark := multiExpG2Gnark(gnarkPoints, gnarkScalars) + if !bytes.Equal(cp.Marshal(), expectedGnark.Marshal()) { + panic("g1 multi exponentiation mismatch") + } + + // blst multi exp + expectedBlst := blst.P2AffinesMult(blstPoints, blstScalars, 256).ToAffine() + if !bytes.Equal(cp.Marshal(), expectedBlst.Serialize()) { + panic("g1 multi exponentiation mismatch, gnark/blst") + } + return 1 +} + +func getG1Points(input io.Reader) (*gnark.G1Affine, *blst.P1Affine, error) { // sample a random scalar s, err := randomScalar(input, fp.Modulus()) if err != nil { - return nil, nil, nil, err + return nil, nil, err } // compute a random point @@ -281,18 +339,6 @@ func getG1Points(input io.Reader) (*bls12381.PointG1, *gnark.G1Affine, *blst.P1A cp.ScalarMultiplication(&g1Gen, s) cpBytes := cp.Marshal() - // marshal gnark point -> geth point - g1 := bls12381.NewG1() - kp, err := g1.FromBytes(cpBytes) - if err != nil { - panic(fmt.Sprintf("Could not marshal gnark.G1 -> geth.G1: %v", err)) - } - - gnarkRes := g1.ToBytes(kp) - if !bytes.Equal(gnarkRes, cpBytes) { - panic(fmt.Sprintf("bytes(gnark.G1) != bytes(geth.G1)\ngnark.G1: %x\ngeth.G1: %x\n", gnarkRes, cpBytes)) - } - // marshal gnark point -> blst point scalar := new(blst.Scalar).FromBEndian(common.LeftPadBytes(s.Bytes(), 32)) p1 := new(blst.P1Affine).From(scalar) @@ -301,43 +347,31 @@ func getG1Points(input io.Reader) (*bls12381.PointG1, *gnark.G1Affine, *blst.P1A panic(fmt.Sprintf("bytes(blst.G1) != bytes(geth.G1)\nblst.G1: %x\ngeth.G1: %x\n", blstRes, cpBytes)) } - return kp, cp, p1, nil + return cp, p1, nil } -func getG2Points(input io.Reader) (*bls12381.PointG2, *gnark.G2Affine, *blst.P2Affine, error) { +func getG2Points(input io.Reader) (*gnark.G2Affine, *blst.P2Affine, error) { // sample a random scalar s, err := randomScalar(input, fp.Modulus()) if err != nil { - return nil, nil, nil, err + return nil, nil, err } // compute a random point - cp := new(gnark.G2Affine) + gp := new(gnark.G2Affine) _, _, _, g2Gen := gnark.Generators() - cp.ScalarMultiplication(&g2Gen, s) - cpBytes := cp.Marshal() - - // marshal gnark point -> geth point - g2 := bls12381.NewG2() - kp, err := g2.FromBytes(cpBytes) - if err != nil { - panic(fmt.Sprintf("Could not marshal gnark.G2 -> geth.G2: %v", err)) - } - - gnarkRes := g2.ToBytes(kp) - if !bytes.Equal(gnarkRes, cpBytes) { - panic(fmt.Sprintf("bytes(gnark.G2) != bytes(geth.G2)\ngnark.G2: %x\ngeth.G2: %x\n", gnarkRes, cpBytes)) - } + gp.ScalarMultiplication(&g2Gen, s) + cpBytes := gp.Marshal() // marshal gnark point -> blst point // Left pad the scalar to 32 bytes scalar := new(blst.Scalar).FromBEndian(common.LeftPadBytes(s.Bytes(), 32)) p2 := new(blst.P2Affine).From(scalar) if !bytes.Equal(p2.Serialize(), cpBytes) { - panic("bytes(blst.G2) != bytes(geth.G2)") + panic("bytes(blst.G2) != bytes(bls12381.G2)") } - return kp, cp, p2, nil + return gp, p2, nil } func randomScalar(r io.Reader, max *big.Int) (k *big.Int, err error) { @@ -348,3 +382,29 @@ func randomScalar(r io.Reader, max *big.Int) (k *big.Int, err error) { } } } + +// multiExpG1Gnark is a naive implementation of G1 multi-exponentiation +func multiExpG1Gnark(gs []gnark.G1Affine, scalars []fr.Element) gnark.G1Affine { + res := gnark.G1Affine{} + for i := 0; i < len(gs); i++ { + tmp := new(gnark.G1Affine) + sb := scalars[i].Bytes() + scalarBytes := new(big.Int).SetBytes(sb[:]) + tmp.ScalarMultiplication(&gs[i], scalarBytes) + res.Add(&res, tmp) + } + return res +} + +// multiExpG1Gnark is a naive implementation of G1 multi-exponentiation +func multiExpG2Gnark(gs []gnark.G2Affine, scalars []fr.Element) gnark.G2Affine { + res := gnark.G2Affine{} + for i := 0; i < len(gs); i++ { + tmp := new(gnark.G2Affine) + sb := scalars[i].Bytes() + scalarBytes := new(big.Int).SetBytes(sb[:]) + tmp.ScalarMultiplication(&gs[i], scalarBytes) + res.Add(&res, tmp) + } + return res +} diff --git a/tests/fuzzers/bls12381/bls12381_test.go b/tests/fuzzers/bls12381/bls12381_test.go index fd782f7813..d4e5e20e04 100644 --- a/tests/fuzzers/bls12381/bls12381_test.go +++ b/tests/fuzzers/bls12381/bls12381_test.go @@ -27,6 +27,12 @@ func FuzzCrossPairing(f *testing.F) { }) } +func FuzzCrossG2MultiExp(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + fuzzCrossG2MultiExp(data) + }) +} + func FuzzCrossG1Add(f *testing.F) { f.Fuzz(func(t *testing.T, data []byte) { fuzzCrossG1Add(data) @@ -51,9 +57,9 @@ func FuzzG1Add(f *testing.F) { }) } -func FuzzG1Mul(f *testing.F) { +func FuzzCrossG1Mul(f *testing.F) { f.Fuzz(func(t *testing.T, data []byte) { - fuzz(blsG1Mul, data) + fuzzCrossG1Mul(data) }) } @@ -69,9 +75,9 @@ func FuzzG2Add(f *testing.F) { }) } -func FuzzG2Mul(f *testing.F) { +func FuzzCrossG2Mul(f *testing.F) { f.Fuzz(func(t *testing.T, data []byte) { - fuzz(blsG2Mul, data) + fuzzCrossG2Mul(data) }) } @@ -110,3 +116,15 @@ func FuzzG2SubgroupChecks(f *testing.F) { fuzzG2SubgroupChecks(data) }) } + +func FuzzG2Mul(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + fuzz(blsG2Mul, data) + }) +} + +func FuzzG1Mul(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + fuzz(blsG1Mul, data) + }) +} From 581e2140f22566655aa8fb2d1e9a6c4a740d3be1 Mon Sep 17 00:00:00 2001 From: jwasinger Date: Tue, 19 Nov 2024 19:35:52 +0700 Subject: [PATCH 03/21] core/txpool, eth/catalyst: clear transaction pool in Rollback (#30534) This adds an API method `DropTransactions` to legacy pool, blob pool and txpool interface. This method removes all txs currently tracked in the pools. It modifies the simulated beacon to use the new method in `Rollback` which removes previous hacky implementation that also erroneously reset the gas tip to 1 gwei. --------- Co-authored-by: Felix Lange --- core/txpool/blobpool/blobpool.go | 50 ++++++++++++++++++++++++++++ core/txpool/legacypool/legacypool.go | 41 +++++++++++++++++++++++ core/txpool/subpool.go | 3 ++ core/txpool/txpool.go | 7 ++++ eth/catalyst/simulated_beacon.go | 9 +---- 5 files changed, 102 insertions(+), 8 deletions(-) diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index 0352ea9783..02d339f99c 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -1714,3 +1714,53 @@ func (p *BlobPool) Status(hash common.Hash) txpool.TxStatus { } return txpool.TxStatusUnknown } + +// Clear implements txpool.SubPool, removing all tracked transactions +// from the blob pool and persistent store. +func (p *BlobPool) Clear() { + p.lock.Lock() + defer p.lock.Unlock() + + // manually iterating and deleting every entry is super sub-optimal + // However, Clear is not currently used in production so + // performance is not critical at the moment. + for hash := range p.lookup.txIndex { + id, _ := p.lookup.storeidOfTx(hash) + if err := p.store.Delete(id); err != nil { + log.Warn("failed to delete blob tx from backing store", "err", err) + } + } + for hash := range p.lookup.blobIndex { + id, _ := p.lookup.storeidOfBlob(hash) + if err := p.store.Delete(id); err != nil { + log.Warn("failed to delete blob from backing store", "err", err) + } + } + + // unreserve each tracked account. Ideally, we could just clear the + // reservation map in the parent txpool context. However, if we clear in + // parent context, to avoid exposing the subpool lock, we have to lock the + // reservations and then lock each subpool. + // + // This creates the potential for a deadlock situation: + // + // * TxPool.Clear locks the reservations + // * a new transaction is received which locks the subpool mutex + // * TxPool.Clear attempts to lock subpool mutex + // + // The transaction addition may attempt to reserve the sender addr which + // can't happen until Clear releases the reservation lock. Clear cannot + // acquire the subpool lock until the transaction addition is completed. + for acct, _ := range p.index { + p.reserve(acct, false) + } + p.lookup = newLookup() + p.index = make(map[common.Address][]*blobTxMeta) + p.spent = make(map[common.Address]*uint256.Int) + + var ( + basefee = uint256.MustFromBig(eip1559.CalcBaseFee(p.chain.Config(), p.head)) + blobfee = uint256.NewInt(params.BlobTxMinBlobGasprice) + ) + p.evict = newPriceHeap(basefee, blobfee, p.index) +} diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index f7495dd39f..89ff86df02 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -1961,3 +1961,44 @@ func (t *lookup) RemotesBelowTip(threshold *big.Int) types.Transactions { func numSlots(tx *types.Transaction) int { return int((tx.Size() + txSlotSize - 1) / txSlotSize) } + +// Clear implements txpool.SubPool, removing all tracked txs from the pool +// and rotating the journal. +func (pool *LegacyPool) Clear() { + pool.mu.Lock() + defer pool.mu.Unlock() + + // unreserve each tracked account. Ideally, we could just clear the + // reservation map in the parent txpool context. However, if we clear in + // parent context, to avoid exposing the subpool lock, we have to lock the + // reservations and then lock each subpool. + // + // This creates the potential for a deadlock situation: + // + // * TxPool.Clear locks the reservations + // * a new transaction is received which locks the subpool mutex + // * TxPool.Clear attempts to lock subpool mutex + // + // The transaction addition may attempt to reserve the sender addr which + // can't happen until Clear releases the reservation lock. Clear cannot + // acquire the subpool lock until the transaction addition is completed. + for _, tx := range pool.all.remotes { + senderAddr, _ := types.Sender(pool.signer, tx) + pool.reserve(senderAddr, false) + } + for localSender, _ := range pool.locals.accounts { + pool.reserve(localSender, false) + } + + pool.all = newLookup() + pool.priced = newPricedList(pool.all) + pool.pending = make(map[common.Address]*list) + pool.queue = make(map[common.Address]*list) + + if !pool.config.NoLocals && pool.config.Journal != "" { + pool.journal = newTxJournal(pool.config.Journal) + if err := pool.journal.rotate(pool.local()); err != nil { + log.Warn("Failed to rotate transaction journal", "err", err) + } + } +} diff --git a/core/txpool/subpool.go b/core/txpool/subpool.go index 180facd217..9ee0a69c0b 100644 --- a/core/txpool/subpool.go +++ b/core/txpool/subpool.go @@ -168,4 +168,7 @@ type SubPool interface { // Status returns the known status (unknown/pending/queued) of a transaction // identified by their hashes. Status(hash common.Hash) TxStatus + + // Clear removes all tracked transactions from the pool + Clear() } diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index 54ae3be569..ce455e806e 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -497,3 +497,10 @@ func (p *TxPool) Sync() error { return errors.New("pool already terminated") } } + +// Clear removes all tracked txs from the subpools. +func (p *TxPool) Clear() { + for _, subpool := range p.subpools { + subpool.Clear() + } +} diff --git a/eth/catalyst/simulated_beacon.go b/eth/catalyst/simulated_beacon.go index db46afc30d..a24ff52101 100644 --- a/eth/catalyst/simulated_beacon.go +++ b/eth/catalyst/simulated_beacon.go @@ -21,7 +21,6 @@ import ( "crypto/sha256" "errors" "fmt" - "math/big" "sync" "time" @@ -34,7 +33,6 @@ import ( "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" ) @@ -287,12 +285,7 @@ func (c *SimulatedBeacon) Commit() common.Hash { // Rollback un-sends previously added transactions. func (c *SimulatedBeacon) Rollback() { - // Flush all transactions from the transaction pools - maxUint256 := new(big.Int).Sub(new(big.Int).Lsh(common.Big1, 256), common.Big1) - c.eth.TxPool().SetGasTip(maxUint256) - // Set the gas tip back to accept new transactions - // TODO (Marius van der Wijden): set gas tip to parameter passed by config - c.eth.TxPool().SetGasTip(big.NewInt(params.GWei)) + c.eth.TxPool().Clear() } // Fork sets the head to the provided hash. From e20150f88886eec249a8e39288fda723ec143751 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5vard=20Anda=20Estensen?= Date: Tue, 19 Nov 2024 13:43:33 +0100 Subject: [PATCH 04/21] rpc: run tests in parallel (#30384) Continuation of https://github.com/ethereum/go-ethereum/pull/30381 --- rpc/client_test.go | 38 ++++++++++++++++++++++++++++++++++++++ rpc/http_test.go | 20 ++++++++++++++++++++ rpc/server_test.go | 10 ++++++++++ rpc/subscription_test.go | 8 ++++++++ rpc/types_test.go | 10 ++++++++++ rpc/websocket_test.go | 4 ++++ 6 files changed, 90 insertions(+) diff --git a/rpc/client_test.go b/rpc/client_test.go index 49f2350b40..6c1a4f8f6c 100644 --- a/rpc/client_test.go +++ b/rpc/client_test.go @@ -38,6 +38,8 @@ import ( ) func TestClientRequest(t *testing.T) { + t.Parallel() + server := newTestServer() defer server.Stop() client := DialInProc(server) @@ -53,6 +55,8 @@ func TestClientRequest(t *testing.T) { } func TestClientResponseType(t *testing.T) { + t.Parallel() + server := newTestServer() defer server.Stop() client := DialInProc(server) @@ -71,6 +75,8 @@ func TestClientResponseType(t *testing.T) { // This test checks calling a method that returns 'null'. func TestClientNullResponse(t *testing.T) { + t.Parallel() + server := newTestServer() defer server.Stop() @@ -91,6 +97,8 @@ func TestClientNullResponse(t *testing.T) { // This test checks that server-returned errors with code and data come out of Client.Call. func TestClientErrorData(t *testing.T) { + t.Parallel() + server := newTestServer() defer server.Stop() client := DialInProc(server) @@ -121,6 +129,8 @@ func TestClientErrorData(t *testing.T) { } func TestClientBatchRequest(t *testing.T) { + t.Parallel() + server := newTestServer() defer server.Stop() client := DialInProc(server) @@ -172,6 +182,8 @@ func TestClientBatchRequest(t *testing.T) { // This checks that, for HTTP connections, the length of batch responses is validated to // match the request exactly. func TestClientBatchRequest_len(t *testing.T) { + t.Parallel() + b, err := json.Marshal([]jsonrpcMessage{ {Version: "2.0", ID: json.RawMessage("1"), Result: json.RawMessage(`"0x1"`)}, {Version: "2.0", ID: json.RawMessage("2"), Result: json.RawMessage(`"0x2"`)}, @@ -188,6 +200,8 @@ func TestClientBatchRequest_len(t *testing.T) { t.Cleanup(s.Close) t.Run("too-few", func(t *testing.T) { + t.Parallel() + client, err := Dial(s.URL) if err != nil { t.Fatal("failed to dial test server:", err) @@ -218,6 +232,8 @@ func TestClientBatchRequest_len(t *testing.T) { }) t.Run("too-many", func(t *testing.T) { + t.Parallel() + client, err := Dial(s.URL) if err != nil { t.Fatal("failed to dial test server:", err) @@ -249,6 +265,8 @@ func TestClientBatchRequest_len(t *testing.T) { // This checks that the client can handle the case where the server doesn't // respond to all requests in a batch. func TestClientBatchRequestLimit(t *testing.T) { + t.Parallel() + server := newTestServer() defer server.Stop() server.SetBatchLimits(2, 100000) @@ -285,6 +303,8 @@ func TestClientBatchRequestLimit(t *testing.T) { } func TestClientNotify(t *testing.T) { + t.Parallel() + server := newTestServer() defer server.Stop() client := DialInProc(server) @@ -392,6 +412,8 @@ func testClientCancel(transport string, t *testing.T) { } func TestClientSubscribeInvalidArg(t *testing.T) { + t.Parallel() + server := newTestServer() defer server.Stop() client := DialInProc(server) @@ -422,6 +444,8 @@ func TestClientSubscribeInvalidArg(t *testing.T) { } func TestClientSubscribe(t *testing.T) { + t.Parallel() + server := newTestServer() defer server.Stop() client := DialInProc(server) @@ -454,6 +478,8 @@ func TestClientSubscribe(t *testing.T) { // In this test, the connection drops while Subscribe is waiting for a response. func TestClientSubscribeClose(t *testing.T) { + t.Parallel() + server := newTestServer() service := ¬ificationTestService{ gotHangSubscriptionReq: make(chan struct{}), @@ -498,6 +524,8 @@ func TestClientSubscribeClose(t *testing.T) { // This test reproduces https://github.com/ethereum/go-ethereum/issues/17837 where the // client hangs during shutdown when Unsubscribe races with Client.Close. func TestClientCloseUnsubscribeRace(t *testing.T) { + t.Parallel() + server := newTestServer() defer server.Stop() @@ -540,6 +568,8 @@ func (b *unsubscribeBlocker) readBatch() ([]*jsonrpcMessage, bool, error) { // not respond. // It reproducers the issue https://github.com/ethereum/go-ethereum/issues/30156 func TestUnsubscribeTimeout(t *testing.T) { + t.Parallel() + srv := NewServer() srv.RegisterName("nftest", new(notificationTestService)) @@ -674,6 +704,8 @@ func TestClientSubscriptionChannelClose(t *testing.T) { // This test checks that Client doesn't lock up when a single subscriber // doesn't read subscription events. func TestClientNotificationStorm(t *testing.T) { + t.Parallel() + server := newTestServer() defer server.Stop() @@ -726,6 +758,8 @@ func TestClientNotificationStorm(t *testing.T) { } func TestClientSetHeader(t *testing.T) { + t.Parallel() + var gotHeader bool srv := newTestServer() httpsrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -762,6 +796,8 @@ func TestClientSetHeader(t *testing.T) { } func TestClientHTTP(t *testing.T) { + t.Parallel() + server := newTestServer() defer server.Stop() @@ -804,6 +840,8 @@ func TestClientHTTP(t *testing.T) { } func TestClientReconnect(t *testing.T) { + t.Parallel() + startServer := func(addr string) (*Server, net.Listener) { srv := newTestServer() l, err := net.Listen("tcp", addr) diff --git a/rpc/http_test.go b/rpc/http_test.go index ad86ca15ae..6c268b6292 100644 --- a/rpc/http_test.go +++ b/rpc/http_test.go @@ -58,24 +58,34 @@ func confirmRequestValidationCode(t *testing.T, method, contentType, body string } func TestHTTPErrorResponseWithDelete(t *testing.T) { + t.Parallel() + confirmRequestValidationCode(t, http.MethodDelete, contentType, "", http.StatusMethodNotAllowed) } func TestHTTPErrorResponseWithPut(t *testing.T) { + t.Parallel() + confirmRequestValidationCode(t, http.MethodPut, contentType, "", http.StatusMethodNotAllowed) } func TestHTTPErrorResponseWithMaxContentLength(t *testing.T) { + t.Parallel() + body := make([]rune, defaultBodyLimit+1) confirmRequestValidationCode(t, http.MethodPost, contentType, string(body), http.StatusRequestEntityTooLarge) } func TestHTTPErrorResponseWithEmptyContentType(t *testing.T) { + t.Parallel() + confirmRequestValidationCode(t, http.MethodPost, "", "", http.StatusUnsupportedMediaType) } func TestHTTPErrorResponseWithValidRequest(t *testing.T) { + t.Parallel() + confirmRequestValidationCode(t, http.MethodPost, contentType, "", 0) } @@ -101,11 +111,15 @@ func confirmHTTPRequestYieldsStatusCode(t *testing.T, method, contentType, body } func TestHTTPResponseWithEmptyGet(t *testing.T) { + t.Parallel() + confirmHTTPRequestYieldsStatusCode(t, http.MethodGet, "", "", http.StatusOK) } // This checks that maxRequestContentLength is not applied to the response of a request. func TestHTTPRespBodyUnlimited(t *testing.T) { + t.Parallel() + const respLength = defaultBodyLimit * 3 s := NewServer() @@ -132,6 +146,8 @@ func TestHTTPRespBodyUnlimited(t *testing.T) { // Tests that an HTTP error results in an HTTPError instance // being returned with the expected attributes. func TestHTTPErrorResponse(t *testing.T) { + t.Parallel() + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Error(w, "error has occurred!", http.StatusTeapot) })) @@ -169,6 +185,8 @@ func TestHTTPErrorResponse(t *testing.T) { } func TestHTTPPeerInfo(t *testing.T) { + t.Parallel() + s := newTestServer() defer s.Stop() ts := httptest.NewServer(s) @@ -205,6 +223,8 @@ func TestHTTPPeerInfo(t *testing.T) { } func TestNewContextWithHeaders(t *testing.T) { + t.Parallel() + expectedHeaders := 0 server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { for i := 0; i < expectedHeaders; i++ { diff --git a/rpc/server_test.go b/rpc/server_test.go index 9d1c7fb5f0..9ee545d81a 100644 --- a/rpc/server_test.go +++ b/rpc/server_test.go @@ -29,6 +29,8 @@ import ( ) func TestServerRegisterName(t *testing.T) { + t.Parallel() + server := NewServer() service := new(testService) @@ -53,6 +55,8 @@ func TestServerRegisterName(t *testing.T) { } func TestServer(t *testing.T) { + t.Parallel() + files, err := os.ReadDir("testdata") if err != nil { t.Fatal("where'd my testdata go?") @@ -64,6 +68,8 @@ func TestServer(t *testing.T) { path := filepath.Join("testdata", f.Name()) name := strings.TrimSuffix(f.Name(), filepath.Ext(f.Name())) t.Run(name, func(t *testing.T) { + t.Parallel() + runTestScript(t, path) }) } @@ -116,6 +122,8 @@ func runTestScript(t *testing.T, file string) { // This test checks that responses are delivered for very short-lived connections that // only carry a single request. func TestServerShortLivedConn(t *testing.T) { + t.Parallel() + server := newTestServer() defer server.Stop() @@ -156,6 +164,8 @@ func TestServerShortLivedConn(t *testing.T) { } func TestServerBatchResponseSizeLimit(t *testing.T) { + t.Parallel() + server := newTestServer() defer server.Stop() server.SetBatchLimits(100, 60) diff --git a/rpc/subscription_test.go b/rpc/subscription_test.go index ab40ab169f..e52f390adb 100644 --- a/rpc/subscription_test.go +++ b/rpc/subscription_test.go @@ -33,6 +33,8 @@ import ( ) func TestNewID(t *testing.T) { + t.Parallel() + hexchars := "0123456789ABCDEFabcdef" for i := 0; i < 100; i++ { id := string(NewID()) @@ -54,6 +56,8 @@ func TestNewID(t *testing.T) { } func TestSubscriptions(t *testing.T) { + t.Parallel() + var ( namespaces = []string{"eth", "bzz"} service = ¬ificationTestService{} @@ -132,6 +136,8 @@ func TestSubscriptions(t *testing.T) { // This test checks that unsubscribing works. func TestServerUnsubscribe(t *testing.T) { + t.Parallel() + p1, p2 := net.Pipe() defer p2.Close() @@ -260,6 +266,8 @@ func BenchmarkNotify(b *testing.B) { } func TestNotify(t *testing.T) { + t.Parallel() + out := new(bytes.Buffer) id := ID("test") notifier := &Notifier{ diff --git a/rpc/types_test.go b/rpc/types_test.go index aba40b5863..9dd6fa6508 100644 --- a/rpc/types_test.go +++ b/rpc/types_test.go @@ -26,6 +26,8 @@ import ( ) func TestBlockNumberJSONUnmarshal(t *testing.T) { + t.Parallel() + tests := []struct { input string mustFail bool @@ -70,6 +72,8 @@ func TestBlockNumberJSONUnmarshal(t *testing.T) { } func TestBlockNumberOrHash_UnmarshalJSON(t *testing.T) { + t.Parallel() + tests := []struct { input string mustFail bool @@ -131,6 +135,8 @@ func TestBlockNumberOrHash_UnmarshalJSON(t *testing.T) { } func TestBlockNumberOrHash_WithNumber_MarshalAndUnmarshal(t *testing.T) { + t.Parallel() + tests := []struct { name string number int64 @@ -144,6 +150,8 @@ func TestBlockNumberOrHash_WithNumber_MarshalAndUnmarshal(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { + t.Parallel() + bnh := BlockNumberOrHashWithNumber(BlockNumber(test.number)) marshalled, err := json.Marshal(bnh) if err != nil { @@ -162,6 +170,8 @@ func TestBlockNumberOrHash_WithNumber_MarshalAndUnmarshal(t *testing.T) { } func TestBlockNumberOrHash_StringAndUnmarshal(t *testing.T) { + t.Parallel() + tests := []BlockNumberOrHash{ BlockNumberOrHashWithNumber(math.MaxInt64), BlockNumberOrHashWithNumber(PendingBlockNumber), diff --git a/rpc/websocket_test.go b/rpc/websocket_test.go index c6ea325d29..10a998b351 100644 --- a/rpc/websocket_test.go +++ b/rpc/websocket_test.go @@ -174,6 +174,8 @@ func TestWebsocketLargeRead(t *testing.T) { } func TestWebsocketPeerInfo(t *testing.T) { + t.Parallel() + var ( s = newTestServer() ts = httptest.NewServer(s.WebsocketHandler([]string{"origin.example.com"})) @@ -259,6 +261,8 @@ func TestClientWebsocketPing(t *testing.T) { // This checks that the websocket transport can deal with large messages. func TestClientWebsocketLargeMessage(t *testing.T) { + t.Parallel() + var ( srv = NewServer() httpsrv = httptest.NewServer(srv.WebsocketHandler(nil)) From c64cf28f9c79c26c1c64e3af6e2bc48d80e71ff1 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 19 Nov 2024 14:46:38 +0100 Subject: [PATCH 05/21] version: go-ethereum v1.14.12 stable --- version/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/version/version.go b/version/version.go index cbd59f3e9a..f010adf035 100644 --- a/version/version.go +++ b/version/version.go @@ -17,8 +17,8 @@ package version const ( - Major = 1 // Major version component of the current release - Minor = 14 // Minor version component of the current release - Patch = 12 // Patch version component of the current release - Meta = "unstable" // Version metadata to append to the version string + Major = 1 // Major version component of the current release + Minor = 14 // Minor version component of the current release + Patch = 12 // Patch version component of the current release + Meta = "stable" // Version metadata to append to the version string ) From 66d8185aad6f3dde13fdeb95e985bafd6e2e0167 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 19 Nov 2024 14:51:54 +0100 Subject: [PATCH 06/21] version: begin v1.14.13 release cycle --- version/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/version/version.go b/version/version.go index f010adf035..562de8d184 100644 --- a/version/version.go +++ b/version/version.go @@ -17,8 +17,8 @@ package version const ( - Major = 1 // Major version component of the current release - Minor = 14 // Minor version component of the current release - Patch = 12 // Patch version component of the current release - Meta = "stable" // Version metadata to append to the version string + Major = 1 // Major version component of the current release + Minor = 14 // Minor version component of the current release + Patch = 13 // Patch version component of the current release + Meta = "ustable" // Version metadata to append to the version string ) From aa636921291744af082cefbe4dd811fb58fbde63 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 19 Nov 2024 16:58:52 +0100 Subject: [PATCH 07/21] version: fix typo in v1.14.13 release cycle name --- version/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/version/version.go b/version/version.go index 562de8d184..52dfa0281e 100644 --- a/version/version.go +++ b/version/version.go @@ -17,8 +17,8 @@ package version const ( - Major = 1 // Major version component of the current release - Minor = 14 // Minor version component of the current release - Patch = 13 // Patch version component of the current release - Meta = "ustable" // Version metadata to append to the version string + Major = 1 // Major version component of the current release + Minor = 14 // Minor version component of the current release + Patch = 13 // Patch version component of the current release + Meta = "unstable" // Version metadata to append to the version string ) From 6d3d252a5ece0687dcaaf197e0bfc03466161ff1 Mon Sep 17 00:00:00 2001 From: Martin HS Date: Wed, 20 Nov 2024 08:40:21 +0100 Subject: [PATCH 08/21] core/vm/program: evm bytecode-building utility (#30725) In many cases, there is a need to create somewhat nontrivial bytecode. A recent example is the verkle statetests, where we want a `CREATE2`- op to create a contract, which can then be invoked, and when invoked does a selfdestruct-to-self. It is overkill to go full solidity, but it is also a bit tricky do assemble this by concatenating bytes. This PR takes an approach that has been used in in goevmlab for several years. Using this utility, the case can be expressed as: ```golang // Some runtime code runtime := program.New().Ops(vm.ADDRESS, vm.SELFDESTRUCT).Bytecode() // A constructor returning the runtime code initcode := program.New().ReturnData(runtime).Bytecode() // A factory invoking the constructor outer := program.New().Create2AndCall(initcode, nil).Bytecode() ``` We have a lot of places in the codebase where we concatenate bytes, cast from `vm.OpCode` . By taking tihs approach instead, thos places can be made a bit more maintainable/robust. --- core/vm/program/program.go | 430 ++++++++++++++++++++++++++++++++ core/vm/program/program_test.go | 311 +++++++++++++++++++++++ core/vm/program/readme.md | 30 +++ core/vm/runtime/runtime_test.go | 252 +++++-------------- 4 files changed, 834 insertions(+), 189 deletions(-) create mode 100644 core/vm/program/program.go create mode 100644 core/vm/program/program_test.go create mode 100644 core/vm/program/readme.md diff --git a/core/vm/program/program.go b/core/vm/program/program.go new file mode 100644 index 0000000000..acc7fd25fc --- /dev/null +++ b/core/vm/program/program.go @@ -0,0 +1,430 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the goevmlab library. If not, see . + +// package program is a utility to create EVM bytecode for testing, but _not_ for production. As such: +// +// - There are not package guarantees. We might iterate heavily on this package, and do backwards-incompatible changes without warning +// - There are no quality-guarantees. These utilities may produce evm-code that is non-functional. YMMV. +// - There are no stability-guarantees. The utility will `panic` if the inputs do not align / make sense. +package program + +import ( + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/core/vm" + "github.com/holiman/uint256" +) + +// Program is a simple bytecode container. It can be used to construct +// simple EVM programs. Errors during construction of a Program typically +// cause panics: so avoid using these programs in production settings or on +// untrusted input. +// This package is mainly meant to aid in testing. This is not a production +// -level "compiler". +type Program struct { + code []byte +} + +// New creates a new Program +func New() *Program { + return &Program{ + code: make([]byte, 0), + } +} + +// add adds the op to the code. +func (p *Program) add(op byte) *Program { + p.code = append(p.code, op) + return p +} + +// pushBig creates a PUSHX instruction and pushes the given val. +// - If the val is nil, it pushes zero +// - If the val is bigger than 32 bytes, it panics +func (p *Program) doPush(val *uint256.Int) { + if val == nil { + val = new(uint256.Int) + } + valBytes := val.Bytes() + if len(valBytes) == 0 { + valBytes = append(valBytes, 0) + } + bLen := len(valBytes) + p.add(byte(vm.PUSH1) - 1 + byte(bLen)) + p.Append(valBytes) +} + +// Append appends the given data to the code. +func (p *Program) Append(data []byte) *Program { + p.code = append(p.code, data...) + return p +} + +// Bytes returns the Program bytecode. OBS: This is not a copy. +func (p *Program) Bytes() []byte { + return p.code +} + +// SetBytes sets the Program bytecode. The combination of Bytes and SetBytes means +// that external callers can implement missing functionality: +// +// ... +// prog.Push(1) +// code := prog.Bytes() +// manipulate(code) +// prog.SetBytes(code) +func (p *Program) SetBytes(code []byte) { + p.code = code +} + +// Hex returns the Program bytecode as a hex string. +func (p *Program) Hex() string { + return fmt.Sprintf("%02x", p.Bytes()) +} + +// Op appends the given opcode(s). +func (p *Program) Op(ops ...vm.OpCode) *Program { + for _, op := range ops { + p.add(byte(op)) + } + return p +} + +// Push creates a PUSHX instruction with the data provided. If zero is being pushed, +// PUSH0 will be avoided in favour of [PUSH1 0], to ensure backwards compatibility. +func (p *Program) Push(val any) *Program { + switch v := val.(type) { + case int: + p.doPush(new(uint256.Int).SetUint64(uint64(v))) + case uint64: + p.doPush(new(uint256.Int).SetUint64(v)) + case uint32: + p.doPush(new(uint256.Int).SetUint64(uint64(v))) + case uint16: + p.doPush(new(uint256.Int).SetUint64(uint64(v))) + case *big.Int: + p.doPush(uint256.MustFromBig(v)) + case *uint256.Int: + p.doPush(v) + case uint256.Int: + p.doPush(&v) + case []byte: + p.doPush(new(uint256.Int).SetBytes(v)) + case byte: + p.doPush(new(uint256.Int).SetUint64(uint64(v))) + case interface{ Bytes() []byte }: + // Here, we jump through some hoops in order to avoid depending on + // go-ethereum types.Address and common.Hash, and instead use the + // interface. This works on both values and pointers! + p.doPush(new(uint256.Int).SetBytes(v.Bytes())) + case nil: + p.doPush(nil) + default: + panic(fmt.Sprintf("unsupported type %T", v)) + } + return p +} + +// Push0 implements PUSH0 (0x5f). +func (p *Program) Push0() *Program { + return p.Op(vm.PUSH0) +} + +// ExtcodeCopy performs an extcodecopy invocation. +func (p *Program) ExtcodeCopy(address, memOffset, codeOffset, length any) *Program { + p.Push(length) + p.Push(codeOffset) + p.Push(memOffset) + p.Push(address) + return p.Op(vm.EXTCODECOPY) +} + +// Call is a convenience function to make a call. If 'gas' is nil, the opcode GAS will +// be used to provide all gas. +func (p *Program) Call(gas *uint256.Int, address, value, inOffset, inSize, outOffset, outSize any) *Program { + if outOffset == outSize && inSize == outSize && inOffset == outSize && value == outSize { + p.Push(outSize).Op(vm.DUP1, vm.DUP1, vm.DUP1, vm.DUP1) + } else { + p.Push(outSize).Push(outOffset).Push(inSize).Push(inOffset).Push(value) + } + p.Push(address) + if gas == nil { + p.Op(vm.GAS) + } else { + p.doPush(gas) + } + return p.Op(vm.CALL) +} + +// DelegateCall is a convenience function to make a delegatecall. If 'gas' is nil, the opcode GAS will +// be used to provide all gas. +func (p *Program) DelegateCall(gas *uint256.Int, address, inOffset, inSize, outOffset, outSize any) *Program { + if outOffset == outSize && inSize == outSize && inOffset == outSize { + p.Push(outSize).Op(vm.DUP1, vm.DUP1, vm.DUP1) + } else { + p.Push(outSize).Push(outOffset).Push(inSize).Push(inOffset) + } + p.Push(address) + if gas == nil { + p.Op(vm.GAS) + } else { + p.doPush(gas) + } + return p.Op(vm.DELEGATECALL) +} + +// StaticCall is a convenience function to make a staticcall. If 'gas' is nil, the opcode GAS will +// be used to provide all gas. +func (p *Program) StaticCall(gas *uint256.Int, address, inOffset, inSize, outOffset, outSize any) *Program { + if outOffset == outSize && inSize == outSize && inOffset == outSize { + p.Push(outSize).Op(vm.DUP1, vm.DUP1, vm.DUP1) + } else { + p.Push(outSize).Push(outOffset).Push(inSize).Push(inOffset) + } + p.Push(address) + if gas == nil { + p.Op(vm.GAS) + } else { + p.doPush(gas) + } + return p.Op(vm.STATICCALL) +} + +// StaticCall is a convenience function to make a callcode. If 'gas' is nil, the opcode GAS will +// be used to provide all gas. +func (p *Program) CallCode(gas *uint256.Int, address, value, inOffset, inSize, outOffset, outSize any) *Program { + if outOffset == outSize && inSize == outSize && inOffset == outSize { + p.Push(outSize).Op(vm.DUP1, vm.DUP1, vm.DUP1) + } else { + p.Push(outSize).Push(outOffset).Push(inSize).Push(inOffset) + } + p.Push(value) + p.Push(address) + if gas == nil { + p.Op(vm.GAS) + } else { + p.doPush(gas) + } + return p.Op(vm.CALLCODE) +} + +// Label returns the PC (of the next instruction). +func (p *Program) Label() uint64 { + return uint64(len(p.code)) +} + +// Jumpdest adds a JUMPDEST op, and returns the PC of that instruction. +func (p *Program) Jumpdest() (*Program, uint64) { + here := p.Label() + p.Op(vm.JUMPDEST) + return p, here +} + +// Jump pushes the destination and adds a JUMP. +func (p *Program) Jump(loc any) *Program { + p.Push(loc) + p.Op(vm.JUMP) + return p +} + +// JumpIf implements JUMPI. +func (p *Program) JumpIf(loc any, condition any) *Program { + p.Push(condition) + p.Push(loc) + p.Op(vm.JUMPI) + return p +} + +// Size returns the current size of the bytecode. +func (p *Program) Size() int { + return len(p.code) +} + +// InputAddressToStack stores the input (calldata) to memory as address (20 bytes). +func (p *Program) InputAddressToStack(inputOffset uint32) *Program { + p.Push(inputOffset) + p.Op(vm.CALLDATALOAD) // Loads [n -> n + 32] of input data to stack top + mask, _ := big.NewInt(0).SetString("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", 16) + p.Push(mask) // turn into address + return p.Op(vm.AND) +} + +// MStore stores the provided data (into the memory area starting at memStart). +func (p *Program) Mstore(data []byte, memStart uint32) *Program { + var idx = 0 + // We need to store it in chunks of 32 bytes + for ; idx+32 <= len(data); idx += 32 { + chunk := data[idx : idx+32] + // push the value + p.Push(chunk) + // push the memory index + p.Push(uint32(idx) + memStart) + p.Op(vm.MSTORE) + } + // Remainders become stored using MSTORE8 + for ; idx < len(data); idx++ { + b := data[idx] + // push the byte + p.Push(b) + p.Push(uint32(idx) + memStart) + p.Op(vm.MSTORE8) + } + return p +} + +// MstoreSmall stores the provided data, which must be smaller than 32 bytes, +// into the memory area starting at memStart. +// The data will be LHS zero-added to align on 32 bytes. +// For example, providing data 0x1122, it will do a PUSH2: +// PUSH2 0x1122, resulting in +// stack: 0x0000000000000000000000000000000000000000000000000000000000001122 +// followed by MSTORE(0,0) +// And thus, the resulting memory will be +// [ 0000000000000000000000000000000000000000000000000000000000001122 ] +func (p *Program) MstoreSmall(data []byte, memStart uint32) *Program { + if len(data) > 32 { + // For larger sizes, use Mstore instead. + panic("only <=32 byte data size supported") + } + if len(data) == 0 { + // Storing 0-length data smells of an error somewhere. + panic("data is zero length") + } + // push the value + p.Push(data) + // push the memory index + p.Push(memStart) + p.Op(vm.MSTORE) + return p +} + +// MemToStorage copies the given memory area into SSTORE slots, +// It expects data to be aligned to 32 byte, and does not zero out +// remainders if some data is not +// I.e, if given a 1-byte area, it will still copy the full 32 bytes to storage. +func (p *Program) MemToStorage(memStart, memSize, startSlot int) *Program { + // We need to store it in chunks of 32 bytes + for idx := memStart; idx < (memStart + memSize); idx += 32 { + dataStart := idx + // Mload the chunk + p.Push(dataStart) + p.Op(vm.MLOAD) + // Value is now on stack, + p.Push(startSlot) + p.Op(vm.SSTORE) + startSlot++ + } + return p +} + +// ReturnViaCodeCopy utilises CODECOPY to place the given data in the bytecode of +// p, loads into memory (offset 0) and returns the code. +// This is a typical "constructor". +// Note: since all indexing is calculated immediately, the preceding bytecode +// must not be expanded or shortened. +func (p *Program) ReturnViaCodeCopy(data []byte) *Program { + p.Push(len(data)) + // For convenience, we'll use PUSH2 for the offset. Then we know we can always + // fit, since code is limited to 0xc000 + p.Op(vm.PUSH2) + offsetPos := p.Size() // Need to update this position later on + p.Append([]byte{0, 0}) // Offset of the code to be copied + p.Push(0) // Offset in memory (destination) + p.Op(vm.CODECOPY) // Copy from code[offset:offset+len] to memory[0:] + p.Return(0, len(data)) // Return memory[0:len] + offset := p.Size() + p.Append(data) // And add the data + + // Now, go back and fix the offset + p.code[offsetPos] = byte(offset >> 8) + p.code[offsetPos+1] = byte(offset) + return p +} + +// Sstore stores the given byte array to the given slot. +// OBS! Does not verify that the value indeed fits into 32 bytes. +// If it does not, it will panic later on via doPush. +func (p *Program) Sstore(slot any, value any) *Program { + p.Push(value) + p.Push(slot) + return p.Op(vm.SSTORE) +} + +// Tstore stores the given byte array to the given t-slot. +// OBS! Does not verify that the value indeed fits into 32 bytes. +// If it does not, it will panic later on via doPush. +func (p *Program) Tstore(slot any, value any) *Program { + p.Push(value) + p.Push(slot) + return p.Op(vm.TSTORE) +} + +// Return implements RETURN +func (p *Program) Return(offset, len int) *Program { + p.Push(len) + p.Push(offset) + return p.Op(vm.RETURN) +} + +// ReturnData loads the given data into memory, and does a return with it +func (p *Program) ReturnData(data []byte) *Program { + p.Mstore(data, 0) + return p.Return(0, len(data)) +} + +// Create2 uses create2 to construct a contract with the given bytecode. +// This operation leaves either '0' or address on the stack. +func (p *Program) Create2(code []byte, salt any) *Program { + var ( + value = 0 + offset = 0 + size = len(code) + ) + // Load the code into mem + p.Mstore(code, 0) + // Create it + return p.Push(salt). + Push(size). + Push(offset). + Push(value). + Op(vm.CREATE2) + // On the stack now, is either + // - zero: in case of failure, OR + // - address: in case of success +} + +// Create2ThenCall calls create2 with the given initcode and salt, and then calls +// into the created contract (or calls into zero, if the creation failed). +func (p *Program) Create2ThenCall(code []byte, salt any) *Program { + p.Create2(code, salt) + // If there happen to be a zero on the stack, it doesn't matter, we're + // not sending any value anyway + p.Push(0).Push(0) // mem out + p.Push(0).Push(0) // mem in + p.Push(0) // value + p.Op(vm.DUP6) // address + p.Op(vm.GAS) + p.Op(vm.CALL) + p.Op(vm.POP) // pop the retval + return p.Op(vm.POP) // pop the address +} + +// Selfdestruct pushes beneficiary and invokes selfdestruct. +func (p *Program) Selfdestruct(beneficiary any) *Program { + p.Push(beneficiary) + return p.Op(vm.SELFDESTRUCT) +} diff --git a/core/vm/program/program_test.go b/core/vm/program/program_test.go new file mode 100644 index 0000000000..0b34210067 --- /dev/null +++ b/core/vm/program/program_test.go @@ -0,0 +1,311 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the goevmlab library. If not, see . + +package program + +import ( + "bytes" + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/holiman/uint256" +) + +func TestPush(t *testing.T) { + tests := []struct { + input interface{} + expected string + }{ + // native ints + {0, "6000"}, + {0xfff, "610fff"}, + {nil, "6000"}, + {uint8(1), "6001"}, + {uint16(1), "6001"}, + {uint32(1), "6001"}, + {uint64(1), "6001"}, + // bigints + {big.NewInt(0), "6000"}, + {big.NewInt(1), "6001"}, + {big.NewInt(0xfff), "610fff"}, + // uint256 + {uint256.NewInt(1), "6001"}, + {uint256.Int{1, 0, 0, 0}, "6001"}, + // Addresses + {common.HexToAddress("0xdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef"), "73deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"}, + {&common.Address{}, "6000"}, + } + for i, tc := range tests { + have := New().Push(tc.input).Hex() + if have != tc.expected { + t.Errorf("test %d: got %v expected %v", i, have, tc.expected) + } + } +} + +func TestCall(t *testing.T) { + { // Nil gas + have := New().Call(nil, common.HexToAddress("0x1337"), big.NewInt(1), 1, 2, 3, 4).Hex() + want := "600460036002600160016113375af1" + if have != want { + t.Errorf("have %v want %v", have, want) + } + } + { // Non nil gas + have := New().Call(uint256.NewInt(0xffff), common.HexToAddress("0x1337"), big.NewInt(1), 1, 2, 3, 4).Hex() + want := "6004600360026001600161133761fffff1" + if have != want { + t.Errorf("have %v want %v", have, want) + } + } +} + +func TestMstore(t *testing.T) { + { + have := New().Mstore(common.FromHex("0xaabb"), 0).Hex() + want := "60aa60005360bb600153" + if have != want { + t.Errorf("have %v want %v", have, want) + } + } + { // store at offset + have := New().Mstore(common.FromHex("0xaabb"), 3).Hex() + want := "60aa60035360bb600453" + if have != want { + t.Errorf("have %v want %v", have, want) + } + } + { // 34 bytes + data := common.FromHex("0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" + + "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" + + "FFFF") + + have := New().Mstore(data, 0).Hex() + want := "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff60005260ff60205360ff602153" + if have != want { + t.Errorf("have %v want %v", have, want) + } + } +} + +func TestMemToStorage(t *testing.T) { + have := New().MemToStorage(0, 33, 1).Hex() + want := "600051600155602051600255" + if have != want { + t.Errorf("have %v want %v", have, want) + } +} + +func TestSstore(t *testing.T) { + have := New().Sstore(0x1337, []byte("1234")).Hex() + want := "633132333461133755" + if have != want { + t.Errorf("have %v want %v", have, want) + } +} + +func TestReturnData(t *testing.T) { + { + have := New().ReturnData([]byte{0xFF}).Hex() + want := "60ff60005360016000f3" + if have != want { + t.Errorf("have %v want %v", have, want) + } + } + { + // 32 bytes + data := common.FromHex("0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF") + have := New().ReturnData(data).Hex() + want := "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff60005260206000f3" + if have != want { + t.Errorf("have %v want %v", have, want) + } + } + { // ReturnViaCodeCopy + data := common.FromHex("0x6001") + have := New().Append([]byte{0x5b, 0x5b, 0x5b}).ReturnViaCodeCopy(data).Hex() + want := "5b5b5b600261001060003960026000f36001" + if have != want { + t.Errorf("have %v want %v", have, want) + } + } + { // ReturnViaCodeCopy larger code + data := common.FromHex("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff60005260206000f3") + have := New().Append([]byte{0x5b, 0x5b, 0x5b}).ReturnViaCodeCopy(data).Hex() + want := "5b5b5b602961001060003960296000f37fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff60005260206000f3" + if have != want { + t.Errorf("have %v want %v", have, want) + } + } +} + +func TestCreateAndCall(t *testing.T) { + // A constructor that stores a slot + ctor := New().Sstore(0, big.NewInt(5)) + + // A runtime bytecode which reads the slot and returns + deployed := New() + deployed.Push(0).Op(vm.SLOAD) // [value] in stack + deployed.Push(0) // [value, 0] + deployed.Op(vm.MSTORE) + deployed.Return(0, 32) + + // Pack them + ctor.ReturnData(deployed.Bytes()) + // Verify constructor + runtime code + { + want := "6005600055606060005360006001536054600253606060035360006004536052600553606060065360206007536060600853600060095360f3600a53600b6000f3" + if got := ctor.Hex(); got != want { + t.Fatalf("1: got %v expected %v", got, want) + } + } +} + +func TestCreate2Call(t *testing.T) { + // Some runtime code + runtime := New().Op(vm.ADDRESS, vm.SELFDESTRUCT).Bytes() + want := common.FromHex("0x30ff") + if !bytes.Equal(want, runtime) { + t.Fatalf("runtime code error\nwant: %x\nhave: %x\n", want, runtime) + } + // A constructor returning the runtime code + initcode := New().ReturnData(runtime).Bytes() + want = common.FromHex("603060005360ff60015360026000f3") + if !bytes.Equal(want, initcode) { + t.Fatalf("initcode error\nwant: %x\nhave: %x\n", want, initcode) + } + // A factory invoking the constructor + outer := New().Create2ThenCall(initcode, nil).Bytes() + want = common.FromHex("60606000536030600153606060025360006003536053600453606060055360ff6006536060600753600160085360536009536060600a536002600b536060600c536000600d5360f3600e536000600f60006000f560006000600060006000855af15050") + if !bytes.Equal(want, outer) { + t.Fatalf("factory error\nwant: %x\nhave: %x\n", want, outer) + } +} + +func TestGenerator(t *testing.T) { + for i, tc := range []struct { + want []byte + haveFn func() []byte + }{ + { // CREATE + want: []byte{ + // Store initcode in memory at 0x00 (5 bytes left-padded to 32 bytes) + byte(vm.PUSH5), + // Init code: PUSH1 0, PUSH1 0, RETURN (3 steps) + byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.RETURN), + byte(vm.PUSH1), 0, + byte(vm.MSTORE), + // length, offset, value + byte(vm.PUSH1), 5, byte(vm.PUSH1), 27, byte(vm.PUSH1), 0, + byte(vm.CREATE), + byte(vm.POP), + }, + haveFn: func() []byte { + initcode := New().Return(0, 0).Bytes() + return New().MstoreSmall(initcode, 0). + Push(len(initcode)). // length + Push(32 - len(initcode)). // offset + Push(0). // value + Op(vm.CREATE). + Op(vm.POP).Bytes() + }, + }, + { // CREATE2 + want: []byte{ + // Store initcode in memory at 0x00 (5 bytes left-padded to 32 bytes) + byte(vm.PUSH5), + // Init code: PUSH1 0, PUSH1 0, RETURN (3 steps) + byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.RETURN), + byte(vm.PUSH1), 0, + byte(vm.MSTORE), + // salt, length, offset, value + byte(vm.PUSH1), 1, byte(vm.PUSH1), 5, byte(vm.PUSH1), 27, byte(vm.PUSH1), 0, + byte(vm.CREATE2), + byte(vm.POP), + }, + haveFn: func() []byte { + initcode := New().Return(0, 0).Bytes() + return New().MstoreSmall(initcode, 0). + Push(1). // salt + Push(len(initcode)). // length + Push(32 - len(initcode)). // offset + Push(0). // value + Op(vm.CREATE2). + Op(vm.POP).Bytes() + }, + }, + { // CALL + want: []byte{ + // outsize, outoffset, insize, inoffset + byte(vm.PUSH1), 0, byte(vm.DUP1), byte(vm.DUP1), byte(vm.DUP1), + byte(vm.DUP1), // value + byte(vm.PUSH1), 0xbb, //address + byte(vm.GAS), // gas + byte(vm.CALL), + byte(vm.POP), + }, + haveFn: func() []byte { + return New().Call(nil, 0xbb, 0, 0, 0, 0, 0).Op(vm.POP).Bytes() + }, + }, + { // CALLCODE + want: []byte{ + // outsize, outoffset, insize, inoffset + byte(vm.PUSH1), 0, byte(vm.DUP1), byte(vm.DUP1), byte(vm.DUP1), + byte(vm.PUSH1), 0, // value + byte(vm.PUSH1), 0xcc, //address + byte(vm.GAS), // gas + byte(vm.CALLCODE), + byte(vm.POP), + }, + haveFn: func() []byte { + return New().CallCode(nil, 0xcc, 0, 0, 0, 0, 0).Op(vm.POP).Bytes() + }, + }, + { // STATICCALL + want: []byte{ + // outsize, outoffset, insize, inoffset + byte(vm.PUSH1), 0, byte(vm.DUP1), byte(vm.DUP1), byte(vm.DUP1), + byte(vm.PUSH1), 0xdd, //address + byte(vm.GAS), // gas + byte(vm.STATICCALL), + byte(vm.POP), + }, + haveFn: func() []byte { + return New().StaticCall(nil, 0xdd, 0, 0, 0, 0).Op(vm.POP).Bytes() + }, + }, + { // DELEGATECALL + want: []byte{ + // outsize, outoffset, insize, inoffset + byte(vm.PUSH1), 0, byte(vm.DUP1), byte(vm.DUP1), byte(vm.DUP1), + byte(vm.PUSH1), 0xee, //address + byte(vm.GAS), // gas + byte(vm.DELEGATECALL), + byte(vm.POP), + }, + haveFn: func() []byte { + return New().DelegateCall(nil, 0xee, 0, 0, 0, 0).Op(vm.POP).Bytes() + }, + }, + } { + if have := tc.haveFn(); !bytes.Equal(have, tc.want) { + t.Fatalf("test %d error\nhave: %x\nwant: %x\n", i, have, tc.want) + } + } +} diff --git a/core/vm/program/readme.md b/core/vm/program/readme.md new file mode 100644 index 0000000000..0e4a54d8f1 --- /dev/null +++ b/core/vm/program/readme.md @@ -0,0 +1,30 @@ +### What is this + +In many cases, we have a need to create somewhat nontrivial bytecode, for testing various +quirks related to state transition or evm execution. + +For example, we want to have a `CREATE2`- op create a contract, which is then invoked, and when invoked does a selfdestruct-to-self. + +It is overkill to go full solidity, but it is also a bit tricky do assemble this by concatenating bytes. + +This utility takes an approach from [goevmlab](https://github.com/holiman/goevmlab/) where it has been used for several years, +a go-lang utility to assemble evm bytecode. + +Using this utility, the case above can be expressed as: +```golang + // Some runtime code + runtime := program.New().Ops(vm.ADDRESS, vm.SELFDESTRUCT).Bytecode() + // A constructor returning the runtime code + initcode := program.New().ReturnData(runtime).Bytecode() + // A factory invoking the constructor + outer := program.New().Create2AndCall(initcode, nil).Bytecode() +``` + +### Warning + +This package is a utility for testing, _not_ for production. As such: + +- There are not package guarantees. We might iterate heavily on this package, and do backwards-incompatible changes without warning +- There are no quality-guarantees. These utilities may produce evm-code that is non-functional. YMMV. +- There are no stability-guarantees. The utility will `panic` if the inputs do not align / make sense. + diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index 97234368ee..7d1345a57b 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/core/vm/program" "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/eth/tracers/logger" "github.com/ethereum/go-ethereum/params" @@ -436,110 +437,46 @@ func benchmarkNonModifyingCode(gas uint64, code []byte, name string, tracerCode // BenchmarkSimpleLoop test a pretty simple loop which loops until OOG // 55 ms func BenchmarkSimpleLoop(b *testing.B) { - staticCallIdentity := []byte{ - byte(vm.JUMPDEST), // [ count ] - // push args for the call - byte(vm.PUSH1), 0, // out size - byte(vm.DUP1), // out offset - byte(vm.DUP1), // out insize - byte(vm.DUP1), // in offset - byte(vm.PUSH1), 0x4, // address of identity - byte(vm.GAS), // gas - byte(vm.STATICCALL), - byte(vm.POP), // pop return value - byte(vm.PUSH1), 0, // jumpdestination - byte(vm.JUMP), - } + p, lbl := program.New().Jumpdest() + // Call identity, and pop return value + staticCallIdentity := p. + StaticCall(nil, 0x4, 0, 0, 0, 0). + Op(vm.POP).Jump(lbl).Bytes() // pop return value and jump to label - callIdentity := []byte{ - byte(vm.JUMPDEST), // [ count ] - // push args for the call - byte(vm.PUSH1), 0, // out size - byte(vm.DUP1), // out offset - byte(vm.DUP1), // out insize - byte(vm.DUP1), // in offset - byte(vm.DUP1), // value - byte(vm.PUSH1), 0x4, // address of identity - byte(vm.GAS), // gas - byte(vm.CALL), - byte(vm.POP), // pop return value - byte(vm.PUSH1), 0, // jumpdestination - byte(vm.JUMP), - } + p, lbl = program.New().Jumpdest() + callIdentity := p. + Call(nil, 0x4, 0, 0, 0, 0, 0). + Op(vm.POP).Jump(lbl).Bytes() // pop return value and jump to label - callInexistant := []byte{ - byte(vm.JUMPDEST), // [ count ] - // push args for the call - byte(vm.PUSH1), 0, // out size - byte(vm.DUP1), // out offset - byte(vm.DUP1), // out insize - byte(vm.DUP1), // in offset - byte(vm.DUP1), // value - byte(vm.PUSH1), 0xff, // address of existing contract - byte(vm.GAS), // gas - byte(vm.CALL), - byte(vm.POP), // pop return value - byte(vm.PUSH1), 0, // jumpdestination - byte(vm.JUMP), - } + p, lbl = program.New().Jumpdest() + callInexistant := p. + Call(nil, 0xff, 0, 0, 0, 0, 0). + Op(vm.POP).Jump(lbl).Bytes() // pop return value and jump to label - callEOA := []byte{ - byte(vm.JUMPDEST), // [ count ] - // push args for the call - byte(vm.PUSH1), 0, // out size - byte(vm.DUP1), // out offset - byte(vm.DUP1), // out insize - byte(vm.DUP1), // in offset - byte(vm.DUP1), // value - byte(vm.PUSH1), 0xE0, // address of EOA - byte(vm.GAS), // gas - byte(vm.CALL), - byte(vm.POP), // pop return value - byte(vm.PUSH1), 0, // jumpdestination - byte(vm.JUMP), - } + p, lbl = program.New().Jumpdest() + callEOA := p. + Call(nil, 0xE0, 0, 0, 0, 0, 0). // call addr of EOA + Op(vm.POP).Jump(lbl).Bytes() // pop return value and jump to label - loopingCode := []byte{ - byte(vm.JUMPDEST), // [ count ] - // push args for the call - byte(vm.PUSH1), 0, // out size - byte(vm.DUP1), // out offset - byte(vm.DUP1), // out insize - byte(vm.DUP1), // in offset - byte(vm.PUSH1), 0x4, // address of identity - byte(vm.GAS), // gas + p, lbl = program.New().Jumpdest() + // Push as if we were making call, then pop it off again, and loop + loopingCode := p.Push(0). + Op(vm.DUP1, vm.DUP1, vm.DUP1). + Push(0x4). + Op(vm.GAS, vm.POP, vm.POP, vm.POP, vm.POP, vm.POP, vm.POP). + Jump(lbl).Bytes() - byte(vm.POP), byte(vm.POP), byte(vm.POP), byte(vm.POP), byte(vm.POP), byte(vm.POP), - byte(vm.PUSH1), 0, // jumpdestination - byte(vm.JUMP), - } + p, lbl = program.New().Jumpdest() + loopingCode2 := p. + Push(0x01020304).Push(uint64(0x0102030405)). + Op(vm.POP, vm.POP). + Op(vm.PUSH6).Append(make([]byte, 6)).Op(vm.JUMP). // Jumpdest zero expressed in 6 bytes + Jump(lbl).Bytes() - loopingCode2 := []byte{ - byte(vm.JUMPDEST), // [ count ] - // push args for the call - byte(vm.PUSH4), 1, 2, 3, 4, - byte(vm.PUSH5), 1, 2, 3, 4, 5, - - byte(vm.POP), byte(vm.POP), - byte(vm.PUSH6), 0, 0, 0, 0, 0, 0, // jumpdestination - byte(vm.JUMP), - } - - callRevertingContractWithInput := []byte{ - byte(vm.JUMPDEST), // - // push args for the call - byte(vm.PUSH1), 0, // out size - byte(vm.DUP1), // out offset - byte(vm.PUSH1), 0x20, // in size - byte(vm.PUSH1), 0x00, // in offset - byte(vm.PUSH1), 0x00, // value - byte(vm.PUSH1), 0xEE, // address of reverting contract - byte(vm.GAS), // gas - byte(vm.CALL), - byte(vm.POP), // pop return value - byte(vm.PUSH1), 0, // jumpdestination - byte(vm.JUMP), - } + p, lbl = program.New().Jumpdest() + callRevertingContractWithInput := p. + Call(nil, 0xee, 0, 0, 0x20, 0x0, 0x0). + Op(vm.POP).Jump(lbl).Bytes() // pop return value and jump to label //tracer := logger.NewJSONLogger(nil, os.Stdout) //Execute(loopingCode, nil, &Config{ @@ -778,104 +715,49 @@ func TestRuntimeJSTracer(t *testing.T) { this.exits++; this.gasUsed = res.getGasUsed(); }}`} + initcode := program.New().Return(0, 0).Bytes() tests := []struct { code []byte // One result per tracer results []string }{ - { - // CREATE - code: []byte{ - // Store initcode in memory at 0x00 (5 bytes left-padded to 32 bytes) - byte(vm.PUSH5), - // Init code: PUSH1 0, PUSH1 0, RETURN (3 steps) - byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.RETURN), - byte(vm.PUSH1), 0, - byte(vm.MSTORE), - // length, offset, value - byte(vm.PUSH1), 5, byte(vm.PUSH1), 27, byte(vm.PUSH1), 0, - byte(vm.CREATE), - byte(vm.POP), - }, + { // CREATE + code: program.New().MstoreSmall(initcode, 0). + Push(len(initcode)). // length + Push(32 - len(initcode)). // offset + Push(0). // value + Op(vm.CREATE). + Op(vm.POP).Bytes(), results: []string{`"1,1,952853,6,12"`, `"1,1,952853,6,0"`}, }, - { - // CREATE2 - code: []byte{ - // Store initcode in memory at 0x00 (5 bytes left-padded to 32 bytes) - byte(vm.PUSH5), - // Init code: PUSH1 0, PUSH1 0, RETURN (3 steps) - byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.RETURN), - byte(vm.PUSH1), 0, - byte(vm.MSTORE), - // salt, length, offset, value - byte(vm.PUSH1), 1, byte(vm.PUSH1), 5, byte(vm.PUSH1), 27, byte(vm.PUSH1), 0, - byte(vm.CREATE2), - byte(vm.POP), - }, + { // CREATE2 + code: program.New().MstoreSmall(initcode, 0). + Push(1). // salt + Push(len(initcode)). // length + Push(32 - len(initcode)). // offset + Push(0). // value + Op(vm.CREATE2). + Op(vm.POP).Bytes(), results: []string{`"1,1,952844,6,13"`, `"1,1,952844,6,0"`}, }, - { - // CALL - code: []byte{ - // outsize, outoffset, insize, inoffset - byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, - byte(vm.PUSH1), 0, // value - byte(vm.PUSH1), 0xbb, //address - byte(vm.GAS), // gas - byte(vm.CALL), - byte(vm.POP), - }, + { // CALL + code: program.New().Call(nil, 0xbb, 0, 0, 0, 0, 0).Op(vm.POP).Bytes(), results: []string{`"1,1,981796,6,13"`, `"1,1,981796,6,0"`}, }, - { - // CALLCODE - code: []byte{ - // outsize, outoffset, insize, inoffset - byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, - byte(vm.PUSH1), 0, // value - byte(vm.PUSH1), 0xcc, //address - byte(vm.GAS), // gas - byte(vm.CALLCODE), - byte(vm.POP), - }, + { // CALLCODE + code: program.New().CallCode(nil, 0xcc, 0, 0, 0, 0, 0).Op(vm.POP).Bytes(), results: []string{`"1,1,981796,6,13"`, `"1,1,981796,6,0"`}, }, - { - // STATICCALL - code: []byte{ - // outsize, outoffset, insize, inoffset - byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, - byte(vm.PUSH1), 0xdd, //address - byte(vm.GAS), // gas - byte(vm.STATICCALL), - byte(vm.POP), - }, + { // STATICCALL + code: program.New().StaticCall(nil, 0xdd, 0, 0, 0, 0).Op(vm.POP).Bytes(), results: []string{`"1,1,981799,6,12"`, `"1,1,981799,6,0"`}, }, - { - // DELEGATECALL - code: []byte{ - // outsize, outoffset, insize, inoffset - byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, - byte(vm.PUSH1), 0xee, //address - byte(vm.GAS), // gas - byte(vm.DELEGATECALL), - byte(vm.POP), - }, + { // DELEGATECALL + code: program.New().DelegateCall(nil, 0xee, 0, 0, 0, 0).Op(vm.POP).Bytes(), results: []string{`"1,1,981799,6,12"`, `"1,1,981799,6,0"`}, }, - { - // CALL self-destructing contract - code: []byte{ - // outsize, outoffset, insize, inoffset - byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, - byte(vm.PUSH1), 0, // value - byte(vm.PUSH1), 0xff, //address - byte(vm.GAS), // gas - byte(vm.CALL), - byte(vm.POP), - }, + { // CALL self-destructing contract + code: program.New().Call(nil, 0xff, 0, 0, 0, 0, 0).Op(vm.POP).Bytes(), results: []string{`"2,2,0,5003,12"`, `"2,2,0,5003,0"`}, }, } @@ -958,16 +840,8 @@ func TestJSTracerCreateTx(t *testing.T) { func BenchmarkTracerStepVsCallFrame(b *testing.B) { // Simply pushes and pops some values in a loop - code := []byte{ - byte(vm.JUMPDEST), - byte(vm.PUSH1), 0, - byte(vm.PUSH1), 0, - byte(vm.POP), - byte(vm.POP), - byte(vm.PUSH1), 0, // jumpdestination - byte(vm.JUMP), - } - + p, lbl := program.New().Jumpdest() + code := p.Push(0).Push(0).Op(vm.POP, vm.POP).Jump(lbl).Bytes() stepTracer := ` { step: function() {}, From e3d61e6db028c412f74bc4d4c7e117a9e29d0de0 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Wed, 20 Nov 2024 19:35:52 +0800 Subject: [PATCH 09/21] core, eth, internal, cmd: rework EVM constructor (#30745) This pull request refactors the EVM constructor by removing the TxContext parameter. The EVM object is frequently overused. Ideally, only a single EVM instance should be created and reused throughout the entire state transition of a block, with the transaction context switched as needed by calling evm.SetTxContext. Unfortunately, in some parts of the code, the EVM object is repeatedly created, resulting in unnecessary complexity. This pull request is the first step towards gradually improving and simplifying this setup. --------- Co-authored-by: Martin Holst Swende --- cmd/evm/internal/t8ntool/execution.go | 18 +-- core/chain_makers.go | 20 +-- core/state_prefetcher.go | 8 +- core/state_processor.go | 35 ++--- core/verkle_witness_test.go | 2 +- core/vm/evm.go | 21 +-- core/vm/gas_table_test.go | 10 +- core/vm/instructions_test.go | 128 ++++++++---------- core/vm/interpreter_test.go | 2 +- core/vm/runtime/env.go | 4 +- eth/api_backend.go | 5 +- eth/gasestimator/gasestimator.go | 3 +- eth/state_accessor.go | 24 ++-- eth/tracers/api.go | 57 ++++---- eth/tracers/api_test.go | 13 +- .../internal/tracetest/calltrace_test.go | 19 ++- .../internal/tracetest/flat_calltrace_test.go | 3 +- .../internal/tracetest/prestate_test.go | 3 +- eth/tracers/js/tracer_test.go | 17 ++- eth/tracers/logger/logger_test.go | 6 +- eth/tracers/tracers_test.go | 3 +- internal/ethapi/api.go | 6 +- internal/ethapi/api_test.go | 5 +- internal/ethapi/backend.go | 2 +- internal/ethapi/simulate.go | 4 +- internal/ethapi/transaction_args_test.go | 2 +- miner/worker.go | 19 +-- tests/state_test.go | 3 +- tests/state_test_util.go | 3 +- 29 files changed, 220 insertions(+), 225 deletions(-) diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index f80dd02c67..a868fbf11c 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -201,15 +201,14 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, chainConfig.DAOForkBlock.Cmp(new(big.Int).SetUint64(pre.Env.Number)) == 0 { misc.ApplyDAOHardFork(statedb) } + evm := vm.NewEVM(vmContext, statedb, chainConfig, vmConfig) if beaconRoot := pre.Env.ParentBeaconBlockRoot; beaconRoot != nil { - evm := vm.NewEVM(vmContext, vm.TxContext{}, statedb, chainConfig, vmConfig) core.ProcessBeaconBlockRoot(*beaconRoot, evm, statedb) } if pre.Env.BlockHashes != nil && chainConfig.IsPrague(new(big.Int).SetUint64(pre.Env.Number), pre.Env.Timestamp) { var ( prevNumber = pre.Env.Number - 1 prevHash = pre.Env.BlockHashes[math.HexOrDecimal64(prevNumber)] - evm = vm.NewEVM(vmContext, vm.TxContext{}, statedb, chainConfig, vmConfig) ) core.ProcessParentBlockHash(prevHash, evm, statedb) } @@ -246,8 +245,10 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, if err != nil { return nil, nil, nil, err } + // TODO (rjl493456442) it's a bit weird to reset the tracer in the + // middle of block execution, please improve it somehow. if tracer != nil { - vmConfig.Tracer = tracer.Hooks + evm.SetTracer(tracer.Hooks) } statedb.SetTxContext(tx.Hash(), txIndex) @@ -256,12 +257,12 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, snapshot = statedb.Snapshot() prevGas = gaspool.Gas() ) - evm := vm.NewEVM(vmContext, txContext, statedb, chainConfig, vmConfig) - if tracer != nil && tracer.OnTxStart != nil { tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) } // (ret []byte, usedGas uint64, failed bool, err error) + + evm.SetTxContext(txContext) msgResult, err := core.ApplyMessage(evm, msg, gaspool) if err != nil { statedb.RevertToSnapshot(snapshot) @@ -375,12 +376,11 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, return nil, nil, nil, NewError(ErrorEVM, fmt.Errorf("could not parse requests logs: %v", err)) } requests = append(requests, depositRequests) - // create EVM for system calls - vmenv := vm.NewEVM(vmContext, vm.TxContext{}, statedb, chainConfig, vm.Config{}) + // EIP-7002 withdrawals - requests = append(requests, core.ProcessWithdrawalQueue(vmenv, statedb)) + requests = append(requests, core.ProcessWithdrawalQueue(evm, statedb)) // EIP-7251 consolidations - requests = append(requests, core.ProcessConsolidationQueue(vmenv, statedb)) + requests = append(requests, core.ProcessConsolidationQueue(evm, statedb)) } // Commit block diff --git a/core/chain_makers.go b/core/chain_makers.go index 586979e772..07448abc3d 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -100,9 +100,9 @@ func (b *BlockGen) SetParentBeaconRoot(root common.Hash) { b.header.ParentBeaconRoot = &root var ( blockContext = NewEVMBlockContext(b.header, b.cm, &b.header.Coinbase) - vmenv = vm.NewEVM(blockContext, vm.TxContext{}, b.statedb, b.cm.config, vm.Config{}) + evm = vm.NewEVM(blockContext, b.statedb, b.cm.config, vm.Config{}) ) - ProcessBeaconBlockRoot(root, vmenv, b.statedb) + ProcessBeaconBlockRoot(root, evm, b.statedb) } // addTx adds a transaction to the generated block. If no coinbase has @@ -116,8 +116,12 @@ func (b *BlockGen) addTx(bc *BlockChain, vmConfig vm.Config, tx *types.Transacti if b.gasPool == nil { b.SetCoinbase(common.Address{}) } + var ( + blockContext = NewEVMBlockContext(b.header, bc, &b.header.Coinbase) + evm = vm.NewEVM(blockContext, b.statedb, b.cm.config, vmConfig) + ) b.statedb.SetTxContext(tx.Hash(), len(b.txs)) - receipt, err := ApplyTransaction(b.cm.config, bc, &b.header.Coinbase, b.gasPool, b.statedb, b.header, tx, &b.header.GasUsed, vmConfig) + receipt, err := ApplyTransaction(b.cm.config, evm, b.gasPool, b.statedb, b.header, tx, &b.header.GasUsed) if err != nil { panic(err) } @@ -360,12 +364,12 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse requests = append(requests, depositRequests) // create EVM for system calls blockContext := NewEVMBlockContext(b.header, cm, &b.header.Coinbase) - vmenv := vm.NewEVM(blockContext, vm.TxContext{}, statedb, cm.config, vm.Config{}) + evm := vm.NewEVM(blockContext, statedb, cm.config, vm.Config{}) // EIP-7002 withdrawals - withdrawalRequests := ProcessWithdrawalQueue(vmenv, statedb) + withdrawalRequests := ProcessWithdrawalQueue(evm, statedb) requests = append(requests, withdrawalRequests) // EIP-7251 consolidations - consolidationRequests := ProcessConsolidationQueue(vmenv, statedb) + consolidationRequests := ProcessConsolidationQueue(evm, statedb) requests = append(requests, consolidationRequests) } if requests != nil { @@ -466,8 +470,8 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine if config.IsPrague(b.header.Number, b.header.Time) { // EIP-2935 blockContext := NewEVMBlockContext(b.header, cm, &b.header.Coinbase) - vmenv := vm.NewEVM(blockContext, vm.TxContext{}, statedb, cm.config, vm.Config{}) - ProcessParentBlockHash(b.header.ParentHash, vmenv, statedb) + evm := vm.NewEVM(blockContext, statedb, cm.config, vm.Config{}) + ProcessParentBlockHash(b.header.ParentHash, evm, statedb) } // Execute any user modifications to the block. diff --git a/core/state_prefetcher.go b/core/state_prefetcher.go index 31405fa078..3b987bb289 100644 --- a/core/state_prefetcher.go +++ b/core/state_prefetcher.go @@ -49,7 +49,7 @@ func (p *statePrefetcher) Prefetch(block *types.Block, statedb *state.StateDB, c header = block.Header() gaspool = new(GasPool).AddGas(block.GasLimit()) blockContext = NewEVMBlockContext(header, p.chain, nil) - evm = vm.NewEVM(blockContext, vm.TxContext{}, statedb, p.config, cfg) + evm = vm.NewEVM(blockContext, statedb, p.config, cfg) signer = types.MakeSigner(p.config, header.Number, header.Time) ) // Iterate over and process the individual transactions @@ -65,7 +65,7 @@ func (p *statePrefetcher) Prefetch(block *types.Block, statedb *state.StateDB, c return // Also invalid block, bail out } statedb.SetTxContext(tx.Hash(), i) - if err := precacheTransaction(msg, p.config, gaspool, statedb, header, evm); err != nil { + if err := precacheTransaction(msg, gaspool, evm); err != nil { return // Ugh, something went horribly wrong, bail out } // If we're pre-byzantium, pre-load trie nodes for the intermediate root @@ -82,9 +82,9 @@ func (p *statePrefetcher) Prefetch(block *types.Block, statedb *state.StateDB, c // precacheTransaction attempts to apply a transaction to the given state database // and uses the input parameters for its environment. The goal is not to execute // the transaction successfully, rather to warm up touched data slots. -func precacheTransaction(msg *Message, config *params.ChainConfig, gaspool *GasPool, statedb *state.StateDB, header *types.Header, evm *vm.EVM) error { +func precacheTransaction(msg *Message, gaspool *GasPool, evm *vm.EVM) error { // Update the evm with the new transaction context. - evm.Reset(NewEVMTxContext(msg), statedb) + evm.SetTxContext(NewEVMTxContext(msg)) // Add addresses to access list if applicable _, err := ApplyMessage(evm, msg, gaspool) return err diff --git a/core/state_processor.go b/core/state_processor.go index c04049e986..74985b69a1 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -74,18 +74,18 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg ) // Apply pre-execution system calls. - context = NewEVMBlockContext(header, p.chain, nil) - - vmenv := vm.NewEVM(context, vm.TxContext{}, statedb, p.config, cfg) var tracingStateDB = vm.StateDB(statedb) if hooks := cfg.Tracer; hooks != nil { tracingStateDB = state.NewHookedState(statedb, hooks) } + context = NewEVMBlockContext(header, p.chain, nil) + evm := vm.NewEVM(context, tracingStateDB, p.config, cfg) + if beaconRoot := block.BeaconRoot(); beaconRoot != nil { - ProcessBeaconBlockRoot(*beaconRoot, vmenv, tracingStateDB) + ProcessBeaconBlockRoot(*beaconRoot, evm, tracingStateDB) } if p.config.IsPrague(block.Number(), block.Time()) { - ProcessParentBlockHash(block.ParentHash(), vmenv, tracingStateDB) + ProcessParentBlockHash(block.ParentHash(), evm, tracingStateDB) } // Iterate over and process the individual transactions @@ -96,7 +96,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg } statedb.SetTxContext(tx.Hash(), i) - receipt, err := ApplyTransactionWithEVM(msg, p.config, gp, statedb, blockNumber, blockHash, tx, usedGas, vmenv) + receipt, err := ApplyTransactionWithEVM(msg, p.config, gp, statedb, blockNumber, blockHash, tx, usedGas, evm) if err != nil { return nil, fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err) } @@ -113,10 +113,10 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg } requests = append(requests, depositRequests) // EIP-7002 withdrawals - withdrawalRequests := ProcessWithdrawalQueue(vmenv, tracingStateDB) + withdrawalRequests := ProcessWithdrawalQueue(evm, tracingStateDB) requests = append(requests, withdrawalRequests) // EIP-7251 consolidations - consolidationRequests := ProcessConsolidationQueue(vmenv, tracingStateDB) + consolidationRequests := ProcessConsolidationQueue(evm, tracingStateDB) requests = append(requests, consolidationRequests) } @@ -135,9 +135,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg // and uses the input parameters for its environment similar to ApplyTransaction. However, // this method takes an already created EVM instance as input. func ApplyTransactionWithEVM(msg *Message, config *params.ChainConfig, gp *GasPool, statedb *state.StateDB, blockNumber *big.Int, blockHash common.Hash, tx *types.Transaction, usedGas *uint64, evm *vm.EVM) (receipt *types.Receipt, err error) { - var tracingStateDB = vm.StateDB(statedb) if hooks := evm.Config.Tracer; hooks != nil { - tracingStateDB = state.NewHookedState(statedb, hooks) if hooks.OnTxStart != nil { hooks.OnTxStart(evm.GetVMContext(), tx, msg.From) } @@ -148,7 +146,7 @@ func ApplyTransactionWithEVM(msg *Message, config *params.ChainConfig, gp *GasPo // Create a new context to be used in the EVM environment. txContext := NewEVMTxContext(msg) - evm.Reset(txContext, tracingStateDB) + evm.SetTxContext(txContext) // Apply the transaction to the current state (included in the env). result, err := ApplyMessage(evm, msg, gp) @@ -159,7 +157,7 @@ func ApplyTransactionWithEVM(msg *Message, config *params.ChainConfig, gp *GasPo // Update the state with pending changes. var root []byte if config.IsByzantium(blockNumber) { - tracingStateDB.Finalise(true) + evm.StateDB.Finalise(true) } else { root = statedb.IntermediateRoot(config.IsEIP158(blockNumber)).Bytes() } @@ -210,16 +208,13 @@ func MakeReceipt(evm *vm.EVM, result *ExecutionResult, statedb *state.StateDB, b // and uses the input parameters for its environment. It returns the receipt // for the transaction, gas used and an error if the transaction failed, // indicating the block was invalid. -func ApplyTransaction(config *params.ChainConfig, bc ChainContext, author *common.Address, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, usedGas *uint64, cfg vm.Config) (*types.Receipt, error) { +func ApplyTransaction(config *params.ChainConfig, evm *vm.EVM, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, usedGas *uint64) (*types.Receipt, error) { msg, err := TransactionToMessage(tx, types.MakeSigner(config, header.Number, header.Time), header.BaseFee) if err != nil { return nil, err } // Create a new context to be used in the EVM environment - blockContext := NewEVMBlockContext(header, bc, author) - txContext := NewEVMTxContext(msg) - vmenv := vm.NewEVM(blockContext, txContext, statedb, config, cfg) - return ApplyTransactionWithEVM(msg, config, gp, statedb, header.Number, header.Hash(), tx, usedGas, vmenv) + return ApplyTransactionWithEVM(msg, config, gp, statedb, header.Number, header.Hash(), tx, usedGas, evm) } // ProcessBeaconBlockRoot applies the EIP-4788 system call to the beacon block root @@ -242,7 +237,7 @@ func ProcessBeaconBlockRoot(beaconRoot common.Hash, vmenv *vm.EVM, statedb vm.St To: ¶ms.BeaconRootsAddress, Data: beaconRoot[:], } - vmenv.Reset(NewEVMTxContext(msg), statedb) + vmenv.SetTxContext(NewEVMTxContext(msg)) statedb.AddAddressToAccessList(params.BeaconRootsAddress) _, _, _ = vmenv.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.U2560) statedb.Finalise(true) @@ -268,7 +263,7 @@ func ProcessParentBlockHash(prevHash common.Hash, vmenv *vm.EVM, statedb vm.Stat To: ¶ms.HistoryStorageAddress, Data: prevHash.Bytes(), } - vmenv.Reset(NewEVMTxContext(msg), statedb) + vmenv.SetTxContext(NewEVMTxContext(msg)) statedb.AddAddressToAccessList(params.HistoryStorageAddress) _, _, _ = vmenv.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.U2560) statedb.Finalise(true) @@ -304,7 +299,7 @@ func processRequestsSystemCall(vmenv *vm.EVM, statedb vm.StateDB, requestType by GasTipCap: common.Big0, To: &addr, } - vmenv.Reset(NewEVMTxContext(msg), statedb) + vmenv.SetTxContext(NewEVMTxContext(msg)) statedb.AddAddressToAccessList(addr) ret, _, _ := vmenv.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.U2560) statedb.Finalise(true) diff --git a/core/verkle_witness_test.go b/core/verkle_witness_test.go index b3088f0f90..80996eee33 100644 --- a/core/verkle_witness_test.go +++ b/core/verkle_witness_test.go @@ -225,7 +225,7 @@ func TestProcessParentBlockHash(t *testing.T) { for i := 1; i <= num; i++ { header := &types.Header{ParentHash: common.Hash{byte(i)}, Number: big.NewInt(int64(i)), Difficulty: new(big.Int)} vmContext := NewEVMBlockContext(header, nil, new(common.Address)) - evm := vm.NewEVM(vmContext, vm.TxContext{}, statedb, params.MergedTestChainConfig, vm.Config{}) + evm := vm.NewEVM(vmContext, statedb, params.MergedTestChainConfig, vm.Config{}) ProcessParentBlockHash(header.ParentHash, evm, statedb) } // Read block hashes for block 0 .. num-1 diff --git a/core/vm/evm.go b/core/vm/evm.go index 0593a32a3e..34e5fa766b 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -116,12 +116,13 @@ type EVM struct { precompiles map[common.Address]PrecompiledContract } -// NewEVM returns a new EVM. The returned EVM is not thread safe and should -// only ever be used *once*. -func NewEVM(blockCtx BlockContext, txCtx TxContext, statedb StateDB, chainConfig *params.ChainConfig, config Config) *EVM { +// NewEVM constructs an EVM instance with the supplied block context, state +// database and several configs. It meant to be used throughout the entire +// state transition of a block, with the transaction context switched as +// needed by calling evm.SetTxContext. +func NewEVM(blockCtx BlockContext, statedb StateDB, chainConfig *params.ChainConfig, config Config) *EVM { evm := &EVM{ Context: blockCtx, - TxContext: txCtx, StateDB: statedb, Config: config, chainConfig: chainConfig, @@ -132,6 +133,11 @@ func NewEVM(blockCtx BlockContext, txCtx TxContext, statedb StateDB, chainConfig return evm } +// SetTracer sets the tracer for following state transition. +func (evm *EVM) SetTracer(tracer *tracing.Hooks) { + evm.Config.Tracer = tracer +} + // SetPrecompiles sets the precompiled contracts for the EVM. // This method is only used through RPC calls. // It is not thread-safe. @@ -139,14 +145,13 @@ func (evm *EVM) SetPrecompiles(precompiles PrecompiledContracts) { evm.precompiles = precompiles } -// Reset resets the EVM with a new transaction context.Reset +// SetTxContext resets the EVM with a new transaction context. // This is not threadsafe and should only be done very cautiously. -func (evm *EVM) Reset(txCtx TxContext, statedb StateDB) { +func (evm *EVM) SetTxContext(txCtx TxContext) { if evm.chainRules.IsEIP4762 { - txCtx.AccessEvents = state.NewAccessEvents(statedb.PointCache()) + txCtx.AccessEvents = state.NewAccessEvents(evm.StateDB.PointCache()) } evm.TxContext = txCtx - evm.StateDB = statedb } // Cancel cancels any running EVM operation. This may be called concurrently and diff --git a/core/vm/gas_table_test.go b/core/vm/gas_table_test.go index babe9a5b6a..be86885261 100644 --- a/core/vm/gas_table_test.go +++ b/core/vm/gas_table_test.go @@ -95,16 +95,16 @@ func TestEIP2200(t *testing.T) { CanTransfer: func(StateDB, common.Address, *uint256.Int) bool { return true }, Transfer: func(StateDB, common.Address, common.Address, *uint256.Int) {}, } - vmenv := NewEVM(vmctx, TxContext{}, statedb, params.AllEthashProtocolChanges, Config{ExtraEips: []int{2200}}) + evm := NewEVM(vmctx, statedb, params.AllEthashProtocolChanges, Config{ExtraEips: []int{2200}}) - _, gas, err := vmenv.Call(AccountRef(common.Address{}), address, nil, tt.gaspool, new(uint256.Int)) + _, gas, err := evm.Call(AccountRef(common.Address{}), address, nil, tt.gaspool, new(uint256.Int)) if !errors.Is(err, tt.failure) { t.Errorf("test %d: failure mismatch: have %v, want %v", i, err, tt.failure) } if used := tt.gaspool - gas; used != tt.used { t.Errorf("test %d: gas used mismatch: have %v, want %v", i, used, tt.used) } - if refund := vmenv.StateDB.GetRefund(); refund != tt.refund { + if refund := evm.StateDB.GetRefund(); refund != tt.refund { t.Errorf("test %d: gas refund mismatch: have %v, want %v", i, refund, tt.refund) } } @@ -151,9 +151,9 @@ func TestCreateGas(t *testing.T) { config.ExtraEips = []int{3860} } - vmenv := NewEVM(vmctx, TxContext{}, statedb, params.AllEthashProtocolChanges, config) + evm := NewEVM(vmctx, statedb, params.AllEthashProtocolChanges, config) var startGas = uint64(testGas) - ret, gas, err := vmenv.Call(AccountRef(common.Address{}), address, nil, startGas, new(uint256.Int)) + ret, gas, err := evm.Call(AccountRef(common.Address{}), address, nil, startGas, new(uint256.Int)) if err != nil { return false } diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index b8e62e1de5..08f2b2bfea 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -104,10 +104,9 @@ func init() { func testTwoOperandOp(t *testing.T, tests []TwoOperandTestcase, opFn executionFunc, name string) { var ( - env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() - pc = uint64(0) - evmInterpreter = env.interpreter + evm = NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) + stack = newstack() + pc = uint64(0) ) for i, test := range tests { @@ -116,7 +115,7 @@ func testTwoOperandOp(t *testing.T, tests []TwoOperandTestcase, opFn executionFu expected := new(uint256.Int).SetBytes(common.Hex2Bytes(test.Expected)) stack.push(x) stack.push(y) - opFn(&pc, evmInterpreter, &ScopeContext{nil, stack, nil}) + opFn(&pc, evm.interpreter, &ScopeContext{nil, stack, nil}) if len(stack.data) != 1 { t.Errorf("Expected one item on stack after %v, got %d: ", name, len(stack.data)) } @@ -203,10 +202,9 @@ func TestSAR(t *testing.T) { func TestAddMod(t *testing.T) { var ( - env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() - evmInterpreter = NewEVMInterpreter(env) - pc = uint64(0) + evm = NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) + stack = newstack() + pc = uint64(0) ) tests := []struct { x string @@ -231,7 +229,7 @@ func TestAddMod(t *testing.T) { stack.push(z) stack.push(y) stack.push(x) - opAddmod(&pc, evmInterpreter, &ScopeContext{nil, stack, nil}) + opAddmod(&pc, evm.interpreter, &ScopeContext{nil, stack, nil}) actual := stack.pop() if actual.Cmp(expected) != 0 { t.Errorf("Testcase %d, expected %x, got %x", i, expected, actual) @@ -247,10 +245,9 @@ func TestWriteExpectedValues(t *testing.T) { // getResult is a convenience function to generate the expected values getResult := func(args []*twoOperandParams, opFn executionFunc) []TwoOperandTestcase { var ( - env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() - pc = uint64(0) - interpreter = env.interpreter + evm = NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) + stack = newstack() + pc = uint64(0) ) result := make([]TwoOperandTestcase, len(args)) for i, param := range args { @@ -258,7 +255,7 @@ func TestWriteExpectedValues(t *testing.T) { y := new(uint256.Int).SetBytes(common.Hex2Bytes(param.y)) stack.push(x) stack.push(y) - opFn(&pc, interpreter, &ScopeContext{nil, stack, nil}) + opFn(&pc, evm.interpreter, &ScopeContext{nil, stack, nil}) actual := stack.pop() result[i] = TwoOperandTestcase{param.x, param.y, fmt.Sprintf("%064x", actual)} } @@ -292,13 +289,10 @@ func TestJsonTestcases(t *testing.T) { func opBenchmark(bench *testing.B, op executionFunc, args ...string) { var ( - env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() - scope = &ScopeContext{nil, stack, nil} - evmInterpreter = NewEVMInterpreter(env) + evm = NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) + stack = newstack() + scope = &ScopeContext{nil, stack, nil} ) - - env.interpreter = evmInterpreter // convert args intArgs := make([]*uint256.Int, len(args)) for i, arg := range args { @@ -310,7 +304,7 @@ func opBenchmark(bench *testing.B, op executionFunc, args ...string) { for _, arg := range intArgs { stack.push(arg) } - op(&pc, evmInterpreter, scope) + op(&pc, evm.interpreter, scope) stack.pop() } bench.StopTimer() @@ -533,25 +527,22 @@ func BenchmarkOpIsZero(b *testing.B) { func TestOpMstore(t *testing.T) { var ( - env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() - mem = NewMemory() - evmInterpreter = NewEVMInterpreter(env) + evm = NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) + stack = newstack() + mem = NewMemory() ) - - env.interpreter = evmInterpreter mem.Resize(64) pc := uint64(0) v := "abcdef00000000000000abba000000000deaf000000c0de00100000000133700" stack.push(new(uint256.Int).SetBytes(common.Hex2Bytes(v))) stack.push(new(uint256.Int)) - opMstore(&pc, evmInterpreter, &ScopeContext{mem, stack, nil}) + opMstore(&pc, evm.interpreter, &ScopeContext{mem, stack, nil}) if got := common.Bytes2Hex(mem.GetCopy(0, 32)); got != v { t.Fatalf("Mstore fail, got %v, expected %v", got, v) } stack.push(new(uint256.Int).SetUint64(0x1)) stack.push(new(uint256.Int)) - opMstore(&pc, evmInterpreter, &ScopeContext{mem, stack, nil}) + opMstore(&pc, evm.interpreter, &ScopeContext{mem, stack, nil}) if common.Bytes2Hex(mem.GetCopy(0, 32)) != "0000000000000000000000000000000000000000000000000000000000000001" { t.Fatalf("Mstore failed to overwrite previous value") } @@ -559,13 +550,10 @@ func TestOpMstore(t *testing.T) { func BenchmarkOpMstore(bench *testing.B) { var ( - env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() - mem = NewMemory() - evmInterpreter = NewEVMInterpreter(env) + evm = NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) + stack = newstack() + mem = NewMemory() ) - - env.interpreter = evmInterpreter mem.Resize(64) pc := uint64(0) memStart := new(uint256.Int) @@ -575,43 +563,41 @@ func BenchmarkOpMstore(bench *testing.B) { for i := 0; i < bench.N; i++ { stack.push(value) stack.push(memStart) - opMstore(&pc, evmInterpreter, &ScopeContext{mem, stack, nil}) + opMstore(&pc, evm.interpreter, &ScopeContext{mem, stack, nil}) } } func TestOpTstore(t *testing.T) { var ( - statedb, _ = state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) - env = NewEVM(BlockContext{}, TxContext{}, statedb, params.TestChainConfig, Config{}) - stack = newstack() - mem = NewMemory() - evmInterpreter = NewEVMInterpreter(env) - caller = common.Address{} - to = common.Address{1} - contractRef = contractRef{caller} - contract = NewContract(contractRef, AccountRef(to), new(uint256.Int), 0) - scopeContext = ScopeContext{mem, stack, contract} - value = common.Hex2Bytes("abcdef00000000000000abba000000000deaf000000c0de00100000000133700") + statedb, _ = state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) + evm = NewEVM(BlockContext{}, statedb, params.TestChainConfig, Config{}) + stack = newstack() + mem = NewMemory() + caller = common.Address{} + to = common.Address{1} + contractRef = contractRef{caller} + contract = NewContract(contractRef, AccountRef(to), new(uint256.Int), 0) + scopeContext = ScopeContext{mem, stack, contract} + value = common.Hex2Bytes("abcdef00000000000000abba000000000deaf000000c0de00100000000133700") ) // Add a stateObject for the caller and the contract being called statedb.CreateAccount(caller) statedb.CreateAccount(to) - env.interpreter = evmInterpreter pc := uint64(0) // push the value to the stack stack.push(new(uint256.Int).SetBytes(value)) // push the location to the stack stack.push(new(uint256.Int)) - opTstore(&pc, evmInterpreter, &scopeContext) + opTstore(&pc, evm.interpreter, &scopeContext) // there should be no elements on the stack after TSTORE if stack.len() != 0 { t.Fatal("stack wrong size") } // push the location to the stack stack.push(new(uint256.Int)) - opTload(&pc, evmInterpreter, &scopeContext) + opTload(&pc, evm.interpreter, &scopeContext) // there should be one element on the stack after TLOAD if stack.len() != 1 { t.Fatal("stack wrong size") @@ -624,12 +610,10 @@ func TestOpTstore(t *testing.T) { func BenchmarkOpKeccak256(bench *testing.B) { var ( - env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() - mem = NewMemory() - evmInterpreter = NewEVMInterpreter(env) + evm = NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) + stack = newstack() + mem = NewMemory() ) - env.interpreter = evmInterpreter mem.Resize(32) pc := uint64(0) start := new(uint256.Int) @@ -638,7 +622,7 @@ func BenchmarkOpKeccak256(bench *testing.B) { for i := 0; i < bench.N; i++ { stack.push(uint256.NewInt(32)) stack.push(start) - opKeccak256(&pc, evmInterpreter, &ScopeContext{mem, stack, nil}) + opKeccak256(&pc, evm.interpreter, &ScopeContext{mem, stack, nil}) } } @@ -728,12 +712,11 @@ func TestRandom(t *testing.T) { {name: "hash(0x010203)", random: crypto.Keccak256Hash([]byte{0x01, 0x02, 0x03})}, } { var ( - env = NewEVM(BlockContext{Random: &tt.random}, TxContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() - pc = uint64(0) - evmInterpreter = env.interpreter + evm = NewEVM(BlockContext{Random: &tt.random}, nil, params.TestChainConfig, Config{}) + stack = newstack() + pc = uint64(0) ) - opRandom(&pc, evmInterpreter, &ScopeContext{nil, stack, nil}) + opRandom(&pc, evm.interpreter, &ScopeContext{nil, stack, nil}) if len(stack.data) != 1 { t.Errorf("Expected one item on stack after %v, got %d: ", tt.name, len(stack.data)) } @@ -769,13 +752,13 @@ func TestBlobHash(t *testing.T) { {name: "out-of-bounds (nil)", idx: 25, expect: zero, hashes: nil}, } { var ( - env = NewEVM(BlockContext{}, TxContext{BlobHashes: tt.hashes}, nil, params.TestChainConfig, Config{}) - stack = newstack() - pc = uint64(0) - evmInterpreter = env.interpreter + evm = NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) + stack = newstack() + pc = uint64(0) ) + evm.SetTxContext(TxContext{BlobHashes: tt.hashes}) stack.push(uint256.NewInt(tt.idx)) - opBlobHash(&pc, evmInterpreter, &ScopeContext{nil, stack, nil}) + opBlobHash(&pc, evm.interpreter, &ScopeContext{nil, stack, nil}) if len(stack.data) != 1 { t.Errorf("Expected one item on stack after %v, got %d: ", tt.name, len(stack.data)) } @@ -872,10 +855,9 @@ func TestOpMCopy(t *testing.T) { }, } { var ( - env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() - pc = uint64(0) - evmInterpreter = env.interpreter + evm = NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) + stack = newstack() + pc = uint64(0) ) data := common.FromHex(strings.ReplaceAll(tc.pre, " ", "")) // Set pre @@ -906,7 +888,7 @@ func TestOpMCopy(t *testing.T) { } // and the dynamic cost var haveGas uint64 - if dynamicCost, err := gasMcopy(env, nil, stack, mem, memorySize); err != nil { + if dynamicCost, err := gasMcopy(evm, nil, stack, mem, memorySize); err != nil { t.Error(err) } else { haveGas = GasFastestStep + dynamicCost @@ -916,7 +898,7 @@ func TestOpMCopy(t *testing.T) { mem.Resize(memorySize) } // Do the copy - opMcopy(&pc, evmInterpreter, &ScopeContext{mem, stack, nil}) + opMcopy(&pc, evm.interpreter, &ScopeContext{mem, stack, nil}) want := common.FromHex(strings.ReplaceAll(tc.want, " ", "")) if have := mem.store; !bytes.Equal(want, have) { t.Errorf("case %d: \nwant: %#x\nhave: %#x\n", i, want, have) diff --git a/core/vm/interpreter_test.go b/core/vm/interpreter_test.go index 7d4b2ddf36..cacad8f813 100644 --- a/core/vm/interpreter_test.go +++ b/core/vm/interpreter_test.go @@ -47,7 +47,7 @@ func TestLoopInterrupt(t *testing.T) { statedb.SetCode(address, common.Hex2Bytes(tt)) statedb.Finalise(true) - evm := NewEVM(vmctx, TxContext{}, statedb, params.AllEthashProtocolChanges, Config{}) + evm := NewEVM(vmctx, statedb, params.AllEthashProtocolChanges, Config{}) errChannel := make(chan error) timeout := make(chan bool) diff --git a/core/vm/runtime/env.go b/core/vm/runtime/env.go index 34335b8e9e..e54041f7e2 100644 --- a/core/vm/runtime/env.go +++ b/core/vm/runtime/env.go @@ -42,5 +42,7 @@ func NewEnv(cfg *Config) *vm.EVM { Random: cfg.Random, } - return vm.NewEVM(blockContext, txContext, cfg.State, cfg.ChainConfig, cfg.EVMConfig) + evm := vm.NewEVM(blockContext, cfg.State, cfg.ChainConfig, cfg.EVMConfig) + evm.SetTxContext(txContext) + return evm } diff --git a/eth/api_backend.go b/eth/api_backend.go index 4e81d68e07..be2101c6ec 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -249,18 +249,17 @@ func (b *EthAPIBackend) GetTd(ctx context.Context, hash common.Hash) *big.Int { return nil } -func (b *EthAPIBackend) GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config, blockCtx *vm.BlockContext) *vm.EVM { +func (b *EthAPIBackend) GetEVM(ctx context.Context, state *state.StateDB, header *types.Header, vmConfig *vm.Config, blockCtx *vm.BlockContext) *vm.EVM { if vmConfig == nil { vmConfig = b.eth.blockchain.GetVMConfig() } - txContext := core.NewEVMTxContext(msg) var context vm.BlockContext if blockCtx != nil { context = *blockCtx } else { context = core.NewEVMBlockContext(header, b.eth.BlockChain(), nil) } - return vm.NewEVM(context, txContext, state, b.ChainConfig(), *vmConfig) + return vm.NewEVM(context, state, b.ChainConfig(), *vmConfig) } func (b *EthAPIBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { diff --git a/eth/gasestimator/gasestimator.go b/eth/gasestimator/gasestimator.go index d43057dda2..5729f84278 100644 --- a/eth/gasestimator/gasestimator.go +++ b/eth/gasestimator/gasestimator.go @@ -230,7 +230,8 @@ func run(ctx context.Context, call *core.Message, opts *Options) (*core.Executio if msgContext.BlobFeeCap != nil && msgContext.BlobFeeCap.BitLen() == 0 { evmContext.BlobBaseFee = new(big.Int) } - evm := vm.NewEVM(evmContext, msgContext, dirtyState, opts.Config, vm.Config{NoBaseFee: true}) + evm := vm.NewEVM(evmContext, dirtyState, opts.Config, vm.Config{NoBaseFee: true}) + evm.SetTxContext(msgContext) // Monitor the outer context and interrupt the EVM upon cancellation. To avoid // a dangling goroutine until the outer estimation finishes, create an internal // context for the lifetime of this method call. diff --git a/eth/state_accessor.go b/eth/state_accessor.go index cb5a233a83..a919030249 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -235,16 +235,14 @@ func (eth *Ethereum) stateAtTransaction(ctx context.Context, block *types.Block, return nil, vm.BlockContext{}, nil, nil, err } // Insert parent beacon block root in the state as per EIP-4788. + context := core.NewEVMBlockContext(block.Header(), eth.blockchain, nil) + evm := vm.NewEVM(context, statedb, eth.blockchain.Config(), vm.Config{}) if beaconRoot := block.BeaconRoot(); beaconRoot != nil { - context := core.NewEVMBlockContext(block.Header(), eth.blockchain, nil) - vmenv := vm.NewEVM(context, vm.TxContext{}, statedb, eth.blockchain.Config(), vm.Config{}) - core.ProcessBeaconBlockRoot(*beaconRoot, vmenv, statedb) + core.ProcessBeaconBlockRoot(*beaconRoot, evm, statedb) } // If prague hardfork, insert parent block hash in the state as per EIP-2935. if eth.blockchain.Config().IsPrague(block.Number(), block.Time()) { - context := core.NewEVMBlockContext(block.Header(), eth.blockchain, nil) - vmenv := vm.NewEVM(context, vm.TxContext{}, statedb, eth.blockchain.Config(), vm.Config{}) - core.ProcessParentBlockHash(block.ParentHash(), vmenv, statedb) + core.ProcessParentBlockHash(block.ParentHash(), evm, statedb) } if txIndex == 0 && len(block.Transactions()) == 0 { return nil, vm.BlockContext{}, statedb, release, nil @@ -252,22 +250,22 @@ func (eth *Ethereum) stateAtTransaction(ctx context.Context, block *types.Block, // Recompute transactions up to the target index. signer := types.MakeSigner(eth.blockchain.Config(), block.Number(), block.Time()) for idx, tx := range block.Transactions() { - // Assemble the transaction call message and return if the requested offset - msg, _ := core.TransactionToMessage(tx, signer, block.BaseFee()) - txContext := core.NewEVMTxContext(msg) - context := core.NewEVMBlockContext(block.Header(), eth.blockchain, nil) if idx == txIndex { return tx, context, statedb, release, nil } + // Assemble the transaction call message and return if the requested offset + msg, _ := core.TransactionToMessage(tx, signer, block.BaseFee()) + txContext := core.NewEVMTxContext(msg) + evm.SetTxContext(txContext) + // Not yet the searched for transaction, execute on top of the current state - vmenv := vm.NewEVM(context, txContext, statedb, eth.blockchain.Config(), vm.Config{}) statedb.SetTxContext(tx.Hash(), idx) - if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { + if _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) } // Ensure any modifications are committed to the state // Only delete empty objects if EIP158/161 (a.k.a Spurious Dragon) is in effect - statedb.Finalise(vmenv.ChainConfig().IsEIP158(block.Number())) + statedb.Finalise(evm.ChainConfig().IsEIP158(block.Number())) } return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction index %d out of range for block %#x", txIndex, block.Hash()) } diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 7d8191c25b..671a86e78a 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -378,16 +378,14 @@ func (api *API) traceChain(start, end *types.Block, config *TraceConfig, closed } // Insert block's parent beacon block root in the state // as per EIP-4788. + context := core.NewEVMBlockContext(next.Header(), api.chainContext(ctx), nil) + evm := vm.NewEVM(context, statedb, api.backend.ChainConfig(), vm.Config{}) if beaconRoot := next.BeaconRoot(); beaconRoot != nil { - context := core.NewEVMBlockContext(next.Header(), api.chainContext(ctx), nil) - vmenv := vm.NewEVM(context, vm.TxContext{}, statedb, api.backend.ChainConfig(), vm.Config{}) - core.ProcessBeaconBlockRoot(*beaconRoot, vmenv, statedb) + core.ProcessBeaconBlockRoot(*beaconRoot, evm, statedb) } // Insert parent hash in history contract. if api.backend.ChainConfig().IsPrague(next.Number(), next.Time()) { - context := core.NewEVMBlockContext(next.Header(), api.chainContext(ctx), nil) - vmenv := vm.NewEVM(context, vm.TxContext{}, statedb, api.backend.ChainConfig(), vm.Config{}) - core.ProcessParentBlockHash(next.ParentHash(), vmenv, statedb) + core.ProcessParentBlockHash(next.ParentHash(), evm, statedb) } // Clean out any pending release functions of trace state. Note this // step must be done after constructing tracing state, because the @@ -537,12 +535,12 @@ func (api *API) IntermediateRoots(ctx context.Context, hash common.Hash, config vmctx = core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) deleteEmptyObjects = chainConfig.IsEIP158(block.Number()) ) + evm := vm.NewEVM(vmctx, statedb, chainConfig, vm.Config{}) if beaconRoot := block.BeaconRoot(); beaconRoot != nil { - vmenv := vm.NewEVM(vmctx, vm.TxContext{}, statedb, chainConfig, vm.Config{}) - core.ProcessBeaconBlockRoot(*beaconRoot, vmenv, statedb) + core.ProcessBeaconBlockRoot(*beaconRoot, evm, statedb) } if chainConfig.IsPrague(block.Number(), block.Time()) { - core.ProcessParentBlockHash(block.ParentHash(), vm.NewEVM(vmctx, vm.TxContext{}, statedb, chainConfig, vm.Config{}), statedb) + core.ProcessParentBlockHash(block.ParentHash(), evm, statedb) } for i, tx := range block.Transactions() { if err := ctx.Err(); err != nil { @@ -551,10 +549,10 @@ func (api *API) IntermediateRoots(ctx context.Context, hash common.Hash, config var ( msg, _ = core.TransactionToMessage(tx, signer, block.BaseFee()) txContext = core.NewEVMTxContext(msg) - vmenv = vm.NewEVM(vmctx, txContext, statedb, chainConfig, vm.Config{}) ) + evm.SetTxContext(txContext) statedb.SetTxContext(tx.Hash(), i) - if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.GasLimit)); err != nil { + if _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(msg.GasLimit)); err != nil { log.Warn("Tracing intermediate roots did not complete", "txindex", i, "txhash", tx.Hash(), "err", err) // We intentionally don't return the error here: if we do, then the RPC server will not // return the roots. Most likely, the caller already knows that a certain transaction fails to @@ -605,13 +603,12 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac defer release() blockCtx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) + evm := vm.NewEVM(blockCtx, statedb, api.backend.ChainConfig(), vm.Config{}) if beaconRoot := block.BeaconRoot(); beaconRoot != nil { - vmenv := vm.NewEVM(blockCtx, vm.TxContext{}, statedb, api.backend.ChainConfig(), vm.Config{}) - core.ProcessBeaconBlockRoot(*beaconRoot, vmenv, statedb) + core.ProcessBeaconBlockRoot(*beaconRoot, evm, statedb) } if api.backend.ChainConfig().IsPrague(block.Number(), block.Time()) { - vmenv := vm.NewEVM(blockCtx, vm.TxContext{}, statedb, api.backend.ChainConfig(), vm.Config{}) - core.ProcessParentBlockHash(block.ParentHash(), vmenv, statedb) + core.ProcessParentBlockHash(block.ParentHash(), evm, statedb) } // JS tracers have high overhead. In this case run a parallel @@ -695,6 +692,8 @@ func (api *API) traceBlockParallel(ctx context.Context, block *types.Block, stat // Feed the transactions into the tracers and return var failed error blockCtx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) + evm := vm.NewEVM(blockCtx, statedb, api.backend.ChainConfig(), vm.Config{}) + txloop: for i, tx := range txs { // Send the trace task over for execution @@ -709,14 +708,14 @@ txloop: // Generate the next state snapshot fast without tracing msg, _ := core.TransactionToMessage(tx, signer, block.BaseFee()) statedb.SetTxContext(tx.Hash(), i) - vmenv := vm.NewEVM(blockCtx, core.NewEVMTxContext(msg), statedb, api.backend.ChainConfig(), vm.Config{}) - if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.GasLimit)); err != nil { + evm.SetTxContext(core.NewEVMTxContext(msg)) + if _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(msg.GasLimit)); err != nil { failed = err break txloop } // Finalize the state so any modifications are written to the trie // Only delete empty objects if EIP158/161 (a.k.a Spurious Dragon) is in effect - statedb.Finalise(vmenv.ChainConfig().IsEIP158(block.Number())) + statedb.Finalise(evm.ChainConfig().IsEIP158(block.Number())) } close(jobs) @@ -783,13 +782,12 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block // Note: This copies the config, to not screw up the main config chainConfig, canon = overrideConfig(chainConfig, config.Overrides) } + evm := vm.NewEVM(vmctx, statedb, chainConfig, vm.Config{}) if beaconRoot := block.BeaconRoot(); beaconRoot != nil { - vmenv := vm.NewEVM(vmctx, vm.TxContext{}, statedb, chainConfig, vm.Config{}) - core.ProcessBeaconBlockRoot(*beaconRoot, vmenv, statedb) + core.ProcessBeaconBlockRoot(*beaconRoot, evm, statedb) } if chainConfig.IsPrague(block.Number(), block.Time()) { - vmenv := vm.NewEVM(vmctx, vm.TxContext{}, statedb, chainConfig, vm.Config{}) - core.ProcessParentBlockHash(block.ParentHash(), vmenv, statedb) + core.ProcessParentBlockHash(block.ParentHash(), evm, statedb) } for i, tx := range block.Transactions() { // Prepare the transaction for un-traced execution @@ -822,12 +820,12 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block } } // Execute the transaction and flush any traces to disk - vmenv := vm.NewEVM(vmctx, txContext, statedb, chainConfig, vmConf) + evm.SetTxContext(txContext) statedb.SetTxContext(tx.Hash(), i) if vmConf.Tracer.OnTxStart != nil { - vmConf.Tracer.OnTxStart(vmenv.GetVMContext(), tx, msg.From) + vmConf.Tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) } - vmRet, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.GasLimit)) + vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(msg.GasLimit)) if vmConf.Tracer.OnTxEnd != nil { vmConf.Tracer.OnTxEnd(&types.Receipt{GasUsed: vmRet.UsedGas}, err) } @@ -843,7 +841,7 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block } // Finalize the state so any modifications are written to the trie // Only delete empty objects if EIP158/161 (a.k.a Spurious Dragon) is in effect - statedb.Finalise(vmenv.ChainConfig().IsEIP158(block.Number())) + statedb.Finalise(evm.ChainConfig().IsEIP158(block.Number())) // If we've traced the transaction we were looking for, abort if tx.Hash() == txHash { @@ -1017,7 +1015,8 @@ func (api *API) traceTx(ctx context.Context, tx *types.Transaction, message *cor } } // The actual TxContext will be created as part of ApplyTransactionWithEVM. - vmenv := vm.NewEVM(vmctx, vm.TxContext{GasPrice: message.GasPrice, BlobFeeCap: message.BlobGasFeeCap}, statedb, api.backend.ChainConfig(), vm.Config{Tracer: tracer.Hooks, NoBaseFee: true}) + evm := vm.NewEVM(vmctx, statedb, api.backend.ChainConfig(), vm.Config{Tracer: tracer.Hooks, NoBaseFee: true}) + evm.SetTxContext(vm.TxContext{GasPrice: message.GasPrice, BlobFeeCap: message.BlobGasFeeCap}) // Define a meaningful timeout of a single transaction trace if config.Timeout != nil { @@ -1031,14 +1030,14 @@ func (api *API) traceTx(ctx context.Context, tx *types.Transaction, message *cor if errors.Is(deadlineCtx.Err(), context.DeadlineExceeded) { tracer.Stop(errors.New("execution timeout")) // Stop evm execution. Note cancellation is not necessarily immediate. - vmenv.Cancel() + evm.Cancel() } }() defer cancel() // Call Prepare to clear out the statedb access list statedb.SetTxContext(txctx.TxHash, txctx.TxIndex) - _, err = core.ApplyTransactionWithEVM(message, api.backend.ChainConfig(), new(core.GasPool).AddGas(message.GasLimit), statedb, vmctx.BlockNumber, txctx.BlockHash, tx, &usedGas, vmenv) + _, err = core.ApplyTransactionWithEVM(message, api.backend.ChainConfig(), new(core.GasPool).AddGas(message.GasLimit), statedb, vmctx.BlockNumber, txctx.BlockHash, tx, &usedGas, evm) if err != nil { return nil, fmt.Errorf("tracing failed: %w", err) } diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index 47e3693495..2b5cfa35c8 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -170,18 +170,19 @@ func (b *testBackend) StateAtTransaction(ctx context.Context, block *types.Block } // Recompute transactions up to the target index. signer := types.MakeSigner(b.chainConfig, block.Number(), block.Time()) + context := core.NewEVMBlockContext(block.Header(), b.chain, nil) + evm := vm.NewEVM(context, statedb, b.chainConfig, vm.Config{}) for idx, tx := range block.Transactions() { - msg, _ := core.TransactionToMessage(tx, signer, block.BaseFee()) - txContext := core.NewEVMTxContext(msg) - context := core.NewEVMBlockContext(block.Header(), b.chain, nil) if idx == txIndex { return tx, context, statedb, release, nil } - vmenv := vm.NewEVM(context, txContext, statedb, b.chainConfig, vm.Config{}) - if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { + msg, _ := core.TransactionToMessage(tx, signer, block.BaseFee()) + txContext := core.NewEVMTxContext(msg) + evm.SetTxContext(txContext) + if _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) } - statedb.Finalise(vmenv.ChainConfig().IsEIP158(block.Number())) + statedb.Finalise(evm.ChainConfig().IsEIP158(block.Number())) } return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction index %d out of range for block %#x", txIndex, block.Hash()) } diff --git a/eth/tracers/internal/tracetest/calltrace_test.go b/eth/tracers/internal/tracetest/calltrace_test.go index cb635e7127..869558c324 100644 --- a/eth/tracers/internal/tracetest/calltrace_test.go +++ b/eth/tracers/internal/tracetest/calltrace_test.go @@ -132,7 +132,8 @@ func testCallTracer(tracerName string, dirPath string, t *testing.T) { if err != nil { t.Fatalf("failed to prepare transaction for tracing: %v", err) } - evm := vm.NewEVM(context, core.NewEVMTxContext(msg), logState, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) + evm := vm.NewEVM(context, logState, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) + evm.SetTxContext(core.NewEVMTxContext(msg)) tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) if err != nil { @@ -220,12 +221,15 @@ func benchTracer(tracerName string, test *callTracerTest, b *testing.B) { b.ReportAllocs() b.ResetTimer() + + tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), nil, test.Genesis.Config) + if err != nil { + b.Fatalf("failed to create call tracer: %v", err) + } + evm := vm.NewEVM(context, state.StateDB, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) + evm.SetTxContext(txContext) + for i := 0; i < b.N; i++ { - tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), nil, test.Genesis.Config) - if err != nil { - b.Fatalf("failed to create call tracer: %v", err) - } - evm := vm.NewEVM(context, txContext, state.StateDB, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) snap := state.StateDB.Snapshot() st := core.NewStateTransition(evm, msg, new(core.GasPool).AddGas(tx.Gas())) if _, err = st.TransitionDb(); err != nil { @@ -372,7 +376,8 @@ func TestInternals(t *testing.T) { Origin: origin, GasPrice: tx.GasPrice(), } - evm := vm.NewEVM(context, txContext, logState, config, vm.Config{Tracer: tc.tracer.Hooks}) + evm := vm.NewEVM(context, logState, config, vm.Config{Tracer: tc.tracer.Hooks}) + evm.SetTxContext(txContext) msg, err := core.TransactionToMessage(tx, signer, big.NewInt(0)) if err != nil { t.Fatalf("test %v: failed to create message: %v", tc.name, err) diff --git a/eth/tracers/internal/tracetest/flat_calltrace_test.go b/eth/tracers/internal/tracetest/flat_calltrace_test.go index 0ec3c367bc..1dbdc7caac 100644 --- a/eth/tracers/internal/tracetest/flat_calltrace_test.go +++ b/eth/tracers/internal/tracetest/flat_calltrace_test.go @@ -98,7 +98,8 @@ func flatCallTracerTestRunner(tracerName string, filename string, dirPath string if err != nil { return fmt.Errorf("failed to prepare transaction for tracing: %v", err) } - evm := vm.NewEVM(context, core.NewEVMTxContext(msg), state.StateDB, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) + evm := vm.NewEVM(context, state.StateDB, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) + evm.SetTxContext(core.NewEVMTxContext(msg)) tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) if err != nil { diff --git a/eth/tracers/internal/tracetest/prestate_test.go b/eth/tracers/internal/tracetest/prestate_test.go index c6cf10a483..b8dec51db1 100644 --- a/eth/tracers/internal/tracetest/prestate_test.go +++ b/eth/tracers/internal/tracetest/prestate_test.go @@ -106,7 +106,8 @@ func testPrestateDiffTracer(tracerName string, dirPath string, t *testing.T) { if err != nil { t.Fatalf("failed to prepare transaction for tracing: %v", err) } - evm := vm.NewEVM(context, core.NewEVMTxContext(msg), state.StateDB, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) + evm := vm.NewEVM(context, state.StateDB, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) + evm.SetTxContext(core.NewEVMTxContext(msg)) tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) if err != nil { diff --git a/eth/tracers/js/tracer_test.go b/eth/tracers/js/tracer_test.go index ed2789d70d..54f628b5f3 100644 --- a/eth/tracers/js/tracer_test.go +++ b/eth/tracers/js/tracer_test.go @@ -64,20 +64,21 @@ func testCtx() *vmContext { func runTrace(tracer *tracers.Tracer, vmctx *vmContext, chaincfg *params.ChainConfig, contractCode []byte) (json.RawMessage, error) { var ( - env = vm.NewEVM(vmctx.blockCtx, vmctx.txCtx, &dummyStatedb{}, chaincfg, vm.Config{Tracer: tracer.Hooks}) + evm = vm.NewEVM(vmctx.blockCtx, &dummyStatedb{}, chaincfg, vm.Config{Tracer: tracer.Hooks}) gasLimit uint64 = 31000 startGas uint64 = 10000 value = uint256.NewInt(0) contract = vm.NewContract(account{}, account{}, value, startGas) ) + evm.SetTxContext(vmctx.txCtx) contract.Code = []byte{byte(vm.PUSH1), 0x1, byte(vm.PUSH1), 0x1, 0x0} if contractCode != nil { contract.Code = contractCode } - tracer.OnTxStart(env.GetVMContext(), types.NewTx(&types.LegacyTx{Gas: gasLimit}), contract.Caller()) + tracer.OnTxStart(evm.GetVMContext(), types.NewTx(&types.LegacyTx{Gas: gasLimit}), contract.Caller()) tracer.OnEnter(0, byte(vm.CALL), contract.Caller(), contract.Address(), []byte{}, startGas, value.ToBig()) - ret, err := env.Interpreter().Run(contract, []byte{}, false) + ret, err := evm.Interpreter().Run(contract, []byte{}, false) tracer.OnExit(0, ret, startGas-contract.Gas, err, true) // Rest gas assumes no refund tracer.OnTxEnd(&types.Receipt{GasUsed: gasLimit - contract.Gas}, nil) @@ -191,8 +192,9 @@ func TestHaltBetweenSteps(t *testing.T) { scope := &vm.ScopeContext{ Contract: vm.NewContract(&account{}, &account{}, uint256.NewInt(0), 0), } - env := vm.NewEVM(vm.BlockContext{BlockNumber: big.NewInt(1)}, vm.TxContext{GasPrice: big.NewInt(1)}, &dummyStatedb{}, chainConfig, vm.Config{Tracer: tracer.Hooks}) - tracer.OnTxStart(env.GetVMContext(), types.NewTx(&types.LegacyTx{}), common.Address{}) + evm := vm.NewEVM(vm.BlockContext{BlockNumber: big.NewInt(1)}, &dummyStatedb{}, chainConfig, vm.Config{Tracer: tracer.Hooks}) + evm.SetTxContext(vm.TxContext{GasPrice: big.NewInt(1)}) + tracer.OnTxStart(evm.GetVMContext(), types.NewTx(&types.LegacyTx{}), common.Address{}) tracer.OnEnter(0, byte(vm.CALL), common.Address{}, common.Address{}, []byte{}, 0, big.NewInt(0)) tracer.OnOpcode(0, 0, 0, 0, scope, nil, 0, nil) timeout := errors.New("stahp") @@ -214,8 +216,9 @@ func TestNoStepExec(t *testing.T) { if err != nil { t.Fatal(err) } - env := vm.NewEVM(vm.BlockContext{BlockNumber: big.NewInt(1)}, vm.TxContext{GasPrice: big.NewInt(100)}, &dummyStatedb{}, chainConfig, vm.Config{Tracer: tracer.Hooks}) - tracer.OnTxStart(env.GetVMContext(), types.NewTx(&types.LegacyTx{}), common.Address{}) + evm := vm.NewEVM(vm.BlockContext{BlockNumber: big.NewInt(1)}, &dummyStatedb{}, chainConfig, vm.Config{Tracer: tracer.Hooks}) + evm.SetTxContext(vm.TxContext{GasPrice: big.NewInt(100)}) + tracer.OnTxStart(evm.GetVMContext(), types.NewTx(&types.LegacyTx{}), common.Address{}) tracer.OnEnter(0, byte(vm.CALL), common.Address{}, common.Address{}, []byte{}, 1000, big.NewInt(0)) tracer.OnExit(0, nil, 0, nil, false) ret, err := tracer.GetResult() diff --git a/eth/tracers/logger/logger_test.go b/eth/tracers/logger/logger_test.go index fb1a0154e3..e7799dde35 100644 --- a/eth/tracers/logger/logger_test.go +++ b/eth/tracers/logger/logger_test.go @@ -58,13 +58,13 @@ func (*dummyStatedb) SetState(_ common.Address, _ common.Hash, _ common.Hash) co func TestStoreCapture(t *testing.T) { var ( logger = NewStructLogger(nil) - env = vm.NewEVM(vm.BlockContext{}, vm.TxContext{}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Tracer: logger.Hooks()}) + evm = vm.NewEVM(vm.BlockContext{}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Tracer: logger.Hooks()}) contract = vm.NewContract(&dummyContractRef{}, &dummyContractRef{}, new(uint256.Int), 100000) ) contract.Code = []byte{byte(vm.PUSH1), 0x1, byte(vm.PUSH1), 0x0, byte(vm.SSTORE)} var index common.Hash - logger.OnTxStart(env.GetVMContext(), nil, common.Address{}) - _, err := env.Interpreter().Run(contract, []byte{}, false) + logger.OnTxStart(evm.GetVMContext(), nil, common.Address{}) + _, err := evm.Interpreter().Run(contract, []byte{}, false) if err != nil { t.Fatal(err) } diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index 31e14b9112..72cc1bde25 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -89,7 +89,8 @@ func BenchmarkTransactionTrace(b *testing.B) { //EnableMemory: false, //EnableReturnData: false, }) - evm := vm.NewEVM(context, txContext, state.StateDB, params.AllEthashProtocolChanges, vm.Config{Tracer: tracer.Hooks()}) + evm := vm.NewEVM(context, state.StateDB, params.AllEthashProtocolChanges, vm.Config{Tracer: tracer.Hooks()}) + evm.SetTxContext(txContext) msg, err := core.TransactionToMessage(tx, signer, context.BaseFee) if err != nil { b.Fatalf("failed to prepare transaction for tracing: %v", err) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index a508b0ca5b..9c08006f1e 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -864,10 +864,11 @@ func applyMessage(ctx context.Context, b Backend, args TransactionArgs, state *s if msg.BlobGasFeeCap != nil && msg.BlobGasFeeCap.BitLen() == 0 { blockContext.BlobBaseFee = new(big.Int) } - evm := b.GetEVM(ctx, msg, state, header, vmConfig, blockContext) + evm := b.GetEVM(ctx, state, header, vmConfig, blockContext) if precompiles != nil { evm.SetPrecompiles(precompiles) } + evm.SetTxContext(core.NewEVMTxContext(msg)) res, err := applyMessageWithEVM(ctx, evm, msg, timeout, gp) // If an internal state error occurred, let that have precedence. Otherwise, // a "trie root missing" type of error will masquerade as e.g. "insufficient gas" @@ -1331,7 +1332,7 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH // Apply the transaction with the access list tracer tracer := logger.NewAccessListTracer(accessList, args.from(), to, precompiles) config := vm.Config{Tracer: tracer.Hooks(), NoBaseFee: true} - vmenv := b.GetEVM(ctx, msg, statedb, header, &config, nil) + vmenv := b.GetEVM(ctx, statedb, header, &config, nil) // Lower the basefee to 0 to avoid breaking EVM // invariants (basefee < feecap). if msg.GasPrice.Sign() == 0 { @@ -1340,6 +1341,7 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH if msg.BlobGasFeeCap != nil && msg.BlobGasFeeCap.BitLen() == 0 { vmenv.Context.BlobBaseFee = new(big.Int) } + vmenv.SetTxContext(core.NewEVMTxContext(msg)) res, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.GasLimit)) if err != nil { return nil, 0, nil, fmt.Errorf("failed to apply transaction: %v err: %v", args.ToTransaction(types.LegacyTxType).Hash(), err) diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index f570c5dc4c..1f5f2dd1d5 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -570,16 +570,15 @@ func (b testBackend) GetTd(ctx context.Context, hash common.Hash) *big.Int { } return big.NewInt(1) } -func (b testBackend) GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config, blockContext *vm.BlockContext) *vm.EVM { +func (b testBackend) GetEVM(ctx context.Context, state *state.StateDB, header *types.Header, vmConfig *vm.Config, blockContext *vm.BlockContext) *vm.EVM { if vmConfig == nil { vmConfig = b.chain.GetVMConfig() } - txContext := core.NewEVMTxContext(msg) context := core.NewEVMBlockContext(header, b.chain, nil) if blockContext != nil { context = *blockContext } - return vm.NewEVM(context, txContext, state, b.chain.Config(), *vmConfig) + return vm.NewEVM(context, state, b.chain.Config(), *vmConfig) } func (b testBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { panic("implement me") diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index 82465ca7d7..3d8f7aa700 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -68,7 +68,7 @@ type Backend interface { StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) Pending() (*types.Block, types.Receipts, *state.StateDB) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) - GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config, blockCtx *vm.BlockContext) *vm.EVM + GetEVM(ctx context.Context, state *state.StateDB, header *types.Header, vmConfig *vm.Config, blockCtx *vm.BlockContext) *vm.EVM SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription diff --git a/internal/ethapi/simulate.go b/internal/ethapi/simulate.go index 81b4633d42..979de352fd 100644 --- a/internal/ethapi/simulate.go +++ b/internal/ethapi/simulate.go @@ -185,12 +185,12 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, NoBaseFee: !sim.validate, Tracer: tracer.Hooks(), } - evm = vm.NewEVM(blockContext, vm.TxContext{GasPrice: new(big.Int)}, sim.state, sim.chainConfig, *vmConfig) ) var tracingStateDB = vm.StateDB(sim.state) if hooks := tracer.Hooks(); hooks != nil { tracingStateDB = state.NewHookedState(sim.state, hooks) } + evm := vm.NewEVM(blockContext, tracingStateDB, sim.chainConfig, *vmConfig) // It is possible to override precompiles with EVM bytecode, or // move them to another address. if precompiles != nil { @@ -208,7 +208,7 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, tracer.reset(tx.Hash(), uint(i)) // EoA check is always skipped, even in validation mode. msg := call.ToMessage(header.BaseFee, !sim.validate, true) - evm.Reset(core.NewEVMTxContext(msg), tracingStateDB) + evm.SetTxContext(core.NewEVMTxContext(msg)) result, err := applyMessageWithEVM(ctx, evm, msg, timeout, sim.gp) if err != nil { txErr := txValidationError(err) diff --git a/internal/ethapi/transaction_args_test.go b/internal/ethapi/transaction_args_test.go index 5f59b491e1..7172fc883f 100644 --- a/internal/ethapi/transaction_args_test.go +++ b/internal/ethapi/transaction_args_test.go @@ -370,7 +370,7 @@ func (b *backendMock) GetLogs(ctx context.Context, blockHash common.Hash, number return nil, nil } func (b *backendMock) GetTd(ctx context.Context, hash common.Hash) *big.Int { return nil } -func (b *backendMock) GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config, blockCtx *vm.BlockContext) *vm.EVM { +func (b *backendMock) GetEVM(ctx context.Context, state *state.StateDB, header *types.Header, vmConfig *vm.Config, blockCtx *vm.BlockContext) *vm.EVM { return nil } func (b *backendMock) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { return nil } diff --git a/miner/worker.go b/miner/worker.go index db2fac3871..51ac5e2e36 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -51,6 +51,7 @@ type environment struct { tcount int // tx count in cycle gasPool *core.GasPool // available gas used to pack transactions coinbase common.Address + evm *vm.EVM header *types.Header txs []*types.Transaction @@ -126,14 +127,11 @@ func (miner *Miner) generateWork(params *generateParams, witness bool) *newPaylo return &newPayloadResult{err: err} } requests = append(requests, depositRequests) - // create EVM for system calls - blockContext := core.NewEVMBlockContext(work.header, miner.chain, &work.header.Coinbase) - vmenv := vm.NewEVM(blockContext, vm.TxContext{}, work.state, miner.chainConfig, vm.Config{}) // EIP-7002 withdrawals - withdrawalRequests := core.ProcessWithdrawalQueue(vmenv, work.state) + withdrawalRequests := core.ProcessWithdrawalQueue(work.evm, work.state) requests = append(requests, withdrawalRequests) // EIP-7251 consolidations - consolidationRequests := core.ProcessConsolidationQueue(vmenv, work.state) + consolidationRequests := core.ProcessConsolidationQueue(work.evm, work.state) requests = append(requests, consolidationRequests) } if requests != nil { @@ -233,14 +231,10 @@ func (miner *Miner) prepareWork(genParams *generateParams, witness bool) (*envir return nil, err } if header.ParentBeaconRoot != nil { - context := core.NewEVMBlockContext(header, miner.chain, nil) - vmenv := vm.NewEVM(context, vm.TxContext{}, env.state, miner.chainConfig, vm.Config{}) - core.ProcessBeaconBlockRoot(*header.ParentBeaconRoot, vmenv, env.state) + core.ProcessBeaconBlockRoot(*header.ParentBeaconRoot, env.evm, env.state) } if miner.chainConfig.IsPrague(header.Number, header.Time) { - context := core.NewEVMBlockContext(header, miner.chain, nil) - vmenv := vm.NewEVM(context, vm.TxContext{}, env.state, miner.chainConfig, vm.Config{}) - core.ProcessParentBlockHash(header.ParentHash, vmenv, env.state) + core.ProcessParentBlockHash(header.ParentHash, env.evm, env.state) } return env, nil } @@ -266,6 +260,7 @@ func (miner *Miner) makeEnv(parent *types.Header, header *types.Header, coinbase coinbase: coinbase, header: header, witness: state.Witness(), + evm: vm.NewEVM(core.NewEVMBlockContext(header, miner.chain, &coinbase), state, miner.chainConfig, vm.Config{}), }, nil } @@ -314,7 +309,7 @@ func (miner *Miner) applyTransaction(env *environment, tx *types.Transaction) (* snap = env.state.Snapshot() gp = env.gasPool.Gas() ) - receipt, err := core.ApplyTransaction(miner.chainConfig, miner.chain, &env.coinbase, env.gasPool, env.state, env.header, tx, &env.header.GasUsed, vm.Config{}) + receipt, err := core.ApplyTransaction(miner.chainConfig, env.evm, env.gasPool, env.state, env.header, tx, &env.header.GasUsed) if err != nil { env.state.RevertToSnapshot(snap) env.gasPool.SetGas(gp) diff --git a/tests/state_test.go b/tests/state_test.go index 76d5a601c7..7b82b05e58 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -302,7 +302,8 @@ func runBenchmark(b *testing.B, t *StateTest) { context := core.NewEVMBlockContext(block.Header(), nil, &t.json.Env.Coinbase) context.GetHash = vmTestBlockHash context.BaseFee = baseFee - evm := vm.NewEVM(context, txContext, state.StateDB, config, vmconfig) + evm := vm.NewEVM(context, state.StateDB, config, vmconfig) + evm.SetTxContext(txContext) // Create "contract" for sender to cache code analysis. sender := vm.NewContract(vm.AccountRef(msg.From), vm.AccountRef(msg.From), diff --git a/tests/state_test_util.go b/tests/state_test_util.go index af2cb63d94..446ffb40d5 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -293,7 +293,8 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh if config.IsCancun(new(big.Int), block.Time()) && t.json.Env.ExcessBlobGas != nil { context.BlobBaseFee = eip4844.CalcBlobFee(*t.json.Env.ExcessBlobGas) } - evm := vm.NewEVM(context, txContext, st.StateDB, config, vmconfig) + evm := vm.NewEVM(context, st.StateDB, config, vmconfig) + evm.SetTxContext(txContext) if tracer := vmconfig.Tracer; tracer != nil && tracer.OnTxStart != nil { tracer.OnTxStart(evm.GetVMContext(), nil, msg.From) From a25be32fa92cfc27b82930a41a10f538d8479e2e Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Fri, 22 Nov 2024 15:17:32 +0800 Subject: [PATCH 10/21] core, eth, internal, miner: remove unnecessary parameters (#30776) Follow-up to #30745 , this change removes some unnecessary parameters. --- cmd/evm/internal/t8ntool/execution.go | 8 ++-- core/chain_makers.go | 15 +++--- core/state_processor.go | 67 +++++++++++++-------------- core/verkle_witness_test.go | 2 +- eth/state_accessor.go | 4 +- eth/tracers/api.go | 18 +++---- internal/ethapi/simulate.go | 2 +- miner/worker.go | 10 ++-- 8 files changed, 61 insertions(+), 65 deletions(-) diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index a868fbf11c..c0623a69bf 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -203,14 +203,14 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, } evm := vm.NewEVM(vmContext, statedb, chainConfig, vmConfig) if beaconRoot := pre.Env.ParentBeaconBlockRoot; beaconRoot != nil { - core.ProcessBeaconBlockRoot(*beaconRoot, evm, statedb) + core.ProcessBeaconBlockRoot(*beaconRoot, evm) } if pre.Env.BlockHashes != nil && chainConfig.IsPrague(new(big.Int).SetUint64(pre.Env.Number), pre.Env.Timestamp) { var ( prevNumber = pre.Env.Number - 1 prevHash = pre.Env.BlockHashes[math.HexOrDecimal64(prevNumber)] ) - core.ProcessParentBlockHash(prevHash, evm, statedb) + core.ProcessParentBlockHash(prevHash, evm) } for i := 0; txIt.Next(); i++ { tx, err := txIt.Tx() @@ -378,9 +378,9 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, requests = append(requests, depositRequests) // EIP-7002 withdrawals - requests = append(requests, core.ProcessWithdrawalQueue(evm, statedb)) + requests = append(requests, core.ProcessWithdrawalQueue(evm)) // EIP-7251 consolidations - requests = append(requests, core.ProcessConsolidationQueue(evm, statedb)) + requests = append(requests, core.ProcessConsolidationQueue(evm)) } // Commit block diff --git a/core/chain_makers.go b/core/chain_makers.go index 07448abc3d..e679a9e557 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -98,11 +98,8 @@ func (b *BlockGen) Difficulty() *big.Int { // block. func (b *BlockGen) SetParentBeaconRoot(root common.Hash) { b.header.ParentBeaconRoot = &root - var ( - blockContext = NewEVMBlockContext(b.header, b.cm, &b.header.Coinbase) - evm = vm.NewEVM(blockContext, b.statedb, b.cm.config, vm.Config{}) - ) - ProcessBeaconBlockRoot(root, evm, b.statedb) + blockContext := NewEVMBlockContext(b.header, b.cm, &b.header.Coinbase) + ProcessBeaconBlockRoot(root, vm.NewEVM(blockContext, b.statedb, b.cm.config, vm.Config{})) } // addTx adds a transaction to the generated block. If no coinbase has @@ -121,7 +118,7 @@ func (b *BlockGen) addTx(bc *BlockChain, vmConfig vm.Config, tx *types.Transacti evm = vm.NewEVM(blockContext, b.statedb, b.cm.config, vmConfig) ) b.statedb.SetTxContext(tx.Hash(), len(b.txs)) - receipt, err := ApplyTransaction(b.cm.config, evm, b.gasPool, b.statedb, b.header, tx, &b.header.GasUsed) + receipt, err := ApplyTransaction(evm, b.gasPool, b.statedb, b.header, tx, &b.header.GasUsed) if err != nil { panic(err) } @@ -366,10 +363,10 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse blockContext := NewEVMBlockContext(b.header, cm, &b.header.Coinbase) evm := vm.NewEVM(blockContext, statedb, cm.config, vm.Config{}) // EIP-7002 withdrawals - withdrawalRequests := ProcessWithdrawalQueue(evm, statedb) + withdrawalRequests := ProcessWithdrawalQueue(evm) requests = append(requests, withdrawalRequests) // EIP-7251 consolidations - consolidationRequests := ProcessConsolidationQueue(evm, statedb) + consolidationRequests := ProcessConsolidationQueue(evm) requests = append(requests, consolidationRequests) } if requests != nil { @@ -471,7 +468,7 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine // EIP-2935 blockContext := NewEVMBlockContext(b.header, cm, &b.header.Coinbase) evm := vm.NewEVM(blockContext, statedb, cm.config, vm.Config{}) - ProcessParentBlockHash(b.header.ParentHash, evm, statedb) + ProcessParentBlockHash(b.header.ParentHash, evm) } // Execute any user modifications to the block. diff --git a/core/state_processor.go b/core/state_processor.go index 74985b69a1..1703377111 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -82,10 +82,10 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg evm := vm.NewEVM(context, tracingStateDB, p.config, cfg) if beaconRoot := block.BeaconRoot(); beaconRoot != nil { - ProcessBeaconBlockRoot(*beaconRoot, evm, tracingStateDB) + ProcessBeaconBlockRoot(*beaconRoot, evm) } if p.config.IsPrague(block.Number(), block.Time()) { - ProcessParentBlockHash(block.ParentHash(), evm, tracingStateDB) + ProcessParentBlockHash(block.ParentHash(), evm) } // Iterate over and process the individual transactions @@ -96,7 +96,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg } statedb.SetTxContext(tx.Hash(), i) - receipt, err := ApplyTransactionWithEVM(msg, p.config, gp, statedb, blockNumber, blockHash, tx, usedGas, evm) + receipt, err := ApplyTransactionWithEVM(msg, gp, statedb, blockNumber, blockHash, tx, usedGas, evm) if err != nil { return nil, fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err) } @@ -113,10 +113,10 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg } requests = append(requests, depositRequests) // EIP-7002 withdrawals - withdrawalRequests := ProcessWithdrawalQueue(evm, tracingStateDB) + withdrawalRequests := ProcessWithdrawalQueue(evm) requests = append(requests, withdrawalRequests) // EIP-7251 consolidations - consolidationRequests := ProcessConsolidationQueue(evm, tracingStateDB) + consolidationRequests := ProcessConsolidationQueue(evm) requests = append(requests, consolidationRequests) } @@ -134,7 +134,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg // ApplyTransactionWithEVM attempts to apply a transaction to the given state database // and uses the input parameters for its environment similar to ApplyTransaction. However, // this method takes an already created EVM instance as input. -func ApplyTransactionWithEVM(msg *Message, config *params.ChainConfig, gp *GasPool, statedb *state.StateDB, blockNumber *big.Int, blockHash common.Hash, tx *types.Transaction, usedGas *uint64, evm *vm.EVM) (receipt *types.Receipt, err error) { +func ApplyTransactionWithEVM(msg *Message, gp *GasPool, statedb *state.StateDB, blockNumber *big.Int, blockHash common.Hash, tx *types.Transaction, usedGas *uint64, evm *vm.EVM) (receipt *types.Receipt, err error) { if hooks := evm.Config.Tracer; hooks != nil { if hooks.OnTxStart != nil { hooks.OnTxStart(evm.GetVMContext(), tx, msg.From) @@ -156,10 +156,10 @@ func ApplyTransactionWithEVM(msg *Message, config *params.ChainConfig, gp *GasPo // Update the state with pending changes. var root []byte - if config.IsByzantium(blockNumber) { + if evm.ChainConfig().IsByzantium(blockNumber) { evm.StateDB.Finalise(true) } else { - root = statedb.IntermediateRoot(config.IsEIP158(blockNumber)).Bytes() + root = statedb.IntermediateRoot(evm.ChainConfig().IsEIP158(blockNumber)).Bytes() } *usedGas += result.UsedGas @@ -208,19 +208,19 @@ func MakeReceipt(evm *vm.EVM, result *ExecutionResult, statedb *state.StateDB, b // and uses the input parameters for its environment. It returns the receipt // for the transaction, gas used and an error if the transaction failed, // indicating the block was invalid. -func ApplyTransaction(config *params.ChainConfig, evm *vm.EVM, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, usedGas *uint64) (*types.Receipt, error) { - msg, err := TransactionToMessage(tx, types.MakeSigner(config, header.Number, header.Time), header.BaseFee) +func ApplyTransaction(evm *vm.EVM, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, usedGas *uint64) (*types.Receipt, error) { + msg, err := TransactionToMessage(tx, types.MakeSigner(evm.ChainConfig(), header.Number, header.Time), header.BaseFee) if err != nil { return nil, err } // Create a new context to be used in the EVM environment - return ApplyTransactionWithEVM(msg, config, gp, statedb, header.Number, header.Hash(), tx, usedGas, evm) + return ApplyTransactionWithEVM(msg, gp, statedb, header.Number, header.Hash(), tx, usedGas, evm) } // ProcessBeaconBlockRoot applies the EIP-4788 system call to the beacon block root // contract. This method is exported to be used in tests. -func ProcessBeaconBlockRoot(beaconRoot common.Hash, vmenv *vm.EVM, statedb vm.StateDB) { - if tracer := vmenv.Config.Tracer; tracer != nil { +func ProcessBeaconBlockRoot(beaconRoot common.Hash, evm *vm.EVM) { + if tracer := evm.Config.Tracer; tracer != nil { if tracer.OnSystemCallStart != nil { tracer.OnSystemCallStart() } @@ -237,16 +237,16 @@ func ProcessBeaconBlockRoot(beaconRoot common.Hash, vmenv *vm.EVM, statedb vm.St To: ¶ms.BeaconRootsAddress, Data: beaconRoot[:], } - vmenv.SetTxContext(NewEVMTxContext(msg)) - statedb.AddAddressToAccessList(params.BeaconRootsAddress) - _, _, _ = vmenv.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.U2560) - statedb.Finalise(true) + evm.SetTxContext(NewEVMTxContext(msg)) + evm.StateDB.AddAddressToAccessList(params.BeaconRootsAddress) + _, _, _ = evm.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.U2560) + evm.StateDB.Finalise(true) } // ProcessParentBlockHash stores the parent block hash in the history storage contract // as per EIP-2935. -func ProcessParentBlockHash(prevHash common.Hash, vmenv *vm.EVM, statedb vm.StateDB) { - if tracer := vmenv.Config.Tracer; tracer != nil { +func ProcessParentBlockHash(prevHash common.Hash, evm *vm.EVM) { + if tracer := evm.Config.Tracer; tracer != nil { if tracer.OnSystemCallStart != nil { tracer.OnSystemCallStart() } @@ -263,26 +263,26 @@ func ProcessParentBlockHash(prevHash common.Hash, vmenv *vm.EVM, statedb vm.Stat To: ¶ms.HistoryStorageAddress, Data: prevHash.Bytes(), } - vmenv.SetTxContext(NewEVMTxContext(msg)) - statedb.AddAddressToAccessList(params.HistoryStorageAddress) - _, _, _ = vmenv.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.U2560) - statedb.Finalise(true) + evm.SetTxContext(NewEVMTxContext(msg)) + evm.StateDB.AddAddressToAccessList(params.HistoryStorageAddress) + _, _, _ = evm.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.U2560) + evm.StateDB.Finalise(true) } // ProcessWithdrawalQueue calls the EIP-7002 withdrawal queue contract. // It returns the opaque request data returned by the contract. -func ProcessWithdrawalQueue(vmenv *vm.EVM, statedb vm.StateDB) []byte { - return processRequestsSystemCall(vmenv, statedb, 0x01, params.WithdrawalQueueAddress) +func ProcessWithdrawalQueue(evm *vm.EVM) []byte { + return processRequestsSystemCall(evm, 0x01, params.WithdrawalQueueAddress) } // ProcessConsolidationQueue calls the EIP-7251 consolidation queue contract. // It returns the opaque request data returned by the contract. -func ProcessConsolidationQueue(vmenv *vm.EVM, statedb vm.StateDB) []byte { - return processRequestsSystemCall(vmenv, statedb, 0x02, params.ConsolidationQueueAddress) +func ProcessConsolidationQueue(evm *vm.EVM) []byte { + return processRequestsSystemCall(evm, 0x02, params.ConsolidationQueueAddress) } -func processRequestsSystemCall(vmenv *vm.EVM, statedb vm.StateDB, requestType byte, addr common.Address) []byte { - if tracer := vmenv.Config.Tracer; tracer != nil { +func processRequestsSystemCall(evm *vm.EVM, requestType byte, addr common.Address) []byte { + if tracer := evm.Config.Tracer; tracer != nil { if tracer.OnSystemCallStart != nil { tracer.OnSystemCallStart() } @@ -290,7 +290,6 @@ func processRequestsSystemCall(vmenv *vm.EVM, statedb vm.StateDB, requestType by defer tracer.OnSystemCallEnd() } } - msg := &Message{ From: params.SystemAddress, GasLimit: 30_000_000, @@ -299,10 +298,10 @@ func processRequestsSystemCall(vmenv *vm.EVM, statedb vm.StateDB, requestType by GasTipCap: common.Big0, To: &addr, } - vmenv.SetTxContext(NewEVMTxContext(msg)) - statedb.AddAddressToAccessList(addr) - ret, _, _ := vmenv.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.U2560) - statedb.Finalise(true) + evm.SetTxContext(NewEVMTxContext(msg)) + evm.StateDB.AddAddressToAccessList(addr) + ret, _, _ := evm.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.U2560) + evm.StateDB.Finalise(true) // Create withdrawals requestsData with prefix 0x01 requestsData := make([]byte, len(ret)+1) diff --git a/core/verkle_witness_test.go b/core/verkle_witness_test.go index 80996eee33..72df5473bb 100644 --- a/core/verkle_witness_test.go +++ b/core/verkle_witness_test.go @@ -226,7 +226,7 @@ func TestProcessParentBlockHash(t *testing.T) { header := &types.Header{ParentHash: common.Hash{byte(i)}, Number: big.NewInt(int64(i)), Difficulty: new(big.Int)} vmContext := NewEVMBlockContext(header, nil, new(common.Address)) evm := vm.NewEVM(vmContext, statedb, params.MergedTestChainConfig, vm.Config{}) - ProcessParentBlockHash(header.ParentHash, evm, statedb) + ProcessParentBlockHash(header.ParentHash, evm) } // Read block hashes for block 0 .. num-1 for i := 0; i < num; i++ { diff --git a/eth/state_accessor.go b/eth/state_accessor.go index a919030249..43432cff31 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -238,11 +238,11 @@ func (eth *Ethereum) stateAtTransaction(ctx context.Context, block *types.Block, context := core.NewEVMBlockContext(block.Header(), eth.blockchain, nil) evm := vm.NewEVM(context, statedb, eth.blockchain.Config(), vm.Config{}) if beaconRoot := block.BeaconRoot(); beaconRoot != nil { - core.ProcessBeaconBlockRoot(*beaconRoot, evm, statedb) + core.ProcessBeaconBlockRoot(*beaconRoot, evm) } // If prague hardfork, insert parent block hash in the state as per EIP-2935. if eth.blockchain.Config().IsPrague(block.Number(), block.Time()) { - core.ProcessParentBlockHash(block.ParentHash(), evm, statedb) + core.ProcessParentBlockHash(block.ParentHash(), evm) } if txIndex == 0 && len(block.Transactions()) == 0 { return nil, vm.BlockContext{}, statedb, release, nil diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 671a86e78a..a2c11e0fe2 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -381,11 +381,11 @@ func (api *API) traceChain(start, end *types.Block, config *TraceConfig, closed context := core.NewEVMBlockContext(next.Header(), api.chainContext(ctx), nil) evm := vm.NewEVM(context, statedb, api.backend.ChainConfig(), vm.Config{}) if beaconRoot := next.BeaconRoot(); beaconRoot != nil { - core.ProcessBeaconBlockRoot(*beaconRoot, evm, statedb) + core.ProcessBeaconBlockRoot(*beaconRoot, evm) } // Insert parent hash in history contract. if api.backend.ChainConfig().IsPrague(next.Number(), next.Time()) { - core.ProcessParentBlockHash(next.ParentHash(), evm, statedb) + core.ProcessParentBlockHash(next.ParentHash(), evm) } // Clean out any pending release functions of trace state. Note this // step must be done after constructing tracing state, because the @@ -537,10 +537,10 @@ func (api *API) IntermediateRoots(ctx context.Context, hash common.Hash, config ) evm := vm.NewEVM(vmctx, statedb, chainConfig, vm.Config{}) if beaconRoot := block.BeaconRoot(); beaconRoot != nil { - core.ProcessBeaconBlockRoot(*beaconRoot, evm, statedb) + core.ProcessBeaconBlockRoot(*beaconRoot, evm) } if chainConfig.IsPrague(block.Number(), block.Time()) { - core.ProcessParentBlockHash(block.ParentHash(), evm, statedb) + core.ProcessParentBlockHash(block.ParentHash(), evm) } for i, tx := range block.Transactions() { if err := ctx.Err(); err != nil { @@ -605,10 +605,10 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac blockCtx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) evm := vm.NewEVM(blockCtx, statedb, api.backend.ChainConfig(), vm.Config{}) if beaconRoot := block.BeaconRoot(); beaconRoot != nil { - core.ProcessBeaconBlockRoot(*beaconRoot, evm, statedb) + core.ProcessBeaconBlockRoot(*beaconRoot, evm) } if api.backend.ChainConfig().IsPrague(block.Number(), block.Time()) { - core.ProcessParentBlockHash(block.ParentHash(), evm, statedb) + core.ProcessParentBlockHash(block.ParentHash(), evm) } // JS tracers have high overhead. In this case run a parallel @@ -784,10 +784,10 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block } evm := vm.NewEVM(vmctx, statedb, chainConfig, vm.Config{}) if beaconRoot := block.BeaconRoot(); beaconRoot != nil { - core.ProcessBeaconBlockRoot(*beaconRoot, evm, statedb) + core.ProcessBeaconBlockRoot(*beaconRoot, evm) } if chainConfig.IsPrague(block.Number(), block.Time()) { - core.ProcessParentBlockHash(block.ParentHash(), evm, statedb) + core.ProcessParentBlockHash(block.ParentHash(), evm) } for i, tx := range block.Transactions() { // Prepare the transaction for un-traced execution @@ -1037,7 +1037,7 @@ func (api *API) traceTx(ctx context.Context, tx *types.Transaction, message *cor // Call Prepare to clear out the statedb access list statedb.SetTxContext(txctx.TxHash, txctx.TxIndex) - _, err = core.ApplyTransactionWithEVM(message, api.backend.ChainConfig(), new(core.GasPool).AddGas(message.GasLimit), statedb, vmctx.BlockNumber, txctx.BlockHash, tx, &usedGas, evm) + _, err = core.ApplyTransactionWithEVM(message, new(core.GasPool).AddGas(message.GasLimit), statedb, vmctx.BlockNumber, txctx.BlockHash, tx, &usedGas, evm) if err != nil { return nil, fmt.Errorf("tracing failed: %w", err) } diff --git a/internal/ethapi/simulate.go b/internal/ethapi/simulate.go index 979de352fd..0b58a27096 100644 --- a/internal/ethapi/simulate.go +++ b/internal/ethapi/simulate.go @@ -265,7 +265,7 @@ func repairLogs(calls []simCallResult, hash common.Hash) { } } -func (sim *simulator) sanitizeCall(call *TransactionArgs, state *state.StateDB, header *types.Header, blockContext vm.BlockContext, gasUsed *uint64) error { +func (sim *simulator) sanitizeCall(call *TransactionArgs, state vm.StateDB, header *types.Header, blockContext vm.BlockContext, gasUsed *uint64) error { if call.Nonce == nil { nonce := state.GetNonce(call.from()) call.Nonce = (*hexutil.Uint64)(&nonce) diff --git a/miner/worker.go b/miner/worker.go index 51ac5e2e36..aeb6cfcdc2 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -128,10 +128,10 @@ func (miner *Miner) generateWork(params *generateParams, witness bool) *newPaylo } requests = append(requests, depositRequests) // EIP-7002 withdrawals - withdrawalRequests := core.ProcessWithdrawalQueue(work.evm, work.state) + withdrawalRequests := core.ProcessWithdrawalQueue(work.evm) requests = append(requests, withdrawalRequests) // EIP-7251 consolidations - consolidationRequests := core.ProcessConsolidationQueue(work.evm, work.state) + consolidationRequests := core.ProcessConsolidationQueue(work.evm) requests = append(requests, consolidationRequests) } if requests != nil { @@ -231,10 +231,10 @@ func (miner *Miner) prepareWork(genParams *generateParams, witness bool) (*envir return nil, err } if header.ParentBeaconRoot != nil { - core.ProcessBeaconBlockRoot(*header.ParentBeaconRoot, env.evm, env.state) + core.ProcessBeaconBlockRoot(*header.ParentBeaconRoot, env.evm) } if miner.chainConfig.IsPrague(header.Number, header.Time) { - core.ProcessParentBlockHash(header.ParentHash, env.evm, env.state) + core.ProcessParentBlockHash(header.ParentHash, env.evm) } return env, nil } @@ -309,7 +309,7 @@ func (miner *Miner) applyTransaction(env *environment, tx *types.Transaction) (* snap = env.state.Snapshot() gp = env.gasPool.Gas() ) - receipt, err := core.ApplyTransaction(miner.chainConfig, env.evm, env.gasPool, env.state, env.header, tx, &env.header.GasUsed) + receipt, err := core.ApplyTransaction(env.evm, env.gasPool, env.state, env.header, tx, &env.header.GasUsed) if err != nil { env.state.RevertToSnapshot(snap) env.gasPool.SetGas(gp) From 2cd25fdd23b695cd8caf3ad0b0701d68bf9d99a3 Mon Sep 17 00:00:00 2001 From: "Hyunsoo Shin (Lake)" Date: Fri, 22 Nov 2024 16:21:20 +0900 Subject: [PATCH 11/21] internal/ethapi: remove double map-clone (#30788) `ActivePrecompiledContracts()` clones the precompiled contract map, thus its callsite does not need to clone it --- internal/ethapi/api.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 9c08006f1e..ded4db50aa 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -21,7 +21,6 @@ import ( "encoding/hex" "errors" "fmt" - "maps" gomath "math" "math/big" "strings" @@ -254,7 +253,7 @@ func (api *TxPoolAPI) Inspect() map[string]map[string]map[string]string { pending, queue := api.b.TxPoolContent() // Define a formatter to flatten a transaction into a string - var format = func(tx *types.Transaction) string { + format := func(tx *types.Transaction) string { if to := tx.To(); to != nil { return fmt.Sprintf("%s: %v wei + %v gas × %v wei", tx.To().Hex(), tx.Value(), tx.Gas(), tx.GasPrice()) } @@ -825,7 +824,7 @@ func doCall(ctx context.Context, b Backend, args TransactionArgs, state *state.S blockOverrides.Apply(&blockCtx) } rules := b.ChainConfig().Rules(blockCtx.BlockNumber, blockCtx.Random != nil, blockCtx.Time) - precompiles := maps.Clone(vm.ActivePrecompiledContracts(rules)) + precompiles := vm.ActivePrecompiledContracts(rules) if err := overrides.Apply(state, precompiles); err != nil { return nil, err } @@ -1762,11 +1761,11 @@ func (api *TransactionAPI) Resend(ctx context.Context, sendArgs TransactionArgs, matchTx := sendArgs.ToTransaction(types.LegacyTxType) // Before replacing the old transaction, ensure the _new_ transaction fee is reasonable. - var price = matchTx.GasPrice() + price := matchTx.GasPrice() if gasPrice != nil { price = gasPrice.ToInt() } - var gas = matchTx.Gas() + gas := matchTx.Gas() if gasLimit != nil { gas = uint64(*gasLimit) } From 16f2f7155fc92b94857d6d43e3758d3f71552618 Mon Sep 17 00:00:00 2001 From: wangjingcun Date: Fri, 22 Nov 2024 16:02:45 +0800 Subject: [PATCH 12/21] all: typos in comments (#30779) fixes some typos --- build/ci.go | 2 +- core/verkle_witness_test.go | 2 +- core/vm/eof.go | 2 +- trie/utils/verkle.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build/ci.go b/build/ci.go index 754d88a86a..e3115af46d 100644 --- a/build/ci.go +++ b/build/ci.go @@ -256,7 +256,7 @@ func buildFlags(env build.Environment, staticLinking bool, buildTags []string) ( // See https://sourceware.org/binutils/docs-2.23.1/ld/Options.html#Options // regarding the options --build-id=none and --strip-all. It is needed for // reproducible builds; removing references to temporary files in C-land, and - // making build-id reproducably absent. + // making build-id reproducibly absent. extld := []string{"-Wl,-z,stack-size=0x800000,--build-id=none,--strip-all"} if staticLinking { extld = append(extld, "-static") diff --git a/core/verkle_witness_test.go b/core/verkle_witness_test.go index 72df5473bb..45b317d3c0 100644 --- a/core/verkle_witness_test.go +++ b/core/verkle_witness_test.go @@ -1033,7 +1033,7 @@ func TestProcessVerkleSelfDestructInSameTxWithSelfBeneficiaryAndPrefundedAccount ) // Prefund the account, at an address that the contract will be deployed at, // before it selfdestrucs. We can therefore check that the account itseld is - // NOT destroyed, which is what the currrent version of the spec requires. + // NOT destroyed, which is what the current version of the spec requires. // TODO(gballet) revisit after the spec has been modified. gspec.Alloc[contract] = types.Account{ Balance: big.NewInt(100), diff --git a/core/vm/eof.go b/core/vm/eof.go index fcced6ae75..a5406283d5 100644 --- a/core/vm/eof.go +++ b/core/vm/eof.go @@ -96,7 +96,7 @@ func (meta *functionMetadata) checkInputs(stackMin int) error { } // checkStackMax checks the if current maximum stack combined with the -// functin max stack will result in a stack overflow, and if so returns an error. +// function max stack will result in a stack overflow, and if so returns an error. func (meta *functionMetadata) checkStackMax(stackMax int) error { newMaxStack := stackMax + int(meta.maxStackHeight) - int(meta.inputs) if newMaxStack > int(params.StackLimit) { diff --git a/trie/utils/verkle.go b/trie/utils/verkle.go index 4294f5e4be..b785e512b1 100644 --- a/trie/utils/verkle.go +++ b/trie/utils/verkle.go @@ -217,7 +217,7 @@ func StorageIndex(storageKey []byte) (*uint256.Int, byte) { // The first MAIN_STORAGE_OFFSET group will see its // first 64 slots unreachable. This is either a typo in the // spec or intended to conserve the 256-u256 - // aligment. If we decide to ever access these 64 + // alignment. If we decide to ever access these 64 // slots, uncomment this. // // Get the new offset since we now know that we are above 64. // pos.Sub(&pos, codeStorageDelta) From 6eeff3ee7dd87125a6791a991977b9664d8924ba Mon Sep 17 00:00:00 2001 From: j2gg0s Date: Fri, 22 Nov 2024 16:16:42 +0800 Subject: [PATCH 13/21] trie: replace custom logic with bytes.HasPrefix (#30771) in `trie` - Replace custom logic with `bytes.HasPrefix` - Remove unnecessary code in `GetNode` --- trie/proof.go | 8 ++++---- trie/trie.go | 7 ++----- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/trie/proof.go b/trie/proof.go index a39d6b4ea3..40836fba33 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -48,7 +48,7 @@ func (t *Trie) Prove(key []byte, proofDb ethdb.KeyValueWriter) error { for len(key) > 0 && tn != nil { switch n := tn.(type) { case *shortNode: - if len(key) < len(n.Key) || !bytes.Equal(n.Key, key[:len(n.Key)]) { + if !bytes.HasPrefix(key, n.Key) { // The trie doesn't contain the key. tn = nil } else { @@ -371,7 +371,7 @@ func unset(parent node, child node, key []byte, pos int, removeLeft bool) error } return unset(cld, cld.Children[key[pos]], key, pos+1, removeLeft) case *shortNode: - if len(key[pos:]) < len(cld.Key) || !bytes.Equal(cld.Key, key[pos:pos+len(cld.Key)]) { + if !bytes.HasPrefix(key[pos:], cld.Key) { // Find the fork point, it's a non-existent branch. if removeLeft { if bytes.Compare(cld.Key, key[pos:]) < 0 { @@ -434,7 +434,7 @@ func hasRightElement(node node, key []byte) bool { } node, pos = rn.Children[key[pos]], pos+1 case *shortNode: - if len(key)-pos < len(rn.Key) || !bytes.Equal(rn.Key, key[pos:pos+len(rn.Key)]) { + if !bytes.HasPrefix(key[pos:], rn.Key) { return bytes.Compare(rn.Key, key[pos:]) > 0 } node, pos = rn.Val, pos+len(rn.Key) @@ -589,7 +589,7 @@ func get(tn node, key []byte, skipResolved bool) ([]byte, node) { for { switch n := tn.(type) { case *shortNode: - if len(key) < len(n.Key) || !bytes.Equal(n.Key, key[:len(n.Key)]) { + if !bytes.HasPrefix(key, n.Key) { return nil, nil } tn = n.Val diff --git a/trie/trie.go b/trie/trie.go index e3f3f39248..ae2a7b21a2 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -163,7 +163,7 @@ func (t *Trie) get(origNode node, key []byte, pos int) (value []byte, newnode no case valueNode: return n, n, false, nil case *shortNode: - if len(key)-pos < len(n.Key) || !bytes.Equal(n.Key, key[pos:pos+len(n.Key)]) { + if !bytes.HasPrefix(key[pos:], n.Key) { // key not found in trie return nil, n, false, nil } @@ -219,9 +219,6 @@ func (t *Trie) GetNode(path []byte) ([]byte, int, error) { if resolved > 0 { t.root = newroot } - if item == nil { - return nil, resolved, nil - } return item, resolved, nil } @@ -254,7 +251,7 @@ func (t *Trie) getNode(origNode node, path []byte, pos int) (item []byte, newnod return nil, nil, 0, nil case *shortNode: - if len(path)-pos < len(n.Key) || !bytes.Equal(n.Key, path[pos:pos+len(n.Key)]) { + if !bytes.HasPrefix(path[pos:], n.Key) { // Path branches off from short node return nil, n, 0, nil } From 6485d5e3ffc08ce9ad071eee28a08110ae7f93fd Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Fri, 22 Nov 2024 16:55:43 +0800 Subject: [PATCH 14/21] core, triedb: remove destruct flag in state snapshot (#30752) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This pull request removes the destruct flag from the state snapshot to simplify the code. Previously, this flag indicated that an account was removed during a state transition, making all associated storage slots inaccessible. Because storage deletion can involve a large number of slots, the actual deletion is deferred until the end of the process, where it is handled in batches. With the deprecation of self-destruct in the Cancun fork, storage deletions are no longer expected. Historically, the largest storage deletion event in Ethereum was around 15 megabytes—manageable in memory. In this pull request, the single destruct flag is replaced by a set of deletion markers for individual storage slots. Each deleted storage slot will now appear in the Storage set with a nil value. This change will simplify a lot logics, such as storage accessing, storage flushing, storage iteration and so on. --- core/state/snapshot/conversion.go | 10 -- core/state/snapshot/difflayer.go | 117 +++++------------------ core/state/snapshot/difflayer_test.go | 123 +++++++++--------------- core/state/snapshot/disklayer.go | 4 +- core/state/snapshot/disklayer_test.go | 86 ++++++++++------- core/state/snapshot/generate_test.go | 2 +- core/state/snapshot/iterator.go | 17 ++-- core/state/snapshot/iterator_binary.go | 33 +------ core/state/snapshot/iterator_fast.go | 10 +- core/state/snapshot/iterator_test.go | 126 ++++++++++++------------- core/state/snapshot/journal.go | 37 +++----- core/state/snapshot/snapshot.go | 61 +++++------- core/state/snapshot/snapshot_test.go | 30 +++--- core/state/snapshot/utils.go | 12 +-- core/state/statedb.go | 69 ++++++++------ core/state/statedb_fuzz_test.go | 105 ++++++++++++++------- core/state/statedb_test.go | 4 +- core/state/stateupdate.go | 50 ++++++---- triedb/states.go | 2 - 19 files changed, 384 insertions(+), 514 deletions(-) diff --git a/core/state/snapshot/conversion.go b/core/state/snapshot/conversion.go index 8a0fd1989a..4b0774f2ae 100644 --- a/core/state/snapshot/conversion.go +++ b/core/state/snapshot/conversion.go @@ -50,16 +50,6 @@ type ( leafCallbackFn func(db ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) ) -// GenerateAccountTrieRoot takes an account iterator and reproduces the root hash. -func GenerateAccountTrieRoot(it AccountIterator) (common.Hash, error) { - return generateTrieRoot(nil, "", it, common.Hash{}, stackTrieGenerate, nil, newGenerateStats(), true) -} - -// GenerateStorageTrieRoot takes a storage iterator and reproduces the root hash. -func GenerateStorageTrieRoot(account common.Hash, it StorageIterator) (common.Hash, error) { - return generateTrieRoot(nil, "", it, account, stackTrieGenerate, nil, newGenerateStats(), true) -} - // GenerateTrie takes the whole snapshot tree as the input, traverses all the // accounts as well as the corresponding storages and regenerate the whole state // (account trie + all storage tries). diff --git a/core/state/snapshot/difflayer.go b/core/state/snapshot/difflayer.go index 779c1ea98c..dce4f79a11 100644 --- a/core/state/snapshot/difflayer.go +++ b/core/state/snapshot/difflayer.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" bloomfilter "github.com/holiman/bloomfilter/v2" + "golang.org/x/exp/maps" ) var ( @@ -73,23 +74,14 @@ var ( // bloom key for an account/slot. This is randomized at init(), so that the // global population of nodes do not all display the exact same behaviour with // regards to bloom content - bloomDestructHasherOffset = 0 - bloomAccountHasherOffset = 0 - bloomStorageHasherOffset = 0 + bloomAccountHasherOffset = 0 + bloomStorageHasherOffset = 0 ) func init() { // Init the bloom offsets in the range [0:24] (requires 8 bytes) - bloomDestructHasherOffset = rand.Intn(25) bloomAccountHasherOffset = rand.Intn(25) bloomStorageHasherOffset = rand.Intn(25) - - // The destruct and account blooms must be different, as the storage slots - // will check for destruction too for every bloom miss. It should not collide - // with modified accounts. - for bloomAccountHasherOffset == bloomDestructHasherOffset { - bloomAccountHasherOffset = rand.Intn(25) - } } // diffLayer represents a collection of modifications made to a state snapshot @@ -106,29 +98,16 @@ type diffLayer struct { root common.Hash // Root hash to which this snapshot diff belongs to stale atomic.Bool // Signals that the layer became stale (state progressed) - // destructSet is a very special helper marker. If an account is marked as - // deleted, then it's recorded in this set. However it's allowed that an account - // is included here but still available in other sets(e.g. storageData). The - // reason is the diff layer includes all the changes in a *block*. It can - // happen that in the tx_1, account A is self-destructed while in the tx_2 - // it's recreated. But we still need this marker to indicate the "old" A is - // deleted, all data in other set belongs to the "new" A. - destructSet map[common.Hash]struct{} // Keyed markers for deleted (and potentially) recreated accounts - accountList []common.Hash // List of account for iteration. If it exists, it's sorted, otherwise it's nil accountData map[common.Hash][]byte // Keyed accounts for direct retrieval (nil means deleted) - storageList map[common.Hash][]common.Hash // List of storage slots for iterated retrievals, one per account. Any existing lists are sorted if non-nil storageData map[common.Hash]map[common.Hash][]byte // Keyed storage slots for direct retrieval. one per account (nil means deleted) + accountList []common.Hash // List of account for iteration. If it exists, it's sorted, otherwise it's nil + storageList map[common.Hash][]common.Hash // List of storage slots for iterated retrievals, one per account. Any existing lists are sorted if non-nil diffed *bloomfilter.Filter // Bloom filter tracking all the diffed items up to the disk layer lock sync.RWMutex } -// destructBloomHash is used to convert a destruct event into a 64 bit mini hash. -func destructBloomHash(h common.Hash) uint64 { - return binary.BigEndian.Uint64(h[bloomDestructHasherOffset : bloomDestructHasherOffset+8]) -} - // accountBloomHash is used to convert an account hash into a 64 bit mini hash. func accountBloomHash(h common.Hash) uint64 { return binary.BigEndian.Uint64(h[bloomAccountHasherOffset : bloomAccountHasherOffset+8]) @@ -142,12 +121,11 @@ func storageBloomHash(h0, h1 common.Hash) uint64 { // newDiffLayer creates a new diff on top of an existing snapshot, whether that's a low // level persistent database or a hierarchical diff already. -func newDiffLayer(parent snapshot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer { +func newDiffLayer(parent snapshot, root common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer { // Create the new layer with some pre-allocated data segments dl := &diffLayer{ parent: parent, root: root, - destructSet: destructs, accountData: accounts, storageData: storage, storageList: make(map[common.Hash][]common.Hash), @@ -161,10 +139,7 @@ func newDiffLayer(parent snapshot, root common.Hash, destructs map[common.Hash]s panic("unknown parent type") } // Sanity check that accounts or storage slots are never nil - for accountHash, blob := range accounts { - if blob == nil { - panic(fmt.Sprintf("account %#x nil", accountHash)) - } + for _, blob := range accounts { // Determine memory size and track the dirty writes dl.memory += uint64(common.HashLength + len(blob)) snapshotDirtyAccountWriteMeter.Mark(int64(len(blob))) @@ -179,7 +154,6 @@ func newDiffLayer(parent snapshot, root common.Hash, destructs map[common.Hash]s snapshotDirtyStorageWriteMeter.Mark(int64(len(data))) } } - dl.memory += uint64(len(destructs) * common.HashLength) return dl } @@ -204,10 +178,6 @@ func (dl *diffLayer) rebloom(origin *diskLayer) { } else { dl.diffed, _ = bloomfilter.New(uint64(bloomSize), uint64(bloomFuncs)) } - // Iterate over all the accounts and storage slots and index them - for hash := range dl.destructSet { - dl.diffed.AddHash(destructBloomHash(hash)) - } for hash := range dl.accountData { dl.diffed.AddHash(accountBloomHash(hash)) } @@ -274,11 +244,8 @@ func (dl *diffLayer) AccountRLP(hash common.Hash) ([]byte, error) { } // Check the bloom filter first whether there's even a point in reaching into // all the maps in all the layers below - hit := dl.diffed.ContainsHash(accountBloomHash(hash)) - if !hit { - hit = dl.diffed.ContainsHash(destructBloomHash(hash)) - } var origin *diskLayer + hit := dl.diffed.ContainsHash(accountBloomHash(hash)) if !hit { origin = dl.origin // extract origin while holding the lock } @@ -310,18 +277,14 @@ func (dl *diffLayer) accountRLP(hash common.Hash, depth int) ([]byte, error) { if data, ok := dl.accountData[hash]; ok { snapshotDirtyAccountHitMeter.Mark(1) snapshotDirtyAccountHitDepthHist.Update(int64(depth)) - snapshotDirtyAccountReadMeter.Mark(int64(len(data))) + if n := len(data); n > 0 { + snapshotDirtyAccountReadMeter.Mark(int64(n)) + } else { + snapshotDirtyAccountInexMeter.Mark(1) + } snapshotBloomAccountTrueHitMeter.Mark(1) return data, nil } - // If the account is known locally, but deleted, return it - if _, ok := dl.destructSet[hash]; ok { - snapshotDirtyAccountHitMeter.Mark(1) - snapshotDirtyAccountHitDepthHist.Update(int64(depth)) - snapshotDirtyAccountInexMeter.Mark(1) - snapshotBloomAccountTrueHitMeter.Mark(1) - return nil, nil - } // Account unknown to this diff, resolve from parent if diff, ok := dl.parent.(*diffLayer); ok { return diff.accountRLP(hash, depth+1) @@ -345,11 +308,8 @@ func (dl *diffLayer) Storage(accountHash, storageHash common.Hash) ([]byte, erro dl.lock.RUnlock() return nil, ErrSnapshotStale } - hit := dl.diffed.ContainsHash(storageBloomHash(accountHash, storageHash)) - if !hit { - hit = dl.diffed.ContainsHash(destructBloomHash(accountHash)) - } var origin *diskLayer + hit := dl.diffed.ContainsHash(storageBloomHash(accountHash, storageHash)) if !hit { origin = dl.origin // extract origin while holding the lock } @@ -391,14 +351,6 @@ func (dl *diffLayer) storage(accountHash, storageHash common.Hash, depth int) ([ return data, nil } } - // If the account is known locally, but deleted, return an empty slot - if _, ok := dl.destructSet[accountHash]; ok { - snapshotDirtyStorageHitMeter.Mark(1) - snapshotDirtyStorageHitDepthHist.Update(int64(depth)) - snapshotDirtyStorageInexMeter.Mark(1) - snapshotBloomStorageTrueHitMeter.Mark(1) - return nil, nil - } // Storage slot unknown to this diff, resolve from parent if diff, ok := dl.parent.(*diffLayer); ok { return diff.storage(accountHash, storageHash, depth+1) @@ -410,8 +362,8 @@ func (dl *diffLayer) storage(accountHash, storageHash common.Hash, depth int) ([ // Update creates a new layer on top of the existing snapshot diff tree with // the specified data items. -func (dl *diffLayer) Update(blockRoot common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer { - return newDiffLayer(dl, blockRoot, destructs, accounts, storage) +func (dl *diffLayer) Update(blockRoot common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer { + return newDiffLayer(dl, blockRoot, accounts, storage) } // flatten pushes all data from this point downwards, flattening everything into @@ -436,12 +388,6 @@ func (dl *diffLayer) flatten() snapshot { if parent.stale.Swap(true) { panic("parent diff layer is stale") // we've flattened into the same parent from two children, boo } - // Overwrite all the updated accounts blindly, merge the sorted list - for hash := range dl.destructSet { - parent.destructSet[hash] = struct{}{} - delete(parent.accountData, hash) - delete(parent.storageData, hash) - } for hash, data := range dl.accountData { parent.accountData[hash] = data } @@ -453,17 +399,13 @@ func (dl *diffLayer) flatten() snapshot { continue } // Storage exists in both parent and child, merge the slots - comboData := parent.storageData[accountHash] - for storageHash, data := range storage { - comboData[storageHash] = data - } + maps.Copy(parent.storageData[accountHash], storage) } // Return the combo parent return &diffLayer{ parent: parent.parent, origin: parent.origin, root: dl.root, - destructSet: parent.destructSet, accountData: parent.accountData, storageData: parent.storageData, storageList: make(map[common.Hash][]common.Hash), @@ -489,15 +431,7 @@ func (dl *diffLayer) AccountList() []common.Hash { dl.lock.Lock() defer dl.lock.Unlock() - dl.accountList = make([]common.Hash, 0, len(dl.destructSet)+len(dl.accountData)) - for hash := range dl.accountData { - dl.accountList = append(dl.accountList, hash) - } - for hash := range dl.destructSet { - if _, ok := dl.accountData[hash]; !ok { - dl.accountList = append(dl.accountList, hash) - } - } + dl.accountList = maps.Keys(dl.accountData) slices.SortFunc(dl.accountList, common.Hash.Cmp) dl.memory += uint64(len(dl.accountList) * common.HashLength) return dl.accountList @@ -512,18 +446,17 @@ func (dl *diffLayer) AccountList() []common.Hash { // not empty but the flag is true. // // Note, the returned slice is not a copy, so do not modify it. -func (dl *diffLayer) StorageList(accountHash common.Hash) ([]common.Hash, bool) { +func (dl *diffLayer) StorageList(accountHash common.Hash) []common.Hash { dl.lock.RLock() - _, destructed := dl.destructSet[accountHash] if _, ok := dl.storageData[accountHash]; !ok { // Account not tracked by this layer dl.lock.RUnlock() - return nil, destructed + return nil } // If an old list already exists, return it if list, exist := dl.storageList[accountHash]; exist { dl.lock.RUnlock() - return list, destructed // the cached list can't be nil + return list // the cached list can't be nil } dl.lock.RUnlock() @@ -531,13 +464,9 @@ func (dl *diffLayer) StorageList(accountHash common.Hash) ([]common.Hash, bool) dl.lock.Lock() defer dl.lock.Unlock() - storageMap := dl.storageData[accountHash] - storageList := make([]common.Hash, 0, len(storageMap)) - for k := range storageMap { - storageList = append(storageList, k) - } + storageList := maps.Keys(dl.storageData[accountHash]) slices.SortFunc(storageList, common.Hash.Cmp) dl.storageList[accountHash] = storageList dl.memory += uint64(len(dl.storageList)*common.HashLength + common.HashLength) - return storageList, destructed + return storageList } diff --git a/core/state/snapshot/difflayer_test.go b/core/state/snapshot/difflayer_test.go index 674a031b16..b212098e75 100644 --- a/core/state/snapshot/difflayer_test.go +++ b/core/state/snapshot/difflayer_test.go @@ -28,14 +28,6 @@ import ( "github.com/ethereum/go-ethereum/ethdb/memorydb" ) -func copyDestructs(destructs map[common.Hash]struct{}) map[common.Hash]struct{} { - copy := make(map[common.Hash]struct{}) - for hash := range destructs { - copy[hash] = struct{}{} - } - return copy -} - func copyAccounts(accounts map[common.Hash][]byte) map[common.Hash][]byte { copy := make(map[common.Hash][]byte) for hash, blob := range accounts { @@ -58,9 +50,8 @@ func copyStorage(storage map[common.Hash]map[common.Hash][]byte) map[common.Hash // TestMergeBasics tests some simple merges func TestMergeBasics(t *testing.T) { var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) ) // Fill up a parent for i := 0; i < 100; i++ { @@ -69,7 +60,7 @@ func TestMergeBasics(t *testing.T) { accounts[h] = data if rand.Intn(4) == 0 { - destructs[h] = struct{}{} + accounts[h] = nil } if rand.Intn(2) == 0 { accStorage := make(map[common.Hash][]byte) @@ -80,11 +71,12 @@ func TestMergeBasics(t *testing.T) { } } // Add some (identical) layers on top - parent := newDiffLayer(emptyLayer(), common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) - child := newDiffLayer(parent, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) - child = newDiffLayer(child, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) - child = newDiffLayer(child, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) - child = newDiffLayer(child, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) + parent := newDiffLayer(emptyLayer(), common.Hash{}, copyAccounts(accounts), copyStorage(storage)) + child := newDiffLayer(parent, common.Hash{}, copyAccounts(accounts), copyStorage(storage)) + child = newDiffLayer(child, common.Hash{}, copyAccounts(accounts), copyStorage(storage)) + child = newDiffLayer(child, common.Hash{}, copyAccounts(accounts), copyStorage(storage)) + child = newDiffLayer(child, common.Hash{}, copyAccounts(accounts), copyStorage(storage)) + // And flatten merged := (child.flatten()).(*diffLayer) @@ -99,18 +91,13 @@ func TestMergeBasics(t *testing.T) { t.Errorf("accountList [2] wrong: have %v, want %v", have, want) } } - { // Check account drops - if have, want := len(merged.destructSet), len(destructs); have != want { - t.Errorf("accountDrop wrong: have %v, want %v", have, want) - } - } { // Check storage lists i := 0 for aHash, sMap := range storage { if have, want := len(merged.storageList), i; have != want { t.Errorf("[1] storageList wrong: have %v, want %v", have, want) } - list, _ := merged.StorageList(aHash) + list := merged.StorageList(aHash) if have, want := len(list), len(sMap); have != want { t.Errorf("[2] StorageList() wrong: have %v, want %v", have, want) } @@ -124,41 +111,32 @@ func TestMergeBasics(t *testing.T) { // TestMergeDelete tests some deletion func TestMergeDelete(t *testing.T) { - var ( - storage = make(map[common.Hash]map[common.Hash][]byte) - ) + storage := make(map[common.Hash]map[common.Hash][]byte) + // Fill up a parent h1 := common.HexToHash("0x01") h2 := common.HexToHash("0x02") - flipDrops := func() map[common.Hash]struct{} { - return map[common.Hash]struct{}{ - h2: {}, - } - } - flipAccs := func() map[common.Hash][]byte { + flip := func() map[common.Hash][]byte { return map[common.Hash][]byte{ h1: randomAccount(), + h2: nil, } } - flopDrops := func() map[common.Hash]struct{} { - return map[common.Hash]struct{}{ - h1: {}, - } - } - flopAccs := func() map[common.Hash][]byte { + flop := func() map[common.Hash][]byte { return map[common.Hash][]byte{ + h1: nil, h2: randomAccount(), } } // Add some flipAccs-flopping layers on top - parent := newDiffLayer(emptyLayer(), common.Hash{}, flipDrops(), flipAccs(), storage) - child := parent.Update(common.Hash{}, flopDrops(), flopAccs(), storage) - child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage) - child = child.Update(common.Hash{}, flopDrops(), flopAccs(), storage) - child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage) - child = child.Update(common.Hash{}, flopDrops(), flopAccs(), storage) - child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage) + parent := newDiffLayer(emptyLayer(), common.Hash{}, flip(), storage) + child := parent.Update(common.Hash{}, flop(), storage) + child = child.Update(common.Hash{}, flip(), storage) + child = child.Update(common.Hash{}, flop(), storage) + child = child.Update(common.Hash{}, flip(), storage) + child = child.Update(common.Hash{}, flop(), storage) + child = child.Update(common.Hash{}, flip(), storage) if data, _ := child.Account(h1); data == nil { t.Errorf("last diff layer: expected %x account to be non-nil", h1) @@ -166,12 +144,7 @@ func TestMergeDelete(t *testing.T) { if data, _ := child.Account(h2); data != nil { t.Errorf("last diff layer: expected %x account to be nil", h2) } - if _, ok := child.destructSet[h1]; ok { - t.Errorf("last diff layer: expected %x drop to be missing", h1) - } - if _, ok := child.destructSet[h2]; !ok { - t.Errorf("last diff layer: expected %x drop to be present", h1) - } + // And flatten merged := (child.flatten()).(*diffLayer) @@ -181,12 +154,6 @@ func TestMergeDelete(t *testing.T) { if data, _ := merged.Account(h2); data != nil { t.Errorf("merged layer: expected %x account to be nil", h2) } - if _, ok := merged.destructSet[h1]; !ok { // Note, drops stay alive until persisted to disk! - t.Errorf("merged diff layer: expected %x drop to be present", h1) - } - if _, ok := merged.destructSet[h2]; !ok { // Note, drops stay alive until persisted to disk! - t.Errorf("merged diff layer: expected %x drop to be present", h1) - } // If we add more granular metering of memory, we can enable this again, // but it's not implemented for now //if have, want := merged.memory, child.memory; have != want { @@ -206,22 +173,20 @@ func TestInsertAndMerge(t *testing.T) { ) { var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) ) - parent = newDiffLayer(emptyLayer(), common.Hash{}, destructs, accounts, storage) + parent = newDiffLayer(emptyLayer(), common.Hash{}, accounts, storage) } { var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) ) accounts[acc] = randomAccount() storage[acc] = make(map[common.Hash][]byte) storage[acc][slot] = []byte{0x01} - child = newDiffLayer(parent, common.Hash{}, destructs, accounts, storage) + child = newDiffLayer(parent, common.Hash{}, accounts, storage) } // And flatten merged := (child.flatten()).(*diffLayer) @@ -250,14 +215,13 @@ func BenchmarkSearch(b *testing.B) { // First, we set up 128 diff layers, with 1K items each fill := func(parent snapshot) *diffLayer { var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) ) for i := 0; i < 10000; i++ { accounts[randomHash()] = randomAccount() } - return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage) + return newDiffLayer(parent, common.Hash{}, accounts, storage) } var layer snapshot layer = emptyLayer() @@ -286,9 +250,8 @@ func BenchmarkSearchSlot(b *testing.B) { accountRLP := randomAccount() fill := func(parent snapshot) *diffLayer { var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) ) accounts[accountKey] = accountRLP @@ -299,7 +262,7 @@ func BenchmarkSearchSlot(b *testing.B) { accStorage[randomHash()] = value storage[accountKey] = accStorage } - return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage) + return newDiffLayer(parent, common.Hash{}, accounts, storage) } var layer snapshot layer = emptyLayer() @@ -320,9 +283,8 @@ func BenchmarkSearchSlot(b *testing.B) { func BenchmarkFlatten(b *testing.B) { fill := func(parent snapshot) *diffLayer { var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) ) for i := 0; i < 100; i++ { accountKey := randomHash() @@ -336,7 +298,7 @@ func BenchmarkFlatten(b *testing.B) { } storage[accountKey] = accStorage } - return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage) + return newDiffLayer(parent, common.Hash{}, accounts, storage) } b.ResetTimer() for i := 0; i < b.N; i++ { @@ -369,9 +331,8 @@ func BenchmarkFlatten(b *testing.B) { func BenchmarkJournal(b *testing.B) { fill := func(parent snapshot) *diffLayer { var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) ) for i := 0; i < 200; i++ { accountKey := randomHash() @@ -385,7 +346,7 @@ func BenchmarkJournal(b *testing.B) { } storage[accountKey] = accStorage } - return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage) + return newDiffLayer(parent, common.Hash{}, accounts, storage) } layer := snapshot(emptyLayer()) for i := 1; i < 128; i++ { diff --git a/core/state/snapshot/disklayer.go b/core/state/snapshot/disklayer.go index 76928edf07..202e6c70ed 100644 --- a/core/state/snapshot/disklayer.go +++ b/core/state/snapshot/disklayer.go @@ -180,8 +180,8 @@ func (dl *diskLayer) Storage(accountHash, storageHash common.Hash) ([]byte, erro // Update creates a new layer on top of the existing snapshot diff tree with // the specified data items. Note, the maps are retained by the method to avoid // copying everything. -func (dl *diskLayer) Update(blockHash common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer { - return newDiffLayer(dl, blockHash, destructs, accounts, storage) +func (dl *diskLayer) Update(blockHash common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer { + return newDiffLayer(dl, blockHash, accounts, storage) } // stopGeneration aborts the state snapshot generation if it is currently running. diff --git a/core/state/snapshot/disklayer_test.go b/core/state/snapshot/disklayer_test.go index 168458c405..6d99a90d61 100644 --- a/core/state/snapshot/disklayer_test.go +++ b/core/state/snapshot/disklayer_test.go @@ -117,20 +117,22 @@ func TestDiskMerge(t *testing.T) { base.Storage(conNukeCache, conNukeCacheSlot) // Modify or delete some accounts, flatten everything onto disk - if err := snaps.Update(diffRoot, baseRoot, map[common.Hash]struct{}{ - accDelNoCache: {}, - accDelCache: {}, - conNukeNoCache: {}, - conNukeCache: {}, - }, map[common.Hash][]byte{ - accModNoCache: reverse(accModNoCache[:]), - accModCache: reverse(accModCache[:]), - }, map[common.Hash]map[common.Hash][]byte{ - conModNoCache: {conModNoCacheSlot: reverse(conModNoCacheSlot[:])}, - conModCache: {conModCacheSlot: reverse(conModCacheSlot[:])}, - conDelNoCache: {conDelNoCacheSlot: nil}, - conDelCache: {conDelCacheSlot: nil}, - }); err != nil { + if err := snaps.Update(diffRoot, baseRoot, + map[common.Hash][]byte{ + accDelNoCache: nil, + accDelCache: nil, + conNukeNoCache: nil, + conNukeCache: nil, + accModNoCache: reverse(accModNoCache[:]), + accModCache: reverse(accModCache[:]), + }, map[common.Hash]map[common.Hash][]byte{ + conNukeNoCache: {conNukeNoCacheSlot: nil}, + conNukeCache: {conNukeCacheSlot: nil}, + conModNoCache: {conModNoCacheSlot: reverse(conModNoCacheSlot[:])}, + conModCache: {conModCacheSlot: reverse(conModCacheSlot[:])}, + conDelNoCache: {conDelNoCacheSlot: nil}, + conDelCache: {conDelCacheSlot: nil}, + }); err != nil { t.Fatalf("failed to update snapshot tree: %v", err) } if err := snaps.Cap(diffRoot, 0); err != nil { @@ -340,20 +342,27 @@ func TestDiskPartialMerge(t *testing.T) { assertStorage(conNukeCache, conNukeCacheSlot, conNukeCacheSlot[:]) // Modify or delete some accounts, flatten everything onto disk - if err := snaps.Update(diffRoot, baseRoot, map[common.Hash]struct{}{ - accDelNoCache: {}, - accDelCache: {}, - conNukeNoCache: {}, - conNukeCache: {}, - }, map[common.Hash][]byte{ - accModNoCache: reverse(accModNoCache[:]), - accModCache: reverse(accModCache[:]), - }, map[common.Hash]map[common.Hash][]byte{ - conModNoCache: {conModNoCacheSlot: reverse(conModNoCacheSlot[:])}, - conModCache: {conModCacheSlot: reverse(conModCacheSlot[:])}, - conDelNoCache: {conDelNoCacheSlot: nil}, - conDelCache: {conDelCacheSlot: nil}, - }); err != nil { + if err := snaps.Update(diffRoot, baseRoot, + map[common.Hash][]byte{ + accDelNoCache: nil, + accDelCache: nil, + conNukeNoCache: nil, + conNukeCache: nil, + accModNoCache: reverse(accModNoCache[:]), + accModCache: reverse(accModCache[:]), + }, + map[common.Hash]map[common.Hash][]byte{ + conNukeNoCache: { + conNukeNoCacheSlot: nil, + }, + conNukeCache: { + conNukeCacheSlot: nil, + }, + conModNoCache: {conModNoCacheSlot: reverse(conModNoCacheSlot[:])}, + conModCache: {conModCacheSlot: reverse(conModCacheSlot[:])}, + conDelNoCache: {conDelNoCacheSlot: nil}, + conDelCache: {conDelCacheSlot: nil}, + }); err != nil { t.Fatalf("test %d: failed to update snapshot tree: %v", i, err) } if err := snaps.Cap(diffRoot, 0); err != nil { @@ -462,9 +471,11 @@ func TestDiskGeneratorPersistence(t *testing.T) { }, } // Modify or delete some accounts, flatten everything onto disk - if err := snaps.Update(diffRoot, baseRoot, nil, map[common.Hash][]byte{ - accTwo: accTwo[:], - }, nil); err != nil { + if err := snaps.Update(diffRoot, baseRoot, + map[common.Hash][]byte{ + accTwo: accTwo[:], + }, nil, + ); err != nil { t.Fatalf("failed to update snapshot tree: %v", err) } if err := snaps.Cap(diffRoot, 0); err != nil { @@ -480,11 +491,14 @@ func TestDiskGeneratorPersistence(t *testing.T) { } // Test scenario 2, the disk layer is fully generated // Modify or delete some accounts, flatten everything onto disk - if err := snaps.Update(diffTwoRoot, diffRoot, nil, map[common.Hash][]byte{ - accThree: accThree.Bytes(), - }, map[common.Hash]map[common.Hash][]byte{ - accThree: {accThreeSlot: accThreeSlot.Bytes()}, - }); err != nil { + if err := snaps.Update(diffTwoRoot, diffRoot, + map[common.Hash][]byte{ + accThree: accThree.Bytes(), + }, + map[common.Hash]map[common.Hash][]byte{ + accThree: {accThreeSlot: accThreeSlot.Bytes()}, + }, + ); err != nil { t.Fatalf("failed to update snapshot tree: %v", err) } diskLayer := snaps.layers[snaps.diskRoot()].(*diskLayer) diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index 56abff348d..661610840a 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -134,7 +134,7 @@ func checkSnapRoot(t *testing.T, snap *diskLayer, trieRoot common.Hash) { snapRoot, err := generateTrieRoot(nil, "", accIt, common.Hash{}, stackTrieGenerate, func(db ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) { - storageIt, _ := snap.StorageIterator(accountHash, common.Hash{}) + storageIt := snap.StorageIterator(accountHash, common.Hash{}) defer storageIt.Release() hash, err := generateTrieRoot(nil, "", storageIt, accountHash, stackTrieGenerate, nil, stat, false) diff --git a/core/state/snapshot/iterator.go b/core/state/snapshot/iterator.go index c1a196c7ff..cd9c225844 100644 --- a/core/state/snapshot/iterator.go +++ b/core/state/snapshot/iterator.go @@ -115,6 +115,7 @@ func (it *diffAccountIterator) Next() bool { } // Iterator seems to be still alive, retrieve and cache the live hash it.curHash = it.keys[0] + // key cached, shift the iterator and notify the user of success it.keys = it.keys[1:] return true @@ -135,7 +136,7 @@ func (it *diffAccountIterator) Hash() common.Hash { // This method may _fail_, if the underlying layer has been flattened between // the call to Next and Account. That type of error will set it.Err. // This method assumes that flattening does not delete elements from -// the accountdata mapping (writing nil into it is fine though), and will panic +// the accountData mapping (writing nil into it is fine though), and will panic // if elements have been deleted. // // Note the returned account is not a copy, please don't modify it. @@ -143,10 +144,6 @@ func (it *diffAccountIterator) Account() []byte { it.layer.lock.RLock() blob, ok := it.layer.accountData[it.curHash] if !ok { - if _, ok := it.layer.destructSet[it.curHash]; ok { - it.layer.lock.RUnlock() - return nil - } panic(fmt.Sprintf("iterator referenced non-existent account: %x", it.curHash)) } it.layer.lock.RUnlock() @@ -247,11 +244,11 @@ type diffStorageIterator struct { // "destructed" returned. If it's true then it means the whole storage is // destructed in this layer(maybe recreated too), don't bother deeper layer // for storage retrieval. -func (dl *diffLayer) StorageIterator(account common.Hash, seek common.Hash) (StorageIterator, bool) { +func (dl *diffLayer) StorageIterator(account common.Hash, seek common.Hash) StorageIterator { // Create the storage for this account even it's marked // as destructed. The iterator is for the new one which // just has the same address as the deleted one. - hashes, destructed := dl.StorageList(account) + hashes := dl.StorageList(account) index := sort.Search(len(hashes), func(i int) bool { return bytes.Compare(seek[:], hashes[i][:]) <= 0 }) @@ -260,7 +257,7 @@ func (dl *diffLayer) StorageIterator(account common.Hash, seek common.Hash) (Sto layer: dl, account: account, keys: hashes[index:], - }, destructed + } } // Next steps the iterator forward one element, returning false if exhausted. @@ -339,13 +336,13 @@ type diskStorageIterator struct { // If the whole storage is destructed, then all entries in the disk // layer are deleted already. So the "destructed" flag returned here // is always false. -func (dl *diskLayer) StorageIterator(account common.Hash, seek common.Hash) (StorageIterator, bool) { +func (dl *diskLayer) StorageIterator(account common.Hash, seek common.Hash) StorageIterator { pos := common.TrimRightZeroes(seek[:]) return &diskStorageIterator{ layer: dl, account: account, it: dl.diskdb.NewIterator(append(rawdb.SnapshotStoragePrefix, account.Bytes()...), pos), - }, false + } } // Next steps the iterator forward one element, returning false if exhausted. diff --git a/core/state/snapshot/iterator_binary.go b/core/state/snapshot/iterator_binary.go index 523c7b9946..254c4b860f 100644 --- a/core/state/snapshot/iterator_binary.go +++ b/core/state/snapshot/iterator_binary.go @@ -67,44 +67,17 @@ func (dl *diffLayer) initBinaryAccountIterator(seek common.Hash) Iterator { func (dl *diffLayer) initBinaryStorageIterator(account, seek common.Hash) Iterator { parent, ok := dl.parent.(*diffLayer) if !ok { - // If the storage in this layer is already destructed, discard all - // deeper layers but still return a valid single-branch iterator. - a, destructed := dl.StorageIterator(account, seek) - if destructed { - l := &binaryIterator{ - a: a, - account: account, - } - l.aDone = !l.a.Next() - l.bDone = true - return l - } - // The parent is disk layer, don't need to take care "destructed" - // anymore. - b, _ := dl.Parent().StorageIterator(account, seek) l := &binaryIterator{ - a: a, - b: b, + a: dl.StorageIterator(account, seek), + b: dl.Parent().StorageIterator(account, seek), account: account, } l.aDone = !l.a.Next() l.bDone = !l.b.Next() return l } - // If the storage in this layer is already destructed, discard all - // deeper layers but still return a valid single-branch iterator. - a, destructed := dl.StorageIterator(account, seek) - if destructed { - l := &binaryIterator{ - a: a, - account: account, - } - l.aDone = !l.a.Next() - l.bDone = true - return l - } l := &binaryIterator{ - a: a, + a: dl.StorageIterator(account, seek), b: parent.initBinaryStorageIterator(account, seek), account: account, } diff --git a/core/state/snapshot/iterator_fast.go b/core/state/snapshot/iterator_fast.go index fa0daea7ba..7f7ba876ff 100644 --- a/core/state/snapshot/iterator_fast.go +++ b/core/state/snapshot/iterator_fast.go @@ -90,18 +90,10 @@ func newFastIterator(tree *Tree, root common.Hash, account common.Hash, seek com priority: depth, }) } else { - // If the whole storage is destructed in this layer, don't - // bother deeper layer anymore. But we should still keep - // the iterator for this layer, since the iterator can contain - // some valid slots which belongs to the re-created account. - it, destructed := current.StorageIterator(account, seek) fi.iterators = append(fi.iterators, &weightedIterator{ - it: it, + it: current.StorageIterator(account, seek), priority: depth, }) - if destructed { - break - } } current = current.Parent() } diff --git a/core/state/snapshot/iterator_test.go b/core/state/snapshot/iterator_test.go index f2f4d19f83..b9fe370b86 100644 --- a/core/state/snapshot/iterator_test.go +++ b/core/state/snapshot/iterator_test.go @@ -32,9 +32,8 @@ import ( // TestAccountIteratorBasics tests some simple single-layer(diff and disk) iteration func TestAccountIteratorBasics(t *testing.T) { var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) ) // Fill up a parent for i := 0; i < 100; i++ { @@ -42,9 +41,6 @@ func TestAccountIteratorBasics(t *testing.T) { data := randomAccount() accounts[h] = data - if rand.Intn(4) == 0 { - destructs[h] = struct{}{} - } if rand.Intn(2) == 0 { accStorage := make(map[common.Hash][]byte) value := make([]byte, 32) @@ -54,7 +50,7 @@ func TestAccountIteratorBasics(t *testing.T) { } } // Add some (identical) layers on top - diffLayer := newDiffLayer(emptyLayer(), common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) + diffLayer := newDiffLayer(emptyLayer(), common.Hash{}, copyAccounts(accounts), copyStorage(storage)) it := diffLayer.AccountIterator(common.Hash{}) verifyIterator(t, 100, it, verifyNothing) // Nil is allowed for single layer iterator @@ -95,15 +91,15 @@ func TestStorageIteratorBasics(t *testing.T) { nilStorage[h] = nilstorage } // Add some (identical) layers on top - diffLayer := newDiffLayer(emptyLayer(), common.Hash{}, nil, copyAccounts(accounts), copyStorage(storage)) + diffLayer := newDiffLayer(emptyLayer(), common.Hash{}, copyAccounts(accounts), copyStorage(storage)) for account := range accounts { - it, _ := diffLayer.StorageIterator(account, common.Hash{}) + it := diffLayer.StorageIterator(account, common.Hash{}) verifyIterator(t, 100, it, verifyNothing) // Nil is allowed for single layer iterator } diskLayer := diffToDisk(diffLayer) for account := range accounts { - it, _ := diskLayer.StorageIterator(account, common.Hash{}) + it := diskLayer.StorageIterator(account, common.Hash{}) verifyIterator(t, 100-nilStorage[account], it, verifyNothing) // Nil is allowed for single layer iterator } } @@ -225,13 +221,13 @@ func TestAccountIteratorTraversal(t *testing.T) { }, } // Stack three diff layers on top with various overlaps - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), randomAccountSet("0xaa", "0xee", "0xff", "0xf0"), nil) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), randomAccountSet("0xbb", "0xdd", "0xf0"), nil) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), randomAccountSet("0xcc", "0xf0", "0xff"), nil) // Verify the single and multi-layer iterators @@ -272,19 +268,19 @@ func TestStorageIteratorTraversal(t *testing.T) { }, } // Stack three diff layers on top with various overlaps - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x02", "0x03"}}, nil)) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x04", "0x05", "0x06"}}, nil)) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x02", "0x03"}}, nil)) // Verify the single and multi-layer iterators head := snaps.Snapshot(common.HexToHash("0x04")) - diffIter, _ := head.(snapshot).StorageIterator(common.HexToHash("0xaa"), common.Hash{}) + diffIter := head.(snapshot).StorageIterator(common.HexToHash("0xaa"), common.Hash{}) verifyIterator(t, 3, diffIter, verifyNothing) verifyIterator(t, 6, head.(*diffLayer).newBinaryStorageIterator(common.HexToHash("0xaa"), common.Hash{}), verifyStorage) @@ -357,14 +353,14 @@ func TestAccountIteratorTraversalValues(t *testing.T) { } } // Assemble a stack of snapshots from the account layers - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, a, nil) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, b, nil) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, c, nil) - snaps.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), nil, d, nil) - snaps.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), nil, e, nil) - snaps.Update(common.HexToHash("0x07"), common.HexToHash("0x06"), nil, f, nil) - snaps.Update(common.HexToHash("0x08"), common.HexToHash("0x07"), nil, g, nil) - snaps.Update(common.HexToHash("0x09"), common.HexToHash("0x08"), nil, h, nil) + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), a, nil) + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), b, nil) + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), c, nil) + snaps.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), d, nil) + snaps.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), e, nil) + snaps.Update(common.HexToHash("0x07"), common.HexToHash("0x06"), f, nil) + snaps.Update(common.HexToHash("0x08"), common.HexToHash("0x07"), g, nil) + snaps.Update(common.HexToHash("0x09"), common.HexToHash("0x08"), h, nil) it, _ := snaps.AccountIterator(common.HexToHash("0x09"), common.Hash{}) head := snaps.Snapshot(common.HexToHash("0x09")) @@ -456,14 +452,14 @@ func TestStorageIteratorTraversalValues(t *testing.T) { } } // Assemble a stack of snapshots from the account layers - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, randomAccountSet("0xaa"), wrapStorage(a)) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, randomAccountSet("0xaa"), wrapStorage(b)) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, randomAccountSet("0xaa"), wrapStorage(c)) - snaps.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), nil, randomAccountSet("0xaa"), wrapStorage(d)) - snaps.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), nil, randomAccountSet("0xaa"), wrapStorage(e)) - snaps.Update(common.HexToHash("0x07"), common.HexToHash("0x06"), nil, randomAccountSet("0xaa"), wrapStorage(e)) - snaps.Update(common.HexToHash("0x08"), common.HexToHash("0x07"), nil, randomAccountSet("0xaa"), wrapStorage(g)) - snaps.Update(common.HexToHash("0x09"), common.HexToHash("0x08"), nil, randomAccountSet("0xaa"), wrapStorage(h)) + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), randomAccountSet("0xaa"), wrapStorage(a)) + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), randomAccountSet("0xaa"), wrapStorage(b)) + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), randomAccountSet("0xaa"), wrapStorage(c)) + snaps.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), randomAccountSet("0xaa"), wrapStorage(d)) + snaps.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), randomAccountSet("0xaa"), wrapStorage(e)) + snaps.Update(common.HexToHash("0x07"), common.HexToHash("0x06"), randomAccountSet("0xaa"), wrapStorage(e)) + snaps.Update(common.HexToHash("0x08"), common.HexToHash("0x07"), randomAccountSet("0xaa"), wrapStorage(g)) + snaps.Update(common.HexToHash("0x09"), common.HexToHash("0x08"), randomAccountSet("0xaa"), wrapStorage(h)) it, _ := snaps.StorageIterator(common.HexToHash("0x09"), common.HexToHash("0xaa"), common.Hash{}) head := snaps.Snapshot(common.HexToHash("0x09")) @@ -526,7 +522,7 @@ func TestAccountIteratorLargeTraversal(t *testing.T) { }, } for i := 1; i < 128; i++ { - snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), nil, makeAccounts(200), nil) + snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), makeAccounts(200), nil) } // Iterate the entire stack and ensure everything is hit only once head := snaps.Snapshot(common.HexToHash("0x80")) @@ -584,13 +580,13 @@ func testAccountIteratorFlattening(t *testing.T, newIterator func(snaps *Tree, r }, } // Create a stack of diffs on top - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), randomAccountSet("0xaa", "0xee", "0xff", "0xf0"), nil) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), randomAccountSet("0xbb", "0xdd", "0xf0"), nil) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), randomAccountSet("0xcc", "0xf0", "0xff"), nil) // Create an iterator and flatten the data from underneath it @@ -630,13 +626,13 @@ func testAccountIteratorSeek(t *testing.T, newIterator func(snaps *Tree, root, s base.root: base, }, } - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), randomAccountSet("0xaa", "0xee", "0xff", "0xf0"), nil) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), randomAccountSet("0xbb", "0xdd", "0xf0"), nil) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), randomAccountSet("0xcc", "0xf0", "0xff"), nil) // Account set is now @@ -708,13 +704,13 @@ func testStorageIteratorSeek(t *testing.T, newIterator func(snaps *Tree, root, a }, } // Stack three diff layers on top with various overlaps - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x03", "0x05"}}, nil)) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x02", "0x05", "0x06"}}, nil)) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x05", "0x08"}}, nil)) // Account set is now @@ -785,21 +781,16 @@ func testAccountIteratorDeletions(t *testing.T, newIterator func(snaps *Tree, ro }, } // Stack three diff layers on top with various overlaps - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), - nil, randomAccountSet("0x11", "0x22", "0x33"), nil) - - deleted := common.HexToHash("0x22") - destructed := map[common.Hash]struct{}{ - deleted: {}, - } - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), - destructed, randomAccountSet("0x11", "0x33"), nil) + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), randomAccountSet("0x11", "0x22", "0x33"), nil) + set := randomAccountSet("0x11", "0x33") + set[common.HexToHash("0x22")] = nil + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), set, nil) snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), - nil, randomAccountSet("0x33", "0x44", "0x55"), nil) + randomAccountSet("0x33", "0x44", "0x55"), nil) // The output should be 11,33,44,55 - it := newIterator(snaps, common.HexToHash("0x04"), (common.Hash{})) + it := newIterator(snaps, common.HexToHash("0x04"), common.Hash{}) // Do a quick check verifyIterator(t, 4, it, verifyAccount) @@ -813,8 +804,8 @@ func testAccountIteratorDeletions(t *testing.T, newIterator func(snaps *Tree, ro if it.Account() == nil { t.Errorf("iterator returned nil-value for hash %x", hash) } - if hash == deleted { - t.Errorf("expected deleted elem %x to not be returned by iterator", deleted) + if hash == common.HexToHash("0x22") { + t.Errorf("expected deleted elem %x to not be returned by iterator", common.HexToHash("0x22")) } } } @@ -846,10 +837,10 @@ func testStorageIteratorDeletions(t *testing.T, newIterator func(snaps *Tree, ro }, } // Stack three diff layers on top with various overlaps - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x03", "0x05"}}, nil)) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x02", "0x04", "0x06"}}, [][]string{{"0x01", "0x03"}})) // The output should be 02,04,05,06 @@ -863,17 +854,16 @@ func testStorageIteratorDeletions(t *testing.T, newIterator func(snaps *Tree, ro it.Release() // Destruct the whole storage - destructed := map[common.Hash]struct{}{ - common.HexToHash("0xaa"): {}, - } - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), destructed, nil, nil) + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), + map[common.Hash][]byte{common.HexToHash("0xaa"): nil}, + randomStorageSet([]string{"0xaa"}, nil, [][]string{{"0x02", "0x04", "0x05", "0x06"}})) it = newIterator(snaps, common.HexToHash("0x04"), common.HexToHash("0xaa"), common.Hash{}) verifyIterator(t, 0, it, verifyStorage) it.Release() // Re-insert the slots of the same account - snaps.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), nil, + snaps.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x07", "0x08", "0x09"}}, nil)) // The output should be 07,08,09 @@ -883,7 +873,9 @@ func testStorageIteratorDeletions(t *testing.T, newIterator func(snaps *Tree, ro it.Release() // Destruct the whole storage but re-create the account in the same layer - snaps.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), destructed, randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x11", "0x12"}}, nil)) + snaps.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), + randomAccountSet("0xaa"), + randomStorageSet([]string{"0xaa"}, [][]string{{"0x11", "0x12"}}, [][]string{{"0x07", "0x08", "0x09"}})) it = newIterator(snaps, common.HexToHash("0x06"), common.HexToHash("0xaa"), common.Hash{}) verifyIterator(t, 2, it, verifyStorage) // The output should be 11,12 it.Release() @@ -928,7 +920,7 @@ func BenchmarkAccountIteratorTraversal(b *testing.B) { }, } for i := 1; i <= 100; i++ { - snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), nil, makeAccounts(200), nil) + snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), makeAccounts(200), nil) } // We call this once before the benchmark, so the creation of // sorted accountlists are not included in the results. @@ -1023,9 +1015,9 @@ func BenchmarkAccountIteratorLargeBaselayer(b *testing.B) { base.root: base, }, } - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, makeAccounts(2000), nil) + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), makeAccounts(2000), nil) for i := 2; i <= 100; i++ { - snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), nil, makeAccounts(20), nil) + snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), makeAccounts(20), nil) } // We call this once before the benchmark, so the creation of // sorted accountlists are not included in the results. diff --git a/core/state/snapshot/journal.go b/core/state/snapshot/journal.go index 8513e73dd0..61552c73fe 100644 --- a/core/state/snapshot/journal.go +++ b/core/state/snapshot/journal.go @@ -33,7 +33,9 @@ import ( "github.com/ethereum/go-ethereum/triedb" ) -const journalVersion uint64 = 0 +// 0: initial version +// 1: destruct flag in diff layer is removed +const journalVersion uint64 = 1 // journalGenerator is a disk layer entry containing the generator progress marker. type journalGenerator struct { @@ -48,11 +50,6 @@ type journalGenerator struct { Storage uint64 } -// journalDestruct is an account deletion entry in a diffLayer's disk journal. -type journalDestruct struct { - Hash common.Hash -} - // journalAccount is an account entry in a diffLayer's disk journal. type journalAccount struct { Hash common.Hash @@ -109,8 +106,8 @@ func loadAndParseJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, jou // is not matched with disk layer; or the it's the legacy-format journal, // etc.), we just discard all diffs and try to recover them later. var current snapshot = base - err := iterateJournal(db, func(parent common.Hash, root common.Hash, destructSet map[common.Hash]struct{}, accountData map[common.Hash][]byte, storageData map[common.Hash]map[common.Hash][]byte) error { - current = newDiffLayer(current, root, destructSet, accountData, storageData) + err := iterateJournal(db, func(parent common.Hash, root common.Hash, accountData map[common.Hash][]byte, storageData map[common.Hash]map[common.Hash][]byte) error { + current = newDiffLayer(current, root, accountData, storageData) return nil }) if err != nil { @@ -238,16 +235,12 @@ func (dl *diffLayer) Journal(buffer *bytes.Buffer) (common.Hash, error) { if err := rlp.Encode(buffer, dl.root); err != nil { return common.Hash{}, err } - destructs := make([]journalDestruct, 0, len(dl.destructSet)) - for hash := range dl.destructSet { - destructs = append(destructs, journalDestruct{Hash: hash}) - } - if err := rlp.Encode(buffer, destructs); err != nil { - return common.Hash{}, err - } accounts := make([]journalAccount, 0, len(dl.accountData)) for hash, blob := range dl.accountData { - accounts = append(accounts, journalAccount{Hash: hash, Blob: blob}) + accounts = append(accounts, journalAccount{ + Hash: hash, + Blob: blob, + }) } if err := rlp.Encode(buffer, accounts); err != nil { return common.Hash{}, err @@ -271,7 +264,7 @@ func (dl *diffLayer) Journal(buffer *bytes.Buffer) (common.Hash, error) { // journalCallback is a function which is invoked by iterateJournal, every // time a difflayer is loaded from disk. -type journalCallback = func(parent common.Hash, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error +type journalCallback = func(parent common.Hash, root common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error // iterateJournal iterates through the journalled difflayers, loading them from // the database, and invoking the callback for each loaded layer. @@ -310,10 +303,8 @@ func iterateJournal(db ethdb.KeyValueReader, callback journalCallback) error { for { var ( root common.Hash - destructs []journalDestruct accounts []journalAccount storage []journalStorage - destructSet = make(map[common.Hash]struct{}) accountData = make(map[common.Hash][]byte) storageData = make(map[common.Hash]map[common.Hash][]byte) ) @@ -325,18 +316,12 @@ func iterateJournal(db ethdb.KeyValueReader, callback journalCallback) error { } return fmt.Errorf("load diff root: %v", err) } - if err := r.Decode(&destructs); err != nil { - return fmt.Errorf("load diff destructs: %v", err) - } if err := r.Decode(&accounts); err != nil { return fmt.Errorf("load diff accounts: %v", err) } if err := r.Decode(&storage); err != nil { return fmt.Errorf("load diff storage: %v", err) } - for _, entry := range destructs { - destructSet[entry.Hash] = struct{}{} - } for _, entry := range accounts { if len(entry.Blob) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that accountData[entry.Hash] = entry.Blob @@ -355,7 +340,7 @@ func iterateJournal(db ethdb.KeyValueReader, callback journalCallback) error { } storageData[entry.Hash] = slots } - if err := callback(parent, root, destructSet, accountData, storageData); err != nil { + if err := callback(parent, root, accountData, storageData); err != nil { return err } parent = root diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index d1ffb5cc2d..7466f12351 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -130,7 +130,7 @@ type snapshot interface { // the specified data items. // // Note, the maps are retained by the method to avoid copying everything. - Update(blockRoot common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer + Update(blockRoot common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer // Journal commits an entire diff hierarchy to disk into a single journal entry. // This is meant to be used during shutdown to persist the snapshot without @@ -145,7 +145,7 @@ type snapshot interface { AccountIterator(seek common.Hash) AccountIterator // StorageIterator creates a storage iterator over an arbitrary layer. - StorageIterator(account common.Hash, seek common.Hash) (StorageIterator, bool) + StorageIterator(account common.Hash, seek common.Hash) StorageIterator } // Config includes the configurations for snapshots. @@ -335,7 +335,7 @@ func (t *Tree) Snapshots(root common.Hash, limits int, nodisk bool) []Snapshot { // Update adds a new snapshot into the tree, if that can be linked to an existing // old parent. It is disallowed to insert a disk layer (the origin of all). -func (t *Tree) Update(blockRoot common.Hash, parentRoot common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { +func (t *Tree) Update(blockRoot common.Hash, parentRoot common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { // Reject noop updates to avoid self-loops in the snapshot tree. This is a // special case that can only happen for Clique networks where empty blocks // don't modify the state (0 block subsidy). @@ -350,7 +350,7 @@ func (t *Tree) Update(blockRoot common.Hash, parentRoot common.Hash, destructs m if parent == nil { return fmt.Errorf("parent [%#x] snapshot missing", parentRoot) } - snap := parent.(snapshot).Update(blockRoot, destructs, accounts, storage) + snap := parent.(snapshot).Update(blockRoot, accounts, storage) // Save the new snapshot for later t.lock.Lock() @@ -539,35 +539,6 @@ func diffToDisk(bottom *diffLayer) *diskLayer { base.stale = true base.lock.Unlock() - // Destroy all the destructed accounts from the database - for hash := range bottom.destructSet { - // Skip any account not covered yet by the snapshot - if base.genMarker != nil && bytes.Compare(hash[:], base.genMarker) > 0 { - continue - } - // Remove all storage slots - rawdb.DeleteAccountSnapshot(batch, hash) - base.cache.Set(hash[:], nil) - - it := rawdb.IterateStorageSnapshots(base.diskdb, hash) - for it.Next() { - key := it.Key() - batch.Delete(key) - base.cache.Del(key[1:]) - snapshotFlushStorageItemMeter.Mark(1) - - // Ensure we don't delete too much data blindly (contract can be - // huge). It's ok to flush, the root will go missing in case of a - // crash and we'll detect and regenerate the snapshot. - if batch.ValueSize() > 64*1024*1024 { - if err := batch.Write(); err != nil { - log.Crit("Failed to write storage deletions", "err", err) - } - batch.Reset() - } - } - it.Release() - } // Push all updated accounts into the database for hash, data := range bottom.accountData { // Skip any account not covered yet by the snapshot @@ -575,10 +546,14 @@ func diffToDisk(bottom *diffLayer) *diskLayer { continue } // Push the account to disk - rawdb.WriteAccountSnapshot(batch, hash, data) - base.cache.Set(hash[:], data) - snapshotCleanAccountWriteMeter.Mark(int64(len(data))) - + if len(data) != 0 { + rawdb.WriteAccountSnapshot(batch, hash, data) + base.cache.Set(hash[:], data) + snapshotCleanAccountWriteMeter.Mark(int64(len(data))) + } else { + rawdb.DeleteAccountSnapshot(batch, hash) + base.cache.Set(hash[:], nil) + } snapshotFlushAccountItemMeter.Mark(1) snapshotFlushAccountSizeMeter.Mark(int64(len(data))) @@ -587,7 +562,7 @@ func diffToDisk(bottom *diffLayer) *diskLayer { // the snapshot. if batch.ValueSize() > 64*1024*1024 { if err := batch.Write(); err != nil { - log.Crit("Failed to write storage deletions", "err", err) + log.Crit("Failed to write state changes", "err", err) } batch.Reset() } @@ -616,6 +591,16 @@ func diffToDisk(bottom *diffLayer) *diskLayer { } snapshotFlushStorageItemMeter.Mark(1) snapshotFlushStorageSizeMeter.Mark(int64(len(data))) + + // Ensure we don't write too much data blindly. It's ok to flush, the + // root will go missing in case of a crash and we'll detect and regen + // the snapshot. + if batch.ValueSize() > 64*1024*1024 { + if err := batch.Write(); err != nil { + log.Crit("Failed to write state changes", "err", err) + } + batch.Reset() + } } } // Update the snapshot block marker and write any remainder data diff --git a/core/state/snapshot/snapshot_test.go b/core/state/snapshot/snapshot_test.go index a9ab3eaea3..34ef61e8d0 100644 --- a/core/state/snapshot/snapshot_test.go +++ b/core/state/snapshot/snapshot_test.go @@ -107,7 +107,7 @@ func TestDiskLayerExternalInvalidationFullFlatten(t *testing.T) { accounts := map[common.Hash][]byte{ common.HexToHash("0xa1"): randomAccount(), } - if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil { + if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), accounts, nil); err != nil { t.Fatalf("failed to create a diff layer: %v", err) } if n := len(snaps.layers); n != 2 { @@ -151,10 +151,10 @@ func TestDiskLayerExternalInvalidationPartialFlatten(t *testing.T) { accounts := map[common.Hash][]byte{ common.HexToHash("0xa1"): randomAccount(), } - if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil { + if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), accounts, nil); err != nil { t.Fatalf("failed to create a diff layer: %v", err) } - if err := snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, accounts, nil); err != nil { + if err := snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), accounts, nil); err != nil { t.Fatalf("failed to create a diff layer: %v", err) } if n := len(snaps.layers); n != 3 { @@ -203,13 +203,13 @@ func TestDiffLayerExternalInvalidationPartialFlatten(t *testing.T) { accounts := map[common.Hash][]byte{ common.HexToHash("0xa1"): randomAccount(), } - if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil { + if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), accounts, nil); err != nil { t.Fatalf("failed to create a diff layer: %v", err) } - if err := snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, accounts, nil); err != nil { + if err := snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), accounts, nil); err != nil { t.Fatalf("failed to create a diff layer: %v", err) } - if err := snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, accounts, nil); err != nil { + if err := snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), accounts, nil); err != nil { t.Fatalf("failed to create a diff layer: %v", err) } if n := len(snaps.layers); n != 4 { @@ -263,12 +263,12 @@ func TestPostCapBasicDataAccess(t *testing.T) { }, } // The lowest difflayer - snaps.Update(common.HexToHash("0xa1"), common.HexToHash("0x01"), nil, setAccount("0xa1"), nil) - snaps.Update(common.HexToHash("0xa2"), common.HexToHash("0xa1"), nil, setAccount("0xa2"), nil) - snaps.Update(common.HexToHash("0xb2"), common.HexToHash("0xa1"), nil, setAccount("0xb2"), nil) + snaps.Update(common.HexToHash("0xa1"), common.HexToHash("0x01"), setAccount("0xa1"), nil) + snaps.Update(common.HexToHash("0xa2"), common.HexToHash("0xa1"), setAccount("0xa2"), nil) + snaps.Update(common.HexToHash("0xb2"), common.HexToHash("0xa1"), setAccount("0xb2"), nil) - snaps.Update(common.HexToHash("0xa3"), common.HexToHash("0xa2"), nil, setAccount("0xa3"), nil) - snaps.Update(common.HexToHash("0xb3"), common.HexToHash("0xb2"), nil, setAccount("0xb3"), nil) + snaps.Update(common.HexToHash("0xa3"), common.HexToHash("0xa2"), setAccount("0xa3"), nil) + snaps.Update(common.HexToHash("0xb3"), common.HexToHash("0xb2"), setAccount("0xb3"), nil) // checkExist verifies if an account exists in a snapshot checkExist := func(layer *diffLayer, key string) error { @@ -363,7 +363,7 @@ func TestSnaphots(t *testing.T) { ) for i := 0; i < 129; i++ { head = makeRoot(uint64(i + 2)) - snaps.Update(head, last, nil, setAccount(fmt.Sprintf("%d", i+2)), nil) + snaps.Update(head, last, setAccount(fmt.Sprintf("%d", i+2)), nil) last = head snaps.Cap(head, 128) // 130 layers (128 diffs + 1 accumulator + 1 disk) } @@ -456,9 +456,9 @@ func TestReadStateDuringFlattening(t *testing.T) { }, } // 4 layers in total, 3 diff layers and 1 disk layers - snaps.Update(common.HexToHash("0xa1"), common.HexToHash("0x01"), nil, setAccount("0xa1"), nil) - snaps.Update(common.HexToHash("0xa2"), common.HexToHash("0xa1"), nil, setAccount("0xa2"), nil) - snaps.Update(common.HexToHash("0xa3"), common.HexToHash("0xa2"), nil, setAccount("0xa3"), nil) + snaps.Update(common.HexToHash("0xa1"), common.HexToHash("0x01"), setAccount("0xa1"), nil) + snaps.Update(common.HexToHash("0xa2"), common.HexToHash("0xa1"), setAccount("0xa2"), nil) + snaps.Update(common.HexToHash("0xa3"), common.HexToHash("0xa2"), setAccount("0xa3"), nil) // Obtain the topmost snapshot handler for state accessing snap := snaps.Snapshot(common.HexToHash("0xa3")) diff --git a/core/state/snapshot/utils.go b/core/state/snapshot/utils.go index 62f073d2e1..c35c82f67a 100644 --- a/core/state/snapshot/utils.go +++ b/core/state/snapshot/utils.go @@ -75,7 +75,7 @@ func checkDanglingDiskStorage(chaindb ethdb.KeyValueStore) error { func checkDanglingMemStorage(db ethdb.KeyValueStore) error { start := time.Now() log.Info("Checking dangling journalled storage") - err := iterateJournal(db, func(pRoot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { + err := iterateJournal(db, func(pRoot, root common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { for accHash := range storage { if _, ok := accounts[accHash]; !ok { log.Error("Dangling storage - missing account", "account", fmt.Sprintf("%#x", accHash), "root", root) @@ -119,12 +119,11 @@ func CheckJournalAccount(db ethdb.KeyValueStore, hash common.Hash) error { } var depth = 0 - return iterateJournal(db, func(pRoot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { + return iterateJournal(db, func(pRoot, root common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { _, a := accounts[hash] - _, b := destructs[hash] - _, c := storage[hash] + _, b := storage[hash] depth++ - if !a && !b && !c { + if !a && !b { return nil } fmt.Printf("Disklayer+%d: Root: %x, parent %x\n", depth, root, pRoot) @@ -138,9 +137,6 @@ func CheckJournalAccount(db ethdb.KeyValueStore, hash common.Hash) error { fmt.Printf("\taccount.root: %x\n", account.Root) fmt.Printf("\taccount.codehash: %x\n", account.CodeHash) } - if _, ok := destructs[hash]; ok { - fmt.Printf("\t Destructed!") - } if data, ok := storage[hash]; ok { fmt.Printf("\tStorage\n") for k, v := range data { diff --git a/core/state/statedb.go b/core/state/statedb.go index d855e5626d..9cc91c9332 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -932,16 +932,17 @@ func (s *StateDB) clearJournalAndRefund() { // of a specific account. It leverages the associated state snapshot for fast // storage iteration and constructs trie node deletion markers by creating // stack trie with iterated slots. -func (s *StateDB) fastDeleteStorage(snaps *snapshot.Tree, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, *trienode.NodeSet, error) { +func (s *StateDB) fastDeleteStorage(snaps *snapshot.Tree, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, map[common.Hash][]byte, *trienode.NodeSet, error) { iter, err := snaps.StorageIterator(s.originalRoot, addrHash, common.Hash{}) if err != nil { - return nil, nil, err + return nil, nil, nil, err } defer iter.Release() var ( - nodes = trienode.NewNodeSet(addrHash) - slots = make(map[common.Hash][]byte) + nodes = trienode.NewNodeSet(addrHash) // the set for trie node mutations (value is nil) + storages = make(map[common.Hash][]byte) // the set for storage mutations (value is nil) + storageOrigins = make(map[common.Hash][]byte) // the set for tracking the original value of slot ) stack := trie.NewStackTrie(func(path []byte, hash common.Hash, blob []byte) { nodes.AddNode(path, trienode.NewDeleted()) @@ -949,42 +950,47 @@ func (s *StateDB) fastDeleteStorage(snaps *snapshot.Tree, addrHash common.Hash, for iter.Next() { slot := common.CopyBytes(iter.Slot()) if err := iter.Error(); err != nil { // error might occur after Slot function - return nil, nil, err + return nil, nil, nil, err } - slots[iter.Hash()] = slot + key := iter.Hash() + storages[key] = nil + storageOrigins[key] = slot - if err := stack.Update(iter.Hash().Bytes(), slot); err != nil { - return nil, nil, err + if err := stack.Update(key.Bytes(), slot); err != nil { + return nil, nil, nil, err } } if err := iter.Error(); err != nil { // error might occur during iteration - return nil, nil, err + return nil, nil, nil, err } if stack.Hash() != root { - return nil, nil, fmt.Errorf("snapshot is not matched, exp %x, got %x", root, stack.Hash()) + return nil, nil, nil, fmt.Errorf("snapshot is not matched, exp %x, got %x", root, stack.Hash()) } - return slots, nodes, nil + return storages, storageOrigins, nodes, nil } // slowDeleteStorage serves as a less-efficient alternative to "fastDeleteStorage," // employed when the associated state snapshot is not available. It iterates the // storage slots along with all internal trie nodes via trie directly. -func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, *trienode.NodeSet, error) { +func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, map[common.Hash][]byte, *trienode.NodeSet, error) { tr, err := s.db.OpenStorageTrie(s.originalRoot, addr, root, s.trie) if err != nil { - return nil, nil, fmt.Errorf("failed to open storage trie, err: %w", err) + return nil, nil, nil, fmt.Errorf("failed to open storage trie, err: %w", err) } it, err := tr.NodeIterator(nil) if err != nil { - return nil, nil, fmt.Errorf("failed to open storage iterator, err: %w", err) + return nil, nil, nil, fmt.Errorf("failed to open storage iterator, err: %w", err) } var ( - nodes = trienode.NewNodeSet(addrHash) - slots = make(map[common.Hash][]byte) + nodes = trienode.NewNodeSet(addrHash) // the set for trie node mutations (value is nil) + storages = make(map[common.Hash][]byte) // the set for storage mutations (value is nil) + storageOrigins = make(map[common.Hash][]byte) // the set for tracking the original value of slot ) for it.Next(true) { if it.Leaf() { - slots[common.BytesToHash(it.LeafKey())] = common.CopyBytes(it.LeafBlob()) + key := common.BytesToHash(it.LeafKey()) + storages[key] = nil + storageOrigins[key] = common.CopyBytes(it.LeafBlob()) continue } if it.Hash() == (common.Hash{}) { @@ -993,35 +999,36 @@ func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, r nodes.AddNode(it.Path(), trienode.NewDeleted()) } if err := it.Error(); err != nil { - return nil, nil, err + return nil, nil, nil, err } - return slots, nodes, nil + return storages, storageOrigins, nodes, nil } // deleteStorage is designed to delete the storage trie of a designated account. // The function will make an attempt to utilize an efficient strategy if the // associated state snapshot is reachable; otherwise, it will resort to a less // efficient approach. -func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, *trienode.NodeSet, error) { +func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, map[common.Hash][]byte, *trienode.NodeSet, error) { var ( - err error - slots map[common.Hash][]byte - nodes *trienode.NodeSet + err error + nodes *trienode.NodeSet // the set for trie node mutations (value is nil) + storages map[common.Hash][]byte // the set for storage mutations (value is nil) + storageOrigins map[common.Hash][]byte // the set for tracking the original value of slot ) // The fast approach can be failed if the snapshot is not fully // generated, or it's internally corrupted. Fallback to the slow // one just in case. snaps := s.db.Snapshot() if snaps != nil { - slots, nodes, err = s.fastDeleteStorage(snaps, addrHash, root) + storages, storageOrigins, nodes, err = s.fastDeleteStorage(snaps, addrHash, root) } if snaps == nil || err != nil { - slots, nodes, err = s.slowDeleteStorage(addr, addrHash, root) + storages, storageOrigins, nodes, err = s.slowDeleteStorage(addr, addrHash, root) } if err != nil { - return nil, nil, err + return nil, nil, nil, err } - return slots, nodes, nil + return storages, storageOrigins, nodes, nil } // handleDestruction processes all destruction markers and deletes the account @@ -1068,16 +1075,16 @@ func (s *StateDB) handleDestruction() (map[common.Hash]*accountDelete, []*trieno deletes[addrHash] = op // Short circuit if the origin storage was empty. - if prev.Root == types.EmptyRootHash || s.db.TrieDB().IsVerkle() { continue } // Remove storage slots belonging to the account. - slots, set, err := s.deleteStorage(addr, addrHash, prev.Root) + storages, storagesOrigin, set, err := s.deleteStorage(addr, addrHash, prev.Root) if err != nil { return nil, nil, fmt.Errorf("failed to delete storage, err: %w", err) } - op.storagesOrigin = slots + op.storages = storages + op.storagesOrigin = storagesOrigin // Aggregate the associated trie node changes. nodes = append(nodes, set) @@ -1267,7 +1274,7 @@ func (s *StateDB) commitAndFlush(block uint64, deleteEmptyObjects bool) (*stateU // If snapshotting is enabled, update the snapshot tree with this new version if snap := s.db.Snapshot(); snap != nil && snap.Snapshot(ret.originRoot) != nil { start := time.Now() - if err := snap.Update(ret.root, ret.originRoot, ret.destructs, ret.accounts, ret.storages); err != nil { + if err := snap.Update(ret.root, ret.originRoot, ret.accounts, ret.storages); err != nil { log.Warn("Failed to update snapshot tree", "from", ret.originRoot, "to", ret.root, "err", err) } // Keep 128 diff layers in the memory, persistent layer is 129th. diff --git a/core/state/statedb_fuzz_test.go b/core/state/statedb_fuzz_test.go index 90250819e3..7cbfd9b9d7 100644 --- a/core/state/statedb_fuzz_test.go +++ b/core/state/statedb_fuzz_test.go @@ -21,6 +21,7 @@ import ( "encoding/binary" "errors" "fmt" + "maps" "math" "math/rand" "reflect" @@ -176,24 +177,16 @@ func (test *stateTest) String() string { func (test *stateTest) run() bool { var ( - roots []common.Hash - accountList []map[common.Address][]byte - storageList []map[common.Address]map[common.Hash][]byte - copyUpdate = func(update *stateUpdate) { - accounts := make(map[common.Address][]byte, len(update.accountsOrigin)) - for key, val := range update.accountsOrigin { - accounts[key] = common.CopyBytes(val) - } - accountList = append(accountList, accounts) - - storages := make(map[common.Address]map[common.Hash][]byte, len(update.storagesOrigin)) - for addr, subset := range update.storagesOrigin { - storages[addr] = make(map[common.Hash][]byte, len(subset)) - for key, val := range subset { - storages[addr][key] = common.CopyBytes(val) - } - } - storageList = append(storageList, storages) + roots []common.Hash + accounts []map[common.Hash][]byte + accountOrigin []map[common.Address][]byte + storages []map[common.Hash]map[common.Hash][]byte + storageOrigin []map[common.Address]map[common.Hash][]byte + copyUpdate = func(update *stateUpdate) { + accounts = append(accounts, maps.Clone(update.accounts)) + accountOrigin = append(accountOrigin, maps.Clone(update.accountsOrigin)) + storages = append(storages, maps.Clone(update.storages)) + storageOrigin = append(storageOrigin, maps.Clone(update.storagesOrigin)) } disk = rawdb.NewMemoryDatabase() tdb = triedb.NewDatabase(disk, &triedb.Config{PathDB: pathdb.Defaults}) @@ -250,7 +243,7 @@ func (test *stateTest) run() bool { if i != 0 { root = roots[i-1] } - test.err = test.verify(root, roots[i], tdb, accountList[i], storageList[i]) + test.err = test.verify(root, roots[i], tdb, accounts[i], accountOrigin[i], storages[i], storageOrigin[i]) if test.err != nil { return false } @@ -265,7 +258,7 @@ func (test *stateTest) run() bool { // - the account was indeed not present in trie // - the account is present in new trie, nil->nil is regarded as invalid // - the slots transition is correct -func (test *stateTest) verifyAccountCreation(next common.Hash, db *triedb.Database, otr, ntr *trie.Trie, addr common.Address, slots map[common.Hash][]byte) error { +func (test *stateTest) verifyAccountCreation(next common.Hash, db *triedb.Database, otr, ntr *trie.Trie, addr common.Address, account []byte, storages map[common.Hash][]byte, storagesOrigin map[common.Hash][]byte) error { // Verify account change addrHash := crypto.Keccak256Hash(addr.Bytes()) oBlob, err := otr.Get(addrHash.Bytes()) @@ -282,6 +275,13 @@ func (test *stateTest) verifyAccountCreation(next common.Hash, db *triedb.Databa if len(nBlob) == 0 { return fmt.Errorf("missing account in new trie, %x", addrHash) } + full, err := types.FullAccountRLP(account) + if err != nil { + return err + } + if !bytes.Equal(nBlob, full) { + return fmt.Errorf("unexpected account data, want: %v, got: %v", full, nBlob) + } // Verify storage changes var nAcct types.StateAccount @@ -290,7 +290,10 @@ func (test *stateTest) verifyAccountCreation(next common.Hash, db *triedb.Databa } // Account has no slot, empty slot set is expected if nAcct.Root == types.EmptyRootHash { - if len(slots) != 0 { + if len(storagesOrigin) != 0 { + return fmt.Errorf("unexpected slot changes %x", addrHash) + } + if len(storages) != 0 { return fmt.Errorf("unexpected slot changes %x", addrHash) } return nil @@ -300,9 +303,22 @@ func (test *stateTest) verifyAccountCreation(next common.Hash, db *triedb.Databa if err != nil { return err } - for key, val := range slots { + for key, val := range storagesOrigin { + if _, exist := storages[key]; !exist { + return errors.New("storage data is not found") + } + got, err := st.Get(key.Bytes()) + if err != nil { + return err + } + if !bytes.Equal(got, storages[key]) { + return fmt.Errorf("unexpected storage data, want: %v, got: %v", storages[key], got) + } st.Update(key.Bytes(), val) } + if len(storagesOrigin) != len(storages) { + return fmt.Errorf("extra storage found, want: %d, got: %d", len(storagesOrigin), len(storages)) + } if st.Hash() != types.EmptyRootHash { return errors.New("invalid slot changes") } @@ -316,7 +332,7 @@ func (test *stateTest) verifyAccountCreation(next common.Hash, db *triedb.Databa // - the account was indeed present in trie // - the account in old trie matches the provided value // - the slots transition is correct -func (test *stateTest) verifyAccountUpdate(next common.Hash, db *triedb.Database, otr, ntr *trie.Trie, addr common.Address, origin []byte, slots map[common.Hash][]byte) error { +func (test *stateTest) verifyAccountUpdate(next common.Hash, db *triedb.Database, otr, ntr *trie.Trie, addr common.Address, account []byte, accountOrigin []byte, storages map[common.Hash][]byte, storageOrigin map[common.Hash][]byte) error { // Verify account change addrHash := crypto.Keccak256Hash(addr.Bytes()) oBlob, err := otr.Get(addrHash.Bytes()) @@ -330,14 +346,23 @@ func (test *stateTest) verifyAccountUpdate(next common.Hash, db *triedb.Database if len(oBlob) == 0 { return fmt.Errorf("missing account in old trie, %x", addrHash) } - full, err := types.FullAccountRLP(origin) + full, err := types.FullAccountRLP(accountOrigin) if err != nil { return err } if !bytes.Equal(full, oBlob) { return fmt.Errorf("account value is not matched, %x", addrHash) } - + if len(nBlob) == 0 { + if len(account) != 0 { + return errors.New("unexpected account data") + } + } else { + full, _ = types.FullAccountRLP(account) + if !bytes.Equal(full, nBlob) { + return fmt.Errorf("unexpected account data, %x, want %v, got: %v", addrHash, full, nBlob) + } + } // Decode accounts var ( oAcct types.StateAccount @@ -361,16 +386,29 @@ func (test *stateTest) verifyAccountUpdate(next common.Hash, db *triedb.Database if err != nil { return err } - for key, val := range slots { + for key, val := range storageOrigin { + if _, exist := storages[key]; !exist { + return errors.New("storage data is not found") + } + got, err := st.Get(key.Bytes()) + if err != nil { + return err + } + if !bytes.Equal(got, storages[key]) { + return fmt.Errorf("unexpected storage data, want: %v, got: %v", storages[key], got) + } st.Update(key.Bytes(), val) } + if len(storageOrigin) != len(storages) { + return fmt.Errorf("extra storage found, want: %d, got: %d", len(storageOrigin), len(storages)) + } if st.Hash() != oAcct.Root { return errors.New("invalid slot changes") } return nil } -func (test *stateTest) verify(root common.Hash, next common.Hash, db *triedb.Database, accountsOrigin map[common.Address][]byte, storagesOrigin map[common.Address]map[common.Hash][]byte) error { +func (test *stateTest) verify(root common.Hash, next common.Hash, db *triedb.Database, accounts map[common.Hash][]byte, accountsOrigin map[common.Address][]byte, storages map[common.Hash]map[common.Hash][]byte, storagesOrigin map[common.Address]map[common.Hash][]byte) error { otr, err := trie.New(trie.StateTrieID(root), db) if err != nil { return err @@ -379,12 +417,15 @@ func (test *stateTest) verify(root common.Hash, next common.Hash, db *triedb.Dat if err != nil { return err } - for addr, account := range accountsOrigin { - var err error - if len(account) == 0 { - err = test.verifyAccountCreation(next, db, otr, ntr, addr, storagesOrigin[addr]) + for addr, accountOrigin := range accountsOrigin { + var ( + err error + addrHash = crypto.Keccak256Hash(addr.Bytes()) + ) + if len(accountOrigin) == 0 { + err = test.verifyAccountCreation(next, db, otr, ntr, addr, accounts[addrHash], storages[addrHash], storagesOrigin[addr]) } else { - err = test.verifyAccountUpdate(next, db, otr, ntr, addr, accountsOrigin[addr], storagesOrigin[addr]) + err = test.verifyAccountUpdate(next, db, otr, ntr, addr, accounts[addrHash], accountsOrigin[addr], storages[addrHash], storagesOrigin[addr]) } if err != nil { return err diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 3647397df6..57886e6e03 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -1305,12 +1305,12 @@ func TestDeleteStorage(t *testing.T) { obj := fastState.getOrNewStateObject(addr) storageRoot := obj.data.Root - _, fastNodes, err := fastState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot) + _, _, fastNodes, err := fastState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot) if err != nil { t.Fatal(err) } - _, slowNodes, err := slowState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot) + _, _, slowNodes, err := slowState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot) if err != nil { t.Fatal(err) } diff --git a/core/state/stateupdate.go b/core/state/stateupdate.go index c9231f0526..a320b72f11 100644 --- a/core/state/stateupdate.go +++ b/core/state/stateupdate.go @@ -17,6 +17,8 @@ package state import ( + "maps" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/trie/trienode" @@ -33,6 +35,7 @@ type contractCode struct { type accountDelete struct { address common.Address // address is the unique account identifier origin []byte // origin is the original value of account data in slim-RLP encoding. + storages map[common.Hash][]byte // storages stores mutated slots, the value should be nil. storagesOrigin map[common.Hash][]byte // storagesOrigin stores the original values of mutated slots in prefix-zero-trimmed RLP format. } @@ -52,7 +55,6 @@ type accountUpdate struct { type stateUpdate struct { originRoot common.Hash // hash of the state before applying mutation root common.Hash // hash of the state after applying mutation - destructs map[common.Hash]struct{} // destructs contains the list of destructed accounts accounts map[common.Hash][]byte // accounts stores mutated accounts in 'slim RLP' encoding accountsOrigin map[common.Address][]byte // accountsOrigin stores the original values of mutated accounts in 'slim RLP' encoding storages map[common.Hash]map[common.Hash][]byte // storages stores mutated slots in 'prefix-zero-trimmed' RLP format @@ -71,7 +73,6 @@ func (sc *stateUpdate) empty() bool { // account deletions and account updates to form a comprehensive state update. func newStateUpdate(originRoot common.Hash, root common.Hash, deletes map[common.Hash]*accountDelete, updates map[common.Hash]*accountUpdate, nodes *trienode.MergedNodeSet) *stateUpdate { var ( - destructs = make(map[common.Hash]struct{}) accounts = make(map[common.Hash][]byte) accountsOrigin = make(map[common.Address][]byte) storages = make(map[common.Hash]map[common.Hash][]byte) @@ -82,8 +83,12 @@ func newStateUpdate(originRoot common.Hash, root common.Hash, deletes map[common // within the same block, the deletions must be aggregated first. for addrHash, op := range deletes { addr := op.address - destructs[addrHash] = struct{}{} + accounts[addrHash] = nil accountsOrigin[addr] = op.origin + + if len(op.storages) > 0 { + storages[addrHash] = op.storages + } if len(op.storagesOrigin) > 0 { storagesOrigin[addr] = op.storagesOrigin } @@ -95,35 +100,41 @@ func newStateUpdate(originRoot common.Hash, root common.Hash, deletes map[common if op.code != nil { codes[addr] = *op.code } - // Aggregate the account changes. The original account value will only - // be tracked if it's not present yet. accounts[addrHash] = op.data + + // Aggregate the account original value. If the account is already + // present in the aggregated accountsOrigin set, skip it. if _, found := accountsOrigin[addr]; !found { accountsOrigin[addr] = op.origin } - // Aggregate the storage changes. The original storage slot value will - // only be tracked if it's not present yet. + // Aggregate the storage mutation list. If a slot in op.storages is + // already present in aggregated storages set, the value will be + // overwritten. if len(op.storages) > 0 { - storages[addrHash] = op.storages - } - if len(op.storagesOrigin) > 0 { - origin := storagesOrigin[addr] - if origin == nil { - storagesOrigin[addr] = op.storagesOrigin - continue + if _, exist := storages[addrHash]; !exist { + storages[addrHash] = op.storages + } else { + maps.Copy(storages[addrHash], op.storages) } - for key, slot := range op.storagesOrigin { - if _, found := origin[key]; !found { - origin[key] = slot + } + // Aggregate the storage original values. If the slot is already present + // in aggregated storagesOrigin set, skip it. + if len(op.storagesOrigin) > 0 { + origin, exist := storagesOrigin[addr] + if !exist { + storagesOrigin[addr] = op.storagesOrigin + } else { + for key, slot := range op.storagesOrigin { + if _, found := origin[key]; !found { + origin[key] = slot + } } } - storagesOrigin[addr] = origin } } return &stateUpdate{ originRoot: types.TrieRootHash(originRoot), root: types.TrieRootHash(root), - destructs: destructs, accounts: accounts, accountsOrigin: accountsOrigin, storages: storages, @@ -139,7 +150,6 @@ func newStateUpdate(originRoot common.Hash, root common.Hash, deletes map[common // package. func (sc *stateUpdate) stateSet() *triedb.StateSet { return &triedb.StateSet{ - Destructs: sc.destructs, Accounts: sc.accounts, AccountsOrigin: sc.accountsOrigin, Storages: sc.storages, diff --git a/triedb/states.go b/triedb/states.go index 1f9a0de522..0b03f2b9f3 100644 --- a/triedb/states.go +++ b/triedb/states.go @@ -23,7 +23,6 @@ import ( // StateSet represents a collection of mutated states during a state transition. type StateSet struct { - Destructs map[common.Hash]struct{} // Destructed accounts Accounts map[common.Hash][]byte // Mutated accounts in 'slim RLP' encoding AccountsOrigin map[common.Address][]byte // Original values of mutated accounts in 'slim RLP' encoding Storages map[common.Hash]map[common.Hash][]byte // Mutated storage slots in 'prefix-zero-trimmed' RLP format @@ -33,7 +32,6 @@ type StateSet struct { // NewStateSet initializes an empty state set. func NewStateSet() *StateSet { return &StateSet{ - Destructs: make(map[common.Hash]struct{}), Accounts: make(map[common.Hash][]byte), AccountsOrigin: make(map[common.Address][]byte), Storages: make(map[common.Hash]map[common.Hash][]byte), From 5e1a39d67f642990b9b98f8391ceca39a5fdb849 Mon Sep 17 00:00:00 2001 From: Daniel Liu <139250065@qq.com> Date: Mon, 25 Nov 2024 03:09:38 +0800 Subject: [PATCH 15/21] internal/flags: fix "flag redefined" bug for alias on custom flags (#30796) This change fixes a bug on the `DirectoryFlag` and the `BigFlag`, which would trigger a `panic` with the message "flag redefined" in case an alias was added to such a flag. --- internal/flags/flags.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/flags/flags.go b/internal/flags/flags.go index 1f6be3d3d3..b858e73d04 100644 --- a/internal/flags/flags.go +++ b/internal/flags/flags.go @@ -90,7 +90,7 @@ func (f *DirectoryFlag) Apply(set *flag.FlagSet) error { } } eachName(f, func(name string) { - set.Var(&f.Value, f.Name, f.Usage) + set.Var(&f.Value, name, f.Usage) }) return nil } @@ -172,7 +172,7 @@ func (f *BigFlag) Apply(set *flag.FlagSet) error { } eachName(f, func(name string) { f.Value = new(big.Int) - set.Var((*bigValue)(f.Value), f.Name, f.Usage) + set.Var((*bigValue)(f.Value), name, f.Usage) }) return nil } From ab4a1cc01f3897b3bd9918c1f7cddc5730baefa9 Mon Sep 17 00:00:00 2001 From: Martin HS Date: Mon, 25 Nov 2024 10:07:50 +0100 Subject: [PATCH 16/21] eth/tracers/logger: fix json-logger output missing (#30804) Fixes a flaw introduced in https://github.com/ethereum/go-ethereum/pull/29795 , discovered while reviewing https://github.com/ethereum/go-ethereum/pull/30633 . --- eth/tracers/logger/logger_json.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/eth/tracers/logger/logger_json.go b/eth/tracers/logger/logger_json.go index 797f7ac658..52ac3945d4 100644 --- a/eth/tracers/logger/logger_json.go +++ b/eth/tracers/logger/logger_json.go @@ -71,7 +71,7 @@ func NewJSONLogger(cfg *Config, writer io.Writer) *tracing.Hooks { l.hooks = &tracing.Hooks{ OnTxStart: l.OnTxStart, OnSystemCallStart: l.onSystemCallStart, - OnExit: l.OnEnd, + OnExit: l.OnExit, OnOpcode: l.OnOpcode, OnFault: l.OnFault, } @@ -152,13 +152,6 @@ func (l *jsonLogger) OnEnter(depth int, typ byte, from common.Address, to common l.encoder.Encode(frame) } -func (l *jsonLogger) OnEnd(depth int, output []byte, gasUsed uint64, err error, reverted bool) { - if depth > 0 { - return - } - l.OnExit(depth, output, gasUsed, err, false) -} - func (l *jsonLogger) OnExit(depth int, output []byte, gasUsed uint64, err error, reverted bool) { type endLog struct { Output string `json:"output"` From 02159d553f8cc02baaa344edd8cc78f0306d6dbd Mon Sep 17 00:00:00 2001 From: Martin HS Date: Mon, 25 Nov 2024 13:29:27 +0100 Subject: [PATCH 17/21] eth/tracers/logger: improve markdown logger (#30805) This PR improves the output of the markdown logger a bit. - Remove `RStack` field, - Move `Stack` last, since it may have very large vertical expansion - Make the pre- and post-exec metadata structured into a bullet-list --- eth/tracers/logger/logger.go | 37 ++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/eth/tracers/logger/logger.go b/eth/tracers/logger/logger.go index f918ce154b..bdd4e07055 100644 --- a/eth/tracers/logger/logger.go +++ b/eth/tracers/logger/logger.go @@ -363,26 +363,35 @@ func (t *mdLogger) OnEnter(depth int, typ byte, from common.Address, to common.A if depth != 0 { return } - create := vm.OpCode(typ) == vm.CREATE - if !create { - fmt.Fprintf(t.out, "From: `%v`\nTo: `%v`\nData: `%#x`\nGas: `%d`\nValue `%v` wei\n", - from.String(), to.String(), - input, gas, value) + if create := vm.OpCode(typ) == vm.CREATE; !create { + fmt.Fprintf(t.out, "Pre-execution info:\n"+ + " - from: `%v`\n"+ + " - to: `%v`\n"+ + " - data: `%#x`\n"+ + " - gas: `%d`\n"+ + " - value: `%v` wei\n", + from.String(), to.String(), input, gas, value) } else { - fmt.Fprintf(t.out, "From: `%v`\nCreate at: `%v`\nData: `%#x`\nGas: `%d`\nValue `%v` wei\n", - from.String(), to.String(), - input, gas, value) + fmt.Fprintf(t.out, "Pre-execution info:\n"+ + " - from: `%v`\n"+ + " - create: `%v`\n"+ + " - data: `%#x`\n"+ + " - gas: `%d`\n"+ + " - value: `%v` wei\n", + from.String(), to.String(), input, gas, value) } - fmt.Fprintf(t.out, ` -| Pc | Op | Cost | Stack | RStack | Refund | -|-------|-------------|------|-----------|-----------|---------| +| Pc | Op | Cost | Refund | Stack | +|-------|-------------|------|-----------|-----------| `) } func (t *mdLogger) OnExit(depth int, output []byte, gasUsed uint64, err error, reverted bool) { if depth == 0 { - fmt.Fprintf(t.out, "\nOutput: `%#x`\nConsumed gas: `%d`\nError: `%v`\n", + fmt.Fprintf(t.out, "\nPost-execution info:\n"+ + " - output: `%#x`\n"+ + " - consumed gas: `%d`\n"+ + " - error: `%v`\n", output, gasUsed, err) } } @@ -390,7 +399,8 @@ func (t *mdLogger) OnExit(depth int, output []byte, gasUsed uint64, err error, r // OnOpcode also tracks SLOAD/SSTORE ops to track storage change. func (t *mdLogger) OnOpcode(pc uint64, op byte, gas, cost uint64, scope tracing.OpContext, rData []byte, depth int, err error) { stack := scope.StackData() - fmt.Fprintf(t.out, "| %4d | %10v | %3d |", pc, vm.OpCode(op).String(), cost) + fmt.Fprintf(t.out, "| %4d | %10v | %3d |%10v |", pc, vm.OpCode(op).String(), + cost, t.env.StateDB.GetRefund()) if !t.cfg.DisableStack { // format stack @@ -401,7 +411,6 @@ func (t *mdLogger) OnOpcode(pc uint64, op byte, gas, cost uint64, scope tracing. b := fmt.Sprintf("[%v]", strings.Join(a, ",")) fmt.Fprintf(t.out, "%10v |", b) } - fmt.Fprintf(t.out, "%10v |", t.env.StateDB.GetRefund()) fmt.Fprintln(t.out, "") if err != nil { fmt.Fprintf(t.out, "Error: %v\n", err) From 19fa71b917dfba9a2f7692aeef524bc541a710aa Mon Sep 17 00:00:00 2001 From: "Hyunsoo Shin (Lake)" Date: Mon, 25 Nov 2024 21:33:28 +0900 Subject: [PATCH 18/21] internal/ethapi: remove double map-clone (#30803) Similar to https://github.com/ethereum/go-ethereum/pull/30788 --- internal/ethapi/simulate.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/ethapi/simulate.go b/internal/ethapi/simulate.go index 0b58a27096..2161e1d5f4 100644 --- a/internal/ethapi/simulate.go +++ b/internal/ethapi/simulate.go @@ -21,7 +21,6 @@ import ( "encoding/json" "errors" "fmt" - "maps" "math/big" "time" @@ -186,7 +185,7 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, Tracer: tracer.Hooks(), } ) - var tracingStateDB = vm.StateDB(sim.state) + tracingStateDB := vm.StateDB(sim.state) if hooks := tracer.Hooks(); hooks != nil { tracingStateDB = state.NewHookedState(sim.state, hooks) } @@ -289,7 +288,7 @@ func (sim *simulator) activePrecompiles(base *types.Header) vm.PrecompiledContra isMerge = (base.Difficulty.Sign() == 0) rules = sim.chainConfig.Rules(base.Number, isMerge, base.Time) ) - return maps.Clone(vm.ActivePrecompiledContracts(rules)) + return vm.ActivePrecompiledContracts(rules) } // sanitizeChain checks the chain integrity. Specifically it checks that From 3c754e2a09d112cd62f4d51f221637f9a147dcc6 Mon Sep 17 00:00:00 2001 From: Jordan Krage Date: Mon, 25 Nov 2024 06:34:50 -0600 Subject: [PATCH 19/21] accounts/abi: fix MakeTopics mutation of big.Int inputs (#30785) #28764 updated `func MakeTopics` to support negative `*big.Int`s. However, it also changed the behavior of the function from just _reading_ the input `*big.Int` via `Bytes()`, to leveraging `big.U256Bytes` which is documented as being _destructive_: This change updates `MakeTopics` to not mutate the original, and also applies the same change in signer/core/apitypes. --- accounts/abi/topics.go | 2 +- accounts/abi/topics_test.go | 17 +++++++++++++++++ signer/core/apitypes/types.go | 2 +- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/accounts/abi/topics.go b/accounts/abi/topics.go index 7ce9b7273c..4819334ae6 100644 --- a/accounts/abi/topics.go +++ b/accounts/abi/topics.go @@ -42,7 +42,7 @@ func MakeTopics(query ...[]interface{}) ([][]common.Hash, error) { case common.Address: copy(topic[common.HashLength-common.AddressLength:], rule[:]) case *big.Int: - copy(topic[:], math.U256Bytes(rule)) + copy(topic[:], math.U256Bytes(new(big.Int).Set(rule))) case bool: if rule { topic[common.HashLength-1] = 1 diff --git a/accounts/abi/topics_test.go b/accounts/abi/topics_test.go index 6a4c50078a..161867e2d9 100644 --- a/accounts/abi/topics_test.go +++ b/accounts/abi/topics_test.go @@ -149,6 +149,23 @@ func TestMakeTopics(t *testing.T) { } }) } + + t.Run("does not mutate big.Int", func(t *testing.T) { + t.Parallel() + want := [][]common.Hash{{common.HexToHash("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")}} + + in := big.NewInt(-1) + got, err := MakeTopics([]interface{}{in}) + if err != nil { + t.Fatalf("makeTopics() error = %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("makeTopics() = %v, want %v", got, want) + } + if orig := big.NewInt(-1); in.Cmp(orig) != 0 { + t.Fatalf("makeTopics() mutated an input parameter from %v to %v", orig, in) + } + }) } type args struct { diff --git a/signer/core/apitypes/types.go b/signer/core/apitypes/types.go index 2ae182279a..b56931c1d1 100644 --- a/signer/core/apitypes/types.go +++ b/signer/core/apitypes/types.go @@ -676,7 +676,7 @@ func (typedData *TypedData) EncodePrimitiveValue(encType string, encValue interf if err != nil { return nil, err } - return math.U256Bytes(b), nil + return math.U256Bytes(new(big.Int).Set(b)), nil } return nil, fmt.Errorf("unrecognized type '%s'", encType) } From 23800122b37695be50565f8221858a16ce1763db Mon Sep 17 00:00:00 2001 From: Arran Schlosberg <519948+ARR4N@users.noreply.github.com> Date: Mon, 25 Nov 2024 12:43:23 +0000 Subject: [PATCH 20/21] core/state/snapshot: simplify snapshot rebuild (#30772) This PR is purely for improved readability; I was doing work involving the file and think this may help others who are trying to understand what's going on. 1. `snapshot.Tree.Rebuild()` now returns a function that blocks until regeneration is complete, allowing `Tree.waitBuild()` to be removed entirely as all it did was search for the `done` channel behind this new function. 2. Its usage inside `New()` is also simplified by (a) only waiting if `!AsyncBuild`; and (b) avoiding the double negative of `if !NoBuild`. --------- Co-authored-by: Martin HS --- core/state/snapshot/snapshot.go | 48 ++++++++++----------------------- 1 file changed, 14 insertions(+), 34 deletions(-) diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 7466f12351..181dc6b746 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -206,47 +206,24 @@ func New(config Config, diskdb ethdb.KeyValueStore, triedb *triedb.Database, roo log.Warn("Snapshot maintenance disabled (syncing)") return snap, nil } - // Create the building waiter iff the background generation is allowed - if !config.NoBuild && !config.AsyncBuild { - defer snap.waitBuild() - } if err != nil { log.Warn("Failed to load snapshot", "err", err) - if !config.NoBuild { - snap.Rebuild(root) - return snap, nil + if config.NoBuild { + return nil, err } - return nil, err // Bail out the error, don't rebuild automatically. + wait := snap.Rebuild(root) + if !config.AsyncBuild { + wait() + } + return snap, nil } // Existing snapshot loaded, seed all the layers - for head != nil { + for ; head != nil; head = head.Parent() { snap.layers[head.Root()] = head - head = head.Parent() } return snap, nil } -// waitBuild blocks until the snapshot finishes rebuilding. This method is meant -// to be used by tests to ensure we're testing what we believe we are. -func (t *Tree) waitBuild() { - // Find the rebuild termination channel - var done chan struct{} - - t.lock.RLock() - for _, layer := range t.layers { - if layer, ok := layer.(*diskLayer); ok { - done = layer.genPending - break - } - } - t.lock.RUnlock() - - // Wait until the snapshot is generated - if done != nil { - <-done - } -} - // Disable interrupts any pending snapshot generator, deletes all the snapshot // layers in memory and marks snapshots disabled globally. In order to resume // the snapshot functionality, the caller must invoke Rebuild. @@ -688,8 +665,9 @@ func (t *Tree) Journal(root common.Hash) (common.Hash, error) { // Rebuild wipes all available snapshot data from the persistent database and // discard all caches and diff layers. Afterwards, it starts a new snapshot -// generator with the given root hash. -func (t *Tree) Rebuild(root common.Hash) { +// generator with the given root hash. The returned function blocks until +// regeneration is complete. +func (t *Tree) Rebuild(root common.Hash) (wait func()) { t.lock.Lock() defer t.lock.Unlock() @@ -721,9 +699,11 @@ func (t *Tree) Rebuild(root common.Hash) { // Start generating a new snapshot from scratch on a background thread. The // generator will run a wiper first if there's not one running right now. log.Info("Rebuilding state snapshot") + disk := generateSnapshot(t.diskdb, t.triedb, t.config.CacheSize, root) t.layers = map[common.Hash]snapshot{ - root: generateSnapshot(t.diskdb, t.triedb, t.config.CacheSize, root), + root: disk, } + return func() { <-disk.genPending } } // AccountIterator creates a new account iterator for the specified root hash and From b4d99e39176f89bb0ffbb32635e9cf17b5fd7367 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 26 Nov 2024 02:49:12 +0100 Subject: [PATCH 21/21] eth/ethconfig: improve error message if TTD missing (#30807) This updates the message you get when trying to initialize Geth with genesis.json that doesn't have `terminalTotalDifficulty`. The previous message was a bit obscure, I had to check the code to find out what the problem was. --- eth/ethconfig/config.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index f1a815e6d3..8542bc97c4 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/params" ) @@ -160,7 +161,8 @@ type Config struct { // only exist on already merged networks. func CreateConsensusEngine(config *params.ChainConfig, db ethdb.Database) (consensus.Engine, error) { if config.TerminalTotalDifficulty == nil { - return nil, fmt.Errorf("only PoS networks are supported, please transition old ones with Geth v1.13.x") + log.Error("Geth only supports PoS networks. Please transition legacy networks using Geth v1.13.x.") + return nil, fmt.Errorf("'terminalTotalDifficulty' is not set in genesis block") } // Wrap previously supported consensus engines into their post-merge counterpart if config.Clique != nil {