From 3d013c193942aad9820361795cf3a17bb99470b1 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Thu, 22 Mar 2018 15:48:52 +0100 Subject: [PATCH 01/19] whisper: some components are still using v5, switch to v6 --- cmd/geth/config.go | 2 +- cmd/utils/flags.go | 2 +- mobile/geth.go | 2 +- whisper/shhclient/client.go | 20 ++++++++++---------- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 50e4de2e78..e6bd4d5bef 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -32,7 +32,7 @@ import ( "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" - whisper "github.com/ethereum/go-ethereum/whisper/whisperv5" + whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" "github.com/naoina/toml" ) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index ff78a0fcc7..4f3d81f5d2 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -55,7 +55,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/nat" "github.com/ethereum/go-ethereum/p2p/netutil" "github.com/ethereum/go-ethereum/params" - whisper "github.com/ethereum/go-ethereum/whisper/whisperv5" + whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" "gopkg.in/urfave/cli.v1" ) diff --git a/mobile/geth.go b/mobile/geth.go index 7e3b8f4915..488a4150fc 100644 --- a/mobile/geth.go +++ b/mobile/geth.go @@ -34,7 +34,7 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/nat" "github.com/ethereum/go-ethereum/params" - whisper "github.com/ethereum/go-ethereum/whisper/whisperv5" + whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" ) // NodeConfig represents the collection of configuration values to fine tune the Geth diff --git a/whisper/shhclient/client.go b/whisper/shhclient/client.go index 61b4775d95..bbe694baaa 100644 --- a/whisper/shhclient/client.go +++ b/whisper/shhclient/client.go @@ -22,10 +22,10 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/rpc" - whisper "github.com/ethereum/go-ethereum/whisper/whisperv5" + whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" ) -// Client defines typed wrappers for the Whisper v5 RPC API. +// Client defines typed wrappers for the Whisper v6 RPC API. type Client struct { c *rpc.Client } @@ -168,27 +168,27 @@ func (sc *Client) Post(ctx context.Context, message whisper.NewMessage) error { // SubscribeMessages subscribes to messages that match the given criteria. This method // is only supported on bi-directional connections such as websockets and IPC. // NewMessageFilter uses polling and is supported over HTTP. -func (ec *Client) SubscribeMessages(ctx context.Context, criteria whisper.Criteria, ch chan<- *whisper.Message) (ethereum.Subscription, error) { - return ec.c.ShhSubscribe(ctx, ch, "messages", criteria) +func (sc *Client) SubscribeMessages(ctx context.Context, criteria whisper.Criteria, ch chan<- *whisper.Message) (ethereum.Subscription, error) { + return sc.c.ShhSubscribe(ctx, ch, "messages", criteria) } // NewMessageFilter creates a filter within the node. This filter can be used to poll // for new messages (see FilterMessages) that satisfy the given criteria. A filter can // timeout when it was polled for in whisper.filterTimeout. -func (ec *Client) NewMessageFilter(ctx context.Context, criteria whisper.Criteria) (string, error) { +func (sc *Client) NewMessageFilter(ctx context.Context, criteria whisper.Criteria) (string, error) { var id string - return id, ec.c.CallContext(ctx, &id, "shh_newMessageFilter", criteria) + return id, sc.c.CallContext(ctx, &id, "shh_newMessageFilter", criteria) } // DeleteMessageFilter removes the filter associated with the given id. -func (ec *Client) DeleteMessageFilter(ctx context.Context, id string) error { +func (sc *Client) DeleteMessageFilter(ctx context.Context, id string) error { var ignored bool - return ec.c.CallContext(ctx, &ignored, "shh_deleteMessageFilter", id) + return sc.c.CallContext(ctx, &ignored, "shh_deleteMessageFilter", id) } // FilterMessages retrieves all messages that are received between the last call to // this function and match the criteria that where given when the filter was created. -func (ec *Client) FilterMessages(ctx context.Context, id string) ([]*whisper.Message, error) { +func (sc *Client) FilterMessages(ctx context.Context, id string) ([]*whisper.Message, error) { var messages []*whisper.Message - return messages, ec.c.CallContext(ctx, &messages, "shh_getFilterMessages", id) + return messages, sc.c.CallContext(ctx, &messages, "shh_getFilterMessages", id) } From 1fae50a199903d28dac76e78ef065ba0ad96cf17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 26 Mar 2018 12:28:46 +0300 Subject: [PATCH 02/19] core: minor evm polishes and optimizations --- core/evm.go | 21 ++++- core/headerchain.go | 2 +- core/vm/instructions.go | 173 ++++++++++++++++++---------------------- core/vm/intpool.go | 16 +++- 4 files changed, 109 insertions(+), 103 deletions(-) diff --git a/core/evm.go b/core/evm.go index 55db53927b..596ea95fb7 100644 --- a/core/evm.go +++ b/core/evm.go @@ -60,13 +60,26 @@ func NewEVMContext(msg Message, header *types.Header, chain ChainContext, author // GetHashFn returns a GetHashFunc which retrieves header hashes by number func GetHashFn(ref *types.Header, chain ChainContext) func(n uint64) common.Hash { + var cache map[uint64]common.Hash + return func(n uint64) common.Hash { - for header := chain.GetHeader(ref.ParentHash, ref.Number.Uint64()-1); header != nil; header = chain.GetHeader(header.ParentHash, header.Number.Uint64()-1) { - if header.Number.Uint64() == n { - return header.Hash() + // If there's no hash cache yet, make one + if cache == nil { + cache = map[uint64]common.Hash{ + ref.Number.Uint64() - 1: ref.ParentHash, + } + } + // Try to fulfill the request from the cache + if hash, ok := cache[n]; ok { + return hash + } + // Not cached, iterate the blocks and cache the hashes + for header := chain.GetHeader(ref.ParentHash, ref.Number.Uint64()-1); header != nil; header = chain.GetHeader(header.ParentHash, header.Number.Uint64()-1) { + cache[header.Number.Uint64()-1] = header.ParentHash + if n == header.Number.Uint64()-1 { + return header.ParentHash } } - return common.Hash{} } } diff --git a/core/headerchain.go b/core/headerchain.go index 73cd5d2c41..2d1b0a2a18 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -23,6 +23,7 @@ import ( "math" "math/big" mrand "math/rand" + "sync/atomic" "time" "github.com/ethereum/go-ethereum/common" @@ -32,7 +33,6 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/hashicorp/golang-lru" - "sync/atomic" ) const ( diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 66e804fb7e..1e494a0eb8 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -39,20 +39,18 @@ var ( ) func opAdd(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - x, y := stack.pop(), stack.pop() - stack.push(math.U256(x.Add(x, y))) - - evm.interpreter.intPool.put(y) + x, y := stack.pop(), stack.peek() + math.U256(y.Add(x, y)) + evm.interpreter.intPool.put(x) return nil, nil } func opSub(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - x, y := stack.pop(), stack.pop() - stack.push(math.U256(x.Sub(x, y))) - - evm.interpreter.intPool.put(y) + x, y := stack.pop(), stack.peek() + math.U256(y.Sub(x, y)) + evm.interpreter.intPool.put(x) return nil, nil } @@ -66,44 +64,39 @@ func opMul(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stac } func opDiv(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - x, y := stack.pop(), stack.pop() + x, y := stack.pop(), stack.peek() if y.Sign() != 0 { - stack.push(math.U256(x.Div(x, y))) + math.U256(y.Div(x, y)) } else { - stack.push(new(big.Int)) + y.SetUint64(0) } - - evm.interpreter.intPool.put(y) - + evm.interpreter.intPool.put(x) return nil, nil } func opSdiv(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { x, y := math.S256(stack.pop()), math.S256(stack.pop()) - if y.Sign() == 0 { - stack.push(new(big.Int)) - return nil, nil + res := evm.interpreter.intPool.getZero() + + if y.Sign() == 0 || x.Sign() == 0 { + stack.push(res) } else { - n := new(big.Int) - if evm.interpreter.intPool.get().Mul(x, y).Sign() < 0 { - n.SetInt64(-1) + if x.Sign() != y.Sign() { + res.Div(x.Abs(x), y.Abs(y)) + res.Neg(res) } else { - n.SetInt64(1) + res.Div(x.Abs(x), y.Abs(y)) } - - res := x.Div(x.Abs(x), y.Abs(y)) - res.Mul(res, n) - stack.push(math.U256(res)) } - evm.interpreter.intPool.put(y) + evm.interpreter.intPool.put(x, y) return nil, nil } func opMod(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { x, y := stack.pop(), stack.pop() if y.Sign() == 0 { - stack.push(new(big.Int)) + stack.push(x.SetUint64(0)) } else { stack.push(math.U256(x.Mod(x, y))) } @@ -113,23 +106,20 @@ func opMod(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stac func opSmod(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { x, y := math.S256(stack.pop()), math.S256(stack.pop()) + res := evm.interpreter.intPool.getZero() if y.Sign() == 0 { - stack.push(new(big.Int)) + stack.push(res) } else { - n := new(big.Int) if x.Sign() < 0 { - n.SetInt64(-1) + res.Mod(x.Abs(x), y.Abs(y)) + res.Neg(res) } else { - n.SetInt64(1) + res.Mod(x.Abs(x), y.Abs(y)) } - - res := x.Mod(x.Abs(x), y.Abs(y)) - res.Mul(res, n) - stack.push(math.U256(res)) } - evm.interpreter.intPool.put(y) + evm.interpreter.intPool.put(x, y) return nil, nil } @@ -163,32 +153,30 @@ func opSignExtend(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stac } func opNot(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - x := stack.pop() - stack.push(math.U256(x.Not(x))) + x := stack.peek() + math.U256(x.Not(x)) return nil, nil } func opLt(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - x, y := stack.pop(), stack.pop() + x, y := stack.pop(), stack.peek() if x.Cmp(y) < 0 { - stack.push(evm.interpreter.intPool.get().SetUint64(1)) + y.SetUint64(1) } else { - stack.push(new(big.Int)) + y.SetUint64(0) } - - evm.interpreter.intPool.put(x, y) + evm.interpreter.intPool.put(x) return nil, nil } func opGt(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - x, y := stack.pop(), stack.pop() + x, y := stack.pop(), stack.peek() if x.Cmp(y) > 0 { - stack.push(evm.interpreter.intPool.get().SetUint64(1)) + y.SetUint64(1) } else { - stack.push(new(big.Int)) + y.SetUint64(0) } - - evm.interpreter.intPool.put(x, y) + evm.interpreter.intPool.put(x) return nil, nil } @@ -270,18 +258,18 @@ func opAnd(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stac } func opOr(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - x, y := stack.pop(), stack.pop() - stack.push(x.Or(x, y)) + x, y := stack.pop(), stack.peek() + y.Or(x, y) - evm.interpreter.intPool.put(y) + evm.interpreter.intPool.put(x) return nil, nil } func opXor(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - x, y := stack.pop(), stack.pop() - stack.push(x.Xor(x, y)) + x, y := stack.pop(), stack.peek() + y.Xor(x, y) - evm.interpreter.intPool.put(y) + evm.interpreter.intPool.put(x) return nil, nil } @@ -300,13 +288,12 @@ func opByte(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Sta func opAddmod(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { x, y, z := stack.pop(), stack.pop(), stack.pop() if z.Cmp(bigZero) > 0 { - add := x.Add(x, y) - add.Mod(add, z) - stack.push(math.U256(add)) + x.Add(x, y) + x.Mod(x, z) + stack.push(math.U256(x)) } else { - stack.push(new(big.Int)) + stack.push(x.SetUint64(0)) } - evm.interpreter.intPool.put(y, z) return nil, nil } @@ -314,13 +301,12 @@ func opAddmod(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *S func opMulmod(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { x, y, z := stack.pop(), stack.pop(), stack.pop() if z.Cmp(bigZero) > 0 { - mul := x.Mul(x, y) - mul.Mod(mul, z) - stack.push(math.U256(mul)) + x.Mul(x, y) + x.Mod(x, z) + stack.push(math.U256(x)) } else { - stack.push(new(big.Int)) + stack.push(x.SetUint64(0)) } - evm.interpreter.intPool.put(y, z) return nil, nil } @@ -393,8 +379,7 @@ func opSha3(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Sta if evm.vmConfig.EnablePreimageRecording { evm.StateDB.AddPreimage(common.BytesToHash(hash), data) } - - stack.push(new(big.Int).SetBytes(hash)) + stack.push(evm.interpreter.intPool.get().SetBytes(hash)) evm.interpreter.intPool.put(offset, size) return nil, nil @@ -406,10 +391,8 @@ func opAddress(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack * } func opBalance(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - addr := common.BigToAddress(stack.pop()) - balance := evm.StateDB.GetBalance(addr) - - stack.push(new(big.Int).Set(balance)) + slot := stack.peek() + slot.Set(evm.StateDB.GetBalance(common.BigToAddress(slot))) return nil, nil } @@ -429,7 +412,7 @@ func opCallValue(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack } func opCallDataLoad(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - stack.push(new(big.Int).SetBytes(getDataBig(contract.Input, stack.pop(), big32))) + stack.push(evm.interpreter.intPool.get().SetBytes(getDataBig(contract.Input, stack.pop(), big32))) return nil, nil } @@ -460,10 +443,11 @@ func opReturnDataCopy(pc *uint64, evm *EVM, contract *Contract, memory *Memory, memOffset = stack.pop() dataOffset = stack.pop() length = stack.pop() - ) - defer evm.interpreter.intPool.put(memOffset, dataOffset, length) - end := new(big.Int).Add(dataOffset, length) + end = evm.interpreter.intPool.get().Add(dataOffset, length) + ) + defer evm.interpreter.intPool.put(memOffset, dataOffset, length, end) + if end.BitLen() > 64 || uint64(len(evm.interpreter.returnData)) < end.Uint64() { return nil, errReturnDataOutOfBounds } @@ -473,11 +457,8 @@ func opReturnDataCopy(pc *uint64, evm *EVM, contract *Contract, memory *Memory, } func opExtCodeSize(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - a := stack.pop() - - addr := common.BigToAddress(a) - a.SetInt64(int64(evm.StateDB.GetCodeSize(addr))) - stack.push(a) + slot := stack.peek() + slot.SetUint64(uint64(evm.StateDB.GetCodeSize(common.BigToAddress(slot)))) return nil, nil } @@ -485,6 +466,7 @@ func opExtCodeSize(pc *uint64, evm *EVM, contract *Contract, memory *Memory, sta func opCodeSize(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { l := evm.interpreter.intPool.get().SetInt64(int64(len(contract.Code))) stack.push(l) + return nil, nil } @@ -527,9 +509,8 @@ func opBlockhash(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack if num.Cmp(n) > 0 && num.Cmp(evm.BlockNumber) < 0 { stack.push(evm.GetHash(num.Uint64()).Big()) } else { - stack.push(new(big.Int)) + stack.push(evm.interpreter.intPool.getZero()) } - evm.interpreter.intPool.put(num, n) return nil, nil } @@ -540,22 +521,22 @@ func opCoinbase(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack } func opTimestamp(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - stack.push(math.U256(new(big.Int).Set(evm.Time))) + stack.push(math.U256(evm.interpreter.intPool.get().Set(evm.Time))) return nil, nil } func opNumber(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - stack.push(math.U256(new(big.Int).Set(evm.BlockNumber))) + stack.push(math.U256(evm.interpreter.intPool.get().Set(evm.BlockNumber))) return nil, nil } func opDifficulty(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - stack.push(math.U256(new(big.Int).Set(evm.Difficulty))) + stack.push(math.U256(evm.interpreter.intPool.get().Set(evm.Difficulty))) return nil, nil } func opGasLimit(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - stack.push(math.U256(new(big.Int).SetUint64(evm.GasLimit))) + stack.push(math.U256(evm.interpreter.intPool.get().SetUint64(evm.GasLimit))) return nil, nil } @@ -566,7 +547,7 @@ func opPop(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stac func opMload(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { offset := stack.pop() - val := new(big.Int).SetBytes(memory.Get(offset.Int64(), 32)) + val := evm.interpreter.intPool.get().SetBytes(memory.Get(offset.Int64(), 32)) stack.push(val) evm.interpreter.intPool.put(offset) @@ -670,9 +651,9 @@ func opCreate(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *S // rule) and treat as an error, if the ruleset is frontier we must // ignore this error and pretend the operation was successful. if evm.ChainConfig().IsHomestead(evm.BlockNumber) && suberr == ErrCodeStoreOutOfGas { - stack.push(new(big.Int)) + stack.push(evm.interpreter.intPool.getZero()) } else if suberr != nil && suberr != ErrCodeStoreOutOfGas { - stack.push(new(big.Int)) + stack.push(evm.interpreter.intPool.getZero()) } else { stack.push(addr.Big()) } @@ -701,9 +682,9 @@ func opCall(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Sta } ret, returnGas, err := evm.Call(contract, toAddr, args, gas, value) if err != nil { - stack.push(new(big.Int)) + stack.push(evm.interpreter.intPool.getZero()) } else { - stack.push(big.NewInt(1)) + stack.push(evm.interpreter.intPool.get().SetUint64(1)) } if err == nil || err == errExecutionReverted { memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) @@ -730,9 +711,9 @@ func opCallCode(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack } ret, returnGas, err := evm.CallCode(contract, toAddr, args, gas, value) if err != nil { - stack.push(new(big.Int)) + stack.push(evm.interpreter.intPool.getZero()) } else { - stack.push(big.NewInt(1)) + stack.push(evm.interpreter.intPool.get().SetUint64(1)) } if err == nil || err == errExecutionReverted { memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) @@ -755,9 +736,9 @@ func opDelegateCall(pc *uint64, evm *EVM, contract *Contract, memory *Memory, st ret, returnGas, err := evm.DelegateCall(contract, toAddr, args, gas) if err != nil { - stack.push(new(big.Int)) + stack.push(evm.interpreter.intPool.getZero()) } else { - stack.push(big.NewInt(1)) + stack.push(evm.interpreter.intPool.get().SetUint64(1)) } if err == nil || err == errExecutionReverted { memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) @@ -780,9 +761,9 @@ func opStaticCall(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stac ret, returnGas, err := evm.StaticCall(contract, toAddr, args, gas) if err != nil { - stack.push(new(big.Int)) + stack.push(evm.interpreter.intPool.getZero()) } else { - stack.push(big.NewInt(1)) + stack.push(evm.interpreter.intPool.get().SetUint64(1)) } if err == nil || err == errExecutionReverted { memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) diff --git a/core/vm/intpool.go b/core/vm/intpool.go index 384f5df59b..5dbda18eee 100644 --- a/core/vm/intpool.go +++ b/core/vm/intpool.go @@ -32,24 +32,36 @@ func newIntPool() *intPool { return &intPool{pool: newstack()} } +// get retrieves a big int from the pool, allocating one if the pool is empty. +// Note, the returned int's value is arbitrary and will not be zeroed! func (p *intPool) get() *big.Int { if p.pool.len() > 0 { return p.pool.pop() } return new(big.Int) } + +// getZero retrieves a big int from the pool, setting it to zero or allocating +// a new one if the pool is empty. +func (p *intPool) getZero() *big.Int { + if p.pool.len() > 0 { + return p.pool.pop().SetUint64(0) + } + return new(big.Int) +} + +// put returns an allocated big int to the pool to be later reused by get calls. +// Note, the values as saved as is; neither put nor get zeroes the ints out! func (p *intPool) put(is ...*big.Int) { if len(p.pool.data) > poolLimit { return } - for _, i := range is { // verifyPool is a build flag. Pool verification makes sure the integrity // of the integer pool by comparing values to a default value. if verifyPool { i.Set(checkVal) } - p.pool.push(i) } } From b6b6f52ec8608e1a694357357c3f1fde669f1e6d Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Wed, 14 Mar 2018 20:15:30 +0800 Subject: [PATCH 03/19] cmd: implement preimage dump and import cmds --- cmd/geth/chaincmd.go | 113 +++++++++++++++++++++++++++++++++++++++++++ cmd/geth/main.go | 2 + ethdb/database.go | 6 +++ 3 files changed, 121 insertions(+) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index c9ab72b6df..692cc2d8d1 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -40,6 +40,11 @@ import ( "gopkg.in/urfave/cli.v1" ) +var ( + // secureKeyPrefix is the database key prefix used to store trie node preimages. + secureKeyPrefix = []byte("secure-key-") +) + var ( initCommand = cli.Command{ Action: utils.MigrateFlags(initGenesis), @@ -141,6 +146,34 @@ Remove blockchain and state databases`, The arguments are interpreted as block numbers or hashes. Use "ethereum dump 0" to dump the genesis block.`, } + preimageDumpCommand = cli.Command{ + Action: utils.MigrateFlags(dumpPreimage), + Name: "preimagedump", + Usage: "Dump the preimage database in json format", + ArgsUsage: "", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.CacheFlag, + utils.LightModeFlag, + }, + Category: "BLOCKCHAIN COMMANDS", + Description: ` +Dump the preimage database in json format`, + } + preimageImportCommand = cli.Command{ + Action: utils.MigrateFlags(importPreimage), + Name: "preimageimport", + Usage: "Import the preimage data from the specified file", + ArgsUsage: "", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.CacheFlag, + utils.LightModeFlag, + }, + Category: "BLOCKCHAIN COMMANDS", + Description: ` +Import the preimage data from the specified file`, + } ) // initGenesis will initialise the given JSON format genesis file and writes it as @@ -406,6 +439,86 @@ func dump(ctx *cli.Context) error { return nil } +// PreimageEntry represents a map between preimage and hash. +type PreimageEntry struct { + Hash string `json:"hash"` + Preimage string `json:"preimage"` +} + +// dumpPreimage dumps the preimage data to specified json file in streaming way. +func dumpPreimage(ctx *cli.Context) error { + // Make sure the export json file has been specified. + if len(ctx.Args()) < 1 { + utils.Fatalf("This command requires an argument.") + } + + // Encode preimage data to json file in streaming way. + file, err := os.Create(ctx.Args().First()) + if err != nil { + return err + } + encoder := json.NewEncoder(file) + + stack := makeFullNode(ctx) + db := utils.MakeChainDatabase(ctx, stack) + + // Dump all preimage entries. + it := db.(*ethdb.LDBDatabase).NewIteratorByPrefix(secureKeyPrefix) + for it.Next() { + hash := it.Key()[len(secureKeyPrefix):] + if err := encoder.Encode(PreimageEntry{common.Bytes2Hex(hash), common.Bytes2Hex(it.Value())}); err != nil { + return err + } + } + return nil +} + +// importPreimages imports preimage data from the specified file. +func importPreimage(ctx *cli.Context) error { + // Make sure the export json file has been specified. + if len(ctx.Args()) < 1 { + utils.Fatalf("This command requires an argument.") + } + + // Decode the preimage data in streaming way. + file, err := os.Open(ctx.Args().First()) + if err != nil { + return err + } + decoder := json.NewDecoder(file) + + stack := makeFullNode(ctx) + db := utils.MakeChainDatabase(ctx, stack) + + var ( + entry PreimageEntry + preimages = make(map[common.Hash][]byte) + ) + + for decoder.More() { + if err := decoder.Decode(&entry); err != nil { + return err + } + preimages[common.HexToHash(entry.Hash)] = common.Hex2Bytes(entry.Preimage) + // Flush to database in batch + if len(preimages) > 1024 { + err := core.WritePreimages(db, 0, preimages) + if err != nil { + return err + } + preimages = make(map[common.Hash][]byte) + } + } + // Flush the last batch preimage data + if len(preimages) > 0 { + err := core.WritePreimages(db, 0, preimages) + if err != nil { + return err + } + } + return nil +} + // hashish returns true for strings that look like hashes. func hashish(x string) bool { _, err := strconv.Atoi(x) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index f5a3fa941a..6e234a704b 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -158,6 +158,8 @@ func init() { copydbCommand, removedbCommand, dumpCommand, + preimageDumpCommand, + preimageImportCommand, // See monitorcmd.go: monitorCommand, // See accountcmd.go: diff --git a/ethdb/database.go b/ethdb/database.go index 8c557e4820..d0256c56f4 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -29,6 +29,7 @@ import ( "github.com/syndtr/goleveldb/leveldb/filter" "github.com/syndtr/goleveldb/leveldb/iterator" "github.com/syndtr/goleveldb/leveldb/opt" + "github.com/syndtr/goleveldb/leveldb/util" ) var OpenFileLimit = 64 @@ -121,6 +122,11 @@ func (db *LDBDatabase) NewIterator() iterator.Iterator { return db.db.NewIterator(nil, nil) } +// NewIteratorByPrefix returns a iterator to iterate over subset of database content with a particular prefix. +func (db *LDBDatabase) NewIteratorByPrefix(prefix []byte) iterator.Iterator { + return db.db.NewIterator(util.BytesPrefix(prefix), nil) +} + func (db *LDBDatabase) Close() { // Stop the metrics collection to avoid internal database races db.quitLock.Lock() From 23ac78333201890deed23576668810e19df2c67e Mon Sep 17 00:00:00 2001 From: David Huie Date: Mon, 26 Mar 2018 03:46:18 -0700 Subject: [PATCH 04/19] ecies: drop randomness parameter from `PrivateKey.Decrypt` (#16374) The parameter `rand` is unused in `PrivateKey.Decrypt`. Decryption in the ECIES encryption scheme is deterministic, so randomness isn't needed. --- crypto/ecies/ecies.go | 6 +++--- crypto/ecies/ecies_test.go | 20 ++++++++++---------- p2p/rlpx.go | 4 ++-- whisper/whisperv5/message.go | 2 +- whisper/whisperv6/message.go | 2 +- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/crypto/ecies/ecies.go b/crypto/ecies/ecies.go index 2ed91c895d..1474181482 100644 --- a/crypto/ecies/ecies.go +++ b/crypto/ecies/ecies.go @@ -230,7 +230,7 @@ func symEncrypt(rand io.Reader, params *ECIESParams, key, m []byte) (ct []byte, // symDecrypt carries out CTR decryption using the block cipher specified in // the parameters -func symDecrypt(rand io.Reader, params *ECIESParams, key, ct []byte) (m []byte, err error) { +func symDecrypt(params *ECIESParams, key, ct []byte) (m []byte, err error) { c, err := params.Cipher(key) if err != nil { return @@ -292,7 +292,7 @@ func Encrypt(rand io.Reader, pub *PublicKey, m, s1, s2 []byte) (ct []byte, err e } // Decrypt decrypts an ECIES ciphertext. -func (prv *PrivateKey) Decrypt(rand io.Reader, c, s1, s2 []byte) (m []byte, err error) { +func (prv *PrivateKey) Decrypt(c, s1, s2 []byte) (m []byte, err error) { if len(c) == 0 { return nil, ErrInvalidMessage } @@ -361,6 +361,6 @@ func (prv *PrivateKey) Decrypt(rand io.Reader, c, s1, s2 []byte) (m []byte, err return } - m, err = symDecrypt(rand, params, Ke, c[mStart:mEnd]) + m, err = symDecrypt(params, Ke, c[mStart:mEnd]) return } diff --git a/crypto/ecies/ecies_test.go b/crypto/ecies/ecies_test.go index 9cd5c79f7e..f33f204d5b 100644 --- a/crypto/ecies/ecies_test.go +++ b/crypto/ecies/ecies_test.go @@ -270,7 +270,7 @@ func TestEncryptDecrypt(t *testing.T) { t.FailNow() } - pt, err := prv2.Decrypt(rand.Reader, ct, nil, nil) + pt, err := prv2.Decrypt(ct, nil, nil) if err != nil { fmt.Println(err.Error()) t.FailNow() @@ -281,7 +281,7 @@ func TestEncryptDecrypt(t *testing.T) { t.FailNow() } - _, err = prv1.Decrypt(rand.Reader, ct, nil, nil) + _, err = prv1.Decrypt(ct, nil, nil) if err == nil { fmt.Println("ecies: encryption should not have succeeded") t.FailNow() @@ -301,7 +301,7 @@ func TestDecryptShared2(t *testing.T) { } // Check that decrypting with correct shared data works. - pt, err := prv.Decrypt(rand.Reader, ct, nil, shared2) + pt, err := prv.Decrypt(ct, nil, shared2) if err != nil { t.Fatal(err) } @@ -310,10 +310,10 @@ func TestDecryptShared2(t *testing.T) { } // Decrypting without shared data or incorrect shared data fails. - if _, err = prv.Decrypt(rand.Reader, ct, nil, nil); err == nil { + if _, err = prv.Decrypt(ct, nil, nil); err == nil { t.Fatal("ecies: decrypting without shared data didn't fail") } - if _, err = prv.Decrypt(rand.Reader, ct, nil, []byte("garbage")); err == nil { + if _, err = prv.Decrypt(ct, nil, []byte("garbage")); err == nil { t.Fatal("ecies: decrypting with incorrect shared data didn't fail") } } @@ -381,7 +381,7 @@ func testParamSelection(t *testing.T, c testCase) { t.FailNow() } - pt, err := prv2.Decrypt(rand.Reader, ct, nil, nil) + pt, err := prv2.Decrypt(ct, nil, nil) if err != nil { fmt.Printf("%s (%s)\n", err.Error(), c.Name) t.FailNow() @@ -393,7 +393,7 @@ func testParamSelection(t *testing.T, c testCase) { t.FailNow() } - _, err = prv1.Decrypt(rand.Reader, ct, nil, nil) + _, err = prv1.Decrypt(ct, nil, nil) if err == nil { fmt.Printf("ecies: encryption should not have succeeded (%s)\n", c.Name) @@ -422,7 +422,7 @@ func TestBasicKeyValidation(t *testing.T) { for _, b := range badBytes { ct[0] = b - _, err := prv.Decrypt(rand.Reader, ct, nil, nil) + _, err := prv.Decrypt(ct, nil, nil) if err != ErrInvalidPublicKey { fmt.Println("ecies: validated an invalid key") t.FailNow() @@ -441,14 +441,14 @@ func TestBox(t *testing.T) { t.Fatal(err) } - pt, err := prv2.Decrypt(rand.Reader, ct, nil, nil) + pt, err := prv2.Decrypt(ct, nil, nil) if err != nil { t.Fatal(err) } if !bytes.Equal(pt, message) { t.Fatal("ecies: plaintext doesn't match message") } - if _, err = prv1.Decrypt(rand.Reader, ct, nil, nil); err == nil { + if _, err = prv1.Decrypt(ct, nil, nil); err == nil { t.Fatal("ecies: encryption should not have succeeded") } } diff --git a/p2p/rlpx.go b/p2p/rlpx.go index 1889edac99..a320e81e7c 100644 --- a/p2p/rlpx.go +++ b/p2p/rlpx.go @@ -491,7 +491,7 @@ func readHandshakeMsg(msg plainDecoder, plainSize int, prv *ecdsa.PrivateKey, r } // Attempt decoding pre-EIP-8 "plain" format. key := ecies.ImportECDSA(prv) - if dec, err := key.Decrypt(rand.Reader, buf, nil, nil); err == nil { + if dec, err := key.Decrypt(buf, nil, nil); err == nil { msg.decodePlain(dec) return buf, nil } @@ -505,7 +505,7 @@ func readHandshakeMsg(msg plainDecoder, plainSize int, prv *ecdsa.PrivateKey, r if _, err := io.ReadFull(r, buf[plainSize:]); err != nil { return buf, err } - dec, err := key.Decrypt(rand.Reader, buf[2:], nil, prefix) + dec, err := key.Decrypt(buf[2:], nil, prefix) if err != nil { return buf, err } diff --git a/whisper/whisperv5/message.go b/whisper/whisperv5/message.go index c27535cd1a..34ce52e64f 100644 --- a/whisper/whisperv5/message.go +++ b/whisper/whisperv5/message.go @@ -277,7 +277,7 @@ func (msg *ReceivedMessage) decryptSymmetric(key []byte, nonce []byte) error { // decryptAsymmetric decrypts an encrypted payload with a private key. func (msg *ReceivedMessage) decryptAsymmetric(key *ecdsa.PrivateKey) error { - decrypted, err := ecies.ImportECDSA(key).Decrypt(crand.Reader, msg.Raw, nil, nil) + decrypted, err := ecies.ImportECDSA(key).Decrypt(msg.Raw, nil, nil) if err == nil { msg.Raw = decrypted } diff --git a/whisper/whisperv6/message.go b/whisper/whisperv6/message.go index b8318cbe8f..2d4e862441 100644 --- a/whisper/whisperv6/message.go +++ b/whisper/whisperv6/message.go @@ -289,7 +289,7 @@ func (msg *ReceivedMessage) decryptSymmetric(key []byte) error { // decryptAsymmetric decrypts an encrypted payload with a private key. func (msg *ReceivedMessage) decryptAsymmetric(key *ecdsa.PrivateKey) error { - decrypted, err := ecies.ImportECDSA(key).Decrypt(crand.Reader, msg.Raw, nil, nil) + decrypted, err := ecies.ImportECDSA(key).Decrypt(msg.Raw, nil, nil) if err == nil { msg.Raw = decrypted } From 84c5db5409cebf97276ab90db38ed73c21cf82ef Mon Sep 17 00:00:00 2001 From: hydai Date: Mon, 26 Mar 2018 18:48:04 +0800 Subject: [PATCH 05/19] core/vm: remove JIT VM codes (#16362) --- core/vm/doc.go | 17 +- core/vm/instructions_test.go | 6 +- core/vm/interpreter.go | 6 +- core/vm/logger_test.go | 2 +- core/vm/runtime/runtime.go | 4 +- core/vm/vm_jit.go | 389 ----------------------------------- core/vm/vm_jit_fake.go | 19 -- 7 files changed, 9 insertions(+), 434 deletions(-) delete mode 100644 core/vm/vm_jit.go delete mode 100644 core/vm/vm_jit_fake.go diff --git a/core/vm/doc.go b/core/vm/doc.go index 239be2cfec..5864d0cfa2 100644 --- a/core/vm/doc.go +++ b/core/vm/doc.go @@ -17,19 +17,8 @@ /* Package vm implements the Ethereum Virtual Machine. -The vm package implements two EVMs, a byte code VM and a JIT VM. The BC -(Byte Code) VM loops over a set of bytes and executes them according to the set -of rules defined in the Ethereum yellow paper. When the BC VM is invoked it -invokes the JIT VM in a separate goroutine and compiles the byte code in JIT -instructions. - -The JIT VM, when invoked, loops around a set of pre-defined instructions until -it either runs of gas, causes an internal error, returns or stops. - -The JIT optimiser attempts to pre-compile instructions in to chunks or segments -such as multiple PUSH operations and static JUMPs. It does this by analysing the -opcodes and attempts to match certain regions to known sets. Whenever the -optimiser finds said segments it creates a new instruction and replaces the -first occurrence in the sequence. +The vm package implements one EVM, a byte code VM. The BC (Byte Code) VM loops +over a set of bytes and executes them according to the set of rules defined +in the Ethereum yellow paper. */ package vm diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index 134363bb74..0de558612c 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -32,7 +32,7 @@ type twoOperandTest struct { func testTwoOperandOp(t *testing.T, tests []twoOperandTest, opFn func(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error)) { var ( - env = NewEVM(Context{}, nil, params.TestChainConfig, Config{EnableJit: false, ForceJit: false}) + env = NewEVM(Context{}, nil, params.TestChainConfig, Config{}) stack = newstack() pc = uint64(0) ) @@ -68,7 +68,7 @@ func testTwoOperandOp(t *testing.T, tests []twoOperandTest, opFn func(pc *uint64 func TestByteOp(t *testing.T) { var ( - env = NewEVM(Context{}, nil, params.TestChainConfig, Config{EnableJit: false, ForceJit: false}) + env = NewEVM(Context{}, nil, params.TestChainConfig, Config{}) stack = newstack() ) tests := []struct { @@ -198,7 +198,7 @@ func TestSLT(t *testing.T) { func opBenchmark(bench *testing.B, op func(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error), args ...string) { var ( - env = NewEVM(Context{}, nil, params.TestChainConfig, Config{EnableJit: false, ForceJit: false}) + env = NewEVM(Context{}, nil, params.TestChainConfig, Config{}) stack = newstack() ) // convert args diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 95490adfc4..47d5e7f2fa 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -28,10 +28,6 @@ import ( type Config struct { // Debug enabled debugging Interpreter options Debug bool - // EnableJit enabled the JIT VM - EnableJit bool - // ForceJit forces the JIT VM - ForceJit bool // Tracer is the op code logger Tracer Tracer // NoRecursion disabled Interpreter call, callcode, @@ -47,7 +43,7 @@ type Config struct { // Interpreter is used to run Ethereum based contracts and will utilise the // passed evmironment to query external sources for state information. -// The Interpreter will run the byte code VM or JIT VM based on the passed +// The Interpreter will run the byte code VM based on the passed // configuration. type Interpreter struct { evm *EVM diff --git a/core/vm/logger_test.go b/core/vm/logger_test.go index 915f7177e7..28830c445c 100644 --- a/core/vm/logger_test.go +++ b/core/vm/logger_test.go @@ -48,7 +48,7 @@ type dummyStateDB struct { func TestStoreCapture(t *testing.T) { var ( - env = NewEVM(Context{}, nil, params.TestChainConfig, Config{EnableJit: false, ForceJit: false}) + env = NewEVM(Context{}, nil, params.TestChainConfig, Config{}) logger = NewStructLogger(nil) mem = NewMemory() stack = newstack() diff --git a/core/vm/runtime/runtime.go b/core/vm/runtime/runtime.go index edbf541766..1e9ed7ae2d 100644 --- a/core/vm/runtime/runtime.go +++ b/core/vm/runtime/runtime.go @@ -41,7 +41,6 @@ type Config struct { GasLimit uint64 GasPrice *big.Int Value *big.Int - DisableJit bool // "disable" so it's enabled by default Debug bool EVMConfig vm.Config @@ -92,8 +91,7 @@ func setDefaults(cfg *Config) { // It returns the EVM's return value, the new state and an error if it failed. // // Executes sets up a in memory, temporarily, environment for the execution of -// the given code. It enabled the JIT by default and make sure that it's restored -// to it's original state afterwards. +// the given code. It makes sure that it's restored to it's original state afterwards. func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) { if cfg == nil { cfg = new(Config) diff --git a/core/vm/vm_jit.go b/core/vm/vm_jit.go deleted file mode 100644 index eb3acfb10d..0000000000 --- a/core/vm/vm_jit.go +++ /dev/null @@ -1,389 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// +build evmjit - -package vm - -/* - -void* evmjit_create(); -int evmjit_run(void* _jit, void* _data, void* _env); -void evmjit_destroy(void* _jit); - -// Shared library evmjit (e.g. libevmjit.so) is expected to be installed in /usr/local/lib -// More: https://github.com/ethereum/evmjit -#cgo LDFLAGS: -levmjit -*/ -import "C" - -/* -import ( - "bytes" - "errors" - "fmt" - "math/big" - "unsafe" - - "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/params" -) - -type JitVm struct { - env EVM - me ContextRef - callerAddr []byte - price *big.Int - data RuntimeData -} - -type i256 [32]byte - -type RuntimeData struct { - gas int64 - gasPrice int64 - callData *byte - callDataSize uint64 - address i256 - caller i256 - origin i256 - callValue i256 - coinBase i256 - difficulty i256 - gasLimit i256 - number uint64 - timestamp int64 - code *byte - codeSize uint64 - codeHash i256 -} - -func hash2llvm(h []byte) i256 { - var m i256 - copy(m[len(m)-len(h):], h) // right aligned copy - return m -} - -func llvm2hash(m *i256) []byte { - return C.GoBytes(unsafe.Pointer(m), C.int(len(m))) -} - -func llvm2hashRef(m *i256) []byte { - return (*[1 << 30]byte)(unsafe.Pointer(m))[:len(m):len(m)] -} - -func address2llvm(addr []byte) i256 { - n := hash2llvm(addr) - bswap(&n) - return n -} - -// bswap swap bytes of the 256-bit integer on LLVM side -// TODO: Do not change memory on LLVM side, that can conflict with memory access optimizations -func bswap(m *i256) *i256 { - for i, l := 0, len(m); i < l/2; i++ { - m[i], m[l-i-1] = m[l-i-1], m[i] - } - return m -} - -func trim(m []byte) []byte { - skip := 0 - for i := 0; i < len(m); i++ { - if m[i] == 0 { - skip++ - } else { - break - } - } - return m[skip:] -} - -func getDataPtr(m []byte) *byte { - var p *byte - if len(m) > 0 { - p = &m[0] - } - return p -} - -func big2llvm(n *big.Int) i256 { - m := hash2llvm(n.Bytes()) - bswap(&m) - return m -} - -func llvm2big(m *i256) *big.Int { - n := big.NewInt(0) - for i := 0; i < len(m); i++ { - b := big.NewInt(int64(m[i])) - b.Lsh(b, uint(i)*8) - n.Add(n, b) - } - return n -} - -// llvm2bytesRef creates a []byte slice that references byte buffer on LLVM side (as of that not controller by GC) -// User must ensure that referenced memory is available to Go until the data is copied or not needed any more -func llvm2bytesRef(data *byte, length uint64) []byte { - if length == 0 { - return nil - } - if data == nil { - panic("Unexpected nil data pointer") - } - return (*[1 << 30]byte)(unsafe.Pointer(data))[:length:length] -} - -func untested(condition bool, message string) { - if condition { - panic("Condition `" + message + "` tested. Remove assert.") - } -} - -func assert(condition bool, message string) { - if !condition { - panic("Assert `" + message + "` failed!") - } -} - -func NewJitVm(env EVM) *JitVm { - return &JitVm{env: env} -} - -func (self *JitVm) Run(me, caller ContextRef, code []byte, value, gas, price *big.Int, callData []byte) (ret []byte, err error) { - // TODO: depth is increased but never checked by VM. VM should not know about it at all. - self.env.SetDepth(self.env.Depth() + 1) - - // TODO: Move it to Env.Call() or sth - if Precompiled[string(me.Address())] != nil { - // if it's address of precompiled contract - // fallback to standard VM - stdVm := New(self.env) - return stdVm.Run(me, caller, code, value, gas, price, callData) - } - - if self.me != nil { - panic("JitVm.Run() can be called only once per JitVm instance") - } - - self.me = me - self.callerAddr = caller.Address() - self.price = price - - self.data.gas = gas.Int64() - self.data.gasPrice = price.Int64() - self.data.callData = getDataPtr(callData) - self.data.callDataSize = uint64(len(callData)) - self.data.address = address2llvm(self.me.Address()) - self.data.caller = address2llvm(caller.Address()) - self.data.origin = address2llvm(self.env.Origin()) - self.data.callValue = big2llvm(value) - self.data.coinBase = address2llvm(self.env.Coinbase()) - self.data.difficulty = big2llvm(self.env.Difficulty()) - self.data.gasLimit = big2llvm(self.env.GasLimit()) - self.data.number = self.env.BlockNumber().Uint64() - self.data.timestamp = self.env.Time() - self.data.code = getDataPtr(code) - self.data.codeSize = uint64(len(code)) - self.data.codeHash = hash2llvm(crypto.Keccak256(code)) // TODO: Get already computed hash? - - jit := C.evmjit_create() - retCode := C.evmjit_run(jit, unsafe.Pointer(&self.data), unsafe.Pointer(self)) - - if retCode < 0 { - err = errors.New("OOG from JIT") - gas.SetInt64(0) // Set gas to 0, JIT does not bother - } else { - gas.SetInt64(self.data.gas) - if retCode == 1 { // RETURN - ret = C.GoBytes(unsafe.Pointer(self.data.callData), C.int(self.data.callDataSize)) - } else if retCode == 2 { // SUICIDE - // TODO: Suicide support logic should be moved to Env to be shared by VM implementations - state := self.Env().State() - receiverAddr := llvm2hashRef(bswap(&self.data.address)) - receiver := state.GetOrNewStateObject(receiverAddr) - balance := state.GetBalance(me.Address()) - receiver.AddBalance(balance) - state.Delete(me.Address()) - } - } - - C.evmjit_destroy(jit) - return -} - -func (self *JitVm) Printf(format string, v ...interface{}) VirtualMachine { - return self -} - -func (self *JitVm) Endl() VirtualMachine { - return self -} - -func (self *JitVm) Env() EVM { - return self.env -} - -//export env_sha3 -func env_sha3(dataPtr *byte, length uint64, resultPtr unsafe.Pointer) { - data := llvm2bytesRef(dataPtr, length) - hash := crypto.Keccak256(data) - result := (*i256)(resultPtr) - *result = hash2llvm(hash) -} - -//export env_sstore -func env_sstore(vmPtr unsafe.Pointer, indexPtr unsafe.Pointer, valuePtr unsafe.Pointer) { - vm := (*JitVm)(vmPtr) - index := llvm2hash(bswap((*i256)(indexPtr))) - value := llvm2hash(bswap((*i256)(valuePtr))) - value = trim(value) - if len(value) == 0 { - prevValue := vm.env.State().GetState(vm.me.Address(), index) - if len(prevValue) != 0 { - vm.Env().State().Refund(vm.callerAddr, GasSStoreRefund) - } - } - - vm.env.State().SetState(vm.me.Address(), index, value) -} - -//export env_sload -func env_sload(vmPtr unsafe.Pointer, indexPtr unsafe.Pointer, resultPtr unsafe.Pointer) { - vm := (*JitVm)(vmPtr) - index := llvm2hash(bswap((*i256)(indexPtr))) - value := vm.env.State().GetState(vm.me.Address(), index) - result := (*i256)(resultPtr) - *result = hash2llvm(value) - bswap(result) -} - -//export env_balance -func env_balance(_vm unsafe.Pointer, _addr unsafe.Pointer, _result unsafe.Pointer) { - vm := (*JitVm)(_vm) - addr := llvm2hash((*i256)(_addr)) - balance := vm.Env().State().GetBalance(addr) - result := (*i256)(_result) - *result = big2llvm(balance) -} - -//export env_blockhash -func env_blockhash(_vm unsafe.Pointer, _number unsafe.Pointer, _result unsafe.Pointer) { - vm := (*JitVm)(_vm) - number := llvm2big((*i256)(_number)) - result := (*i256)(_result) - - currNumber := vm.Env().BlockNumber() - limit := big.NewInt(0).Sub(currNumber, big.NewInt(256)) - if number.Cmp(limit) >= 0 && number.Cmp(currNumber) < 0 { - hash := vm.Env().GetHash(uint64(number.Int64())) - *result = hash2llvm(hash) - } else { - *result = i256{} - } -} - -//export env_call -func env_call(_vm unsafe.Pointer, _gas *int64, _receiveAddr unsafe.Pointer, _value unsafe.Pointer, inDataPtr unsafe.Pointer, inDataLen uint64, outDataPtr *byte, outDataLen uint64, _codeAddr unsafe.Pointer) bool { - vm := (*JitVm)(_vm) - - //fmt.Printf("env_call (depth %d)\n", vm.Env().Depth()) - - defer func() { - if r := recover(); r != nil { - fmt.Printf("Recovered in env_call (depth %d, out %p %d): %s\n", vm.Env().Depth(), outDataPtr, outDataLen, r) - } - }() - - balance := vm.Env().State().GetBalance(vm.me.Address()) - value := llvm2big((*i256)(_value)) - - if balance.Cmp(value) >= 0 { - receiveAddr := llvm2hash((*i256)(_receiveAddr)) - inData := C.GoBytes(inDataPtr, C.int(inDataLen)) - outData := llvm2bytesRef(outDataPtr, outDataLen) - codeAddr := llvm2hash((*i256)(_codeAddr)) - gas := big.NewInt(*_gas) - var out []byte - var err error - if bytes.Equal(codeAddr, receiveAddr) { - out, err = vm.env.Call(vm.me, codeAddr, inData, gas, vm.price, value) - } else { - out, err = vm.env.CallCode(vm.me, codeAddr, inData, gas, vm.price, value) - } - *_gas = gas.Int64() - if err == nil { - copy(outData, out) - return true - } - } - - return false -} - -//export env_create -func env_create(_vm unsafe.Pointer, _gas *int64, _value unsafe.Pointer, initDataPtr unsafe.Pointer, initDataLen uint64, _result unsafe.Pointer) { - vm := (*JitVm)(_vm) - - value := llvm2big((*i256)(_value)) - initData := C.GoBytes(initDataPtr, C.int(initDataLen)) // TODO: Unnecessary if low balance - result := (*i256)(_result) - *result = i256{} - - gas := big.NewInt(*_gas) - ret, suberr, ref := vm.env.Create(vm.me, nil, initData, gas, vm.price, value) - if suberr == nil { - dataGas := big.NewInt(int64(len(ret))) // TODO: Not the best design. env.Create can do it, it has the reference to gas counter - dataGas.Mul(dataGas, params.CreateDataGas) - gas.Sub(gas, dataGas) - *result = hash2llvm(ref.Address()) - } - *_gas = gas.Int64() -} - -//export env_log -func env_log(_vm unsafe.Pointer, dataPtr unsafe.Pointer, dataLen uint64, _topic1 unsafe.Pointer, _topic2 unsafe.Pointer, _topic3 unsafe.Pointer, _topic4 unsafe.Pointer) { - vm := (*JitVm)(_vm) - - data := C.GoBytes(dataPtr, C.int(dataLen)) - - topics := make([][]byte, 0, 4) - if _topic1 != nil { - topics = append(topics, llvm2hash((*i256)(_topic1))) - } - if _topic2 != nil { - topics = append(topics, llvm2hash((*i256)(_topic2))) - } - if _topic3 != nil { - topics = append(topics, llvm2hash((*i256)(_topic3))) - } - if _topic4 != nil { - topics = append(topics, llvm2hash((*i256)(_topic4))) - } - - vm.Env().AddLog(state.NewLog(vm.me.Address(), topics, data, vm.env.BlockNumber().Uint64())) -} - -//export env_extcode -func env_extcode(_vm unsafe.Pointer, _addr unsafe.Pointer, o_size *uint64) *byte { - vm := (*JitVm)(_vm) - addr := llvm2hash((*i256)(_addr)) - code := vm.Env().State().GetCode(addr) - *o_size = uint64(len(code)) - return getDataPtr(code) -}*/ diff --git a/core/vm/vm_jit_fake.go b/core/vm/vm_jit_fake.go deleted file mode 100644 index 44b60abf6f..0000000000 --- a/core/vm/vm_jit_fake.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// +build !evmjit - -package vm From 7c131f4d6d936e3102b3908b6797ca3521addf36 Mon Sep 17 00:00:00 2001 From: hydai Date: Mon, 26 Mar 2018 18:48:39 +0800 Subject: [PATCH 06/19] core/asm: fixed typo (posititon -> position) (#16366) --- core/asm/compiler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/asm/compiler.go b/core/asm/compiler.go index 1b9025a549..18dc0877ff 100644 --- a/core/asm/compiler.go +++ b/core/asm/compiler.go @@ -114,7 +114,7 @@ func (c *Compiler) Compile() (string, []error) { } // next returns the next token and increments the -// posititon. +// position. func (c *Compiler) next() token { token := c.tokens[c.pos] c.pos++ From 495bdb0c713ce6deafa51fa25cb7ea66426b6b2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 26 Mar 2018 13:34:21 +0300 Subject: [PATCH 07/19] cmd: export preimages in RLP, support GZIP, uniform with block export --- cmd/geth/chaincmd.go | 175 +++++++++++++++---------------------------- cmd/geth/main.go | 4 +- cmd/utils/cmd.go | 94 ++++++++++++++++++++++- ethdb/database.go | 4 +- 4 files changed, 156 insertions(+), 121 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 692cc2d8d1..d3086921b9 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -40,11 +40,6 @@ import ( "gopkg.in/urfave/cli.v1" ) -var ( - // secureKeyPrefix is the database key prefix used to store trie node preimages. - secureKeyPrefix = []byte("secure-key-") -) - var ( initCommand = cli.Command{ Action: utils.MigrateFlags(initGenesis), @@ -100,6 +95,34 @@ Requires a first argument of the file to write to. Optional second and third arguments control the first and last block to write. In this mode, the file will be appended if already existing.`, + } + importPreimagesCommand = cli.Command{ + Action: utils.MigrateFlags(importPreimages), + Name: "import-preimages", + Usage: "Import the preimage database from an RLP stream", + ArgsUsage: "", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.CacheFlag, + utils.LightModeFlag, + }, + Category: "BLOCKCHAIN COMMANDS", + Description: ` + The import-preimages command imports hash preimages from an RLP encoded stream.`, + } + exportPreimagesCommand = cli.Command{ + Action: utils.MigrateFlags(exportPreimages), + Name: "export-preimages", + Usage: "Export the preimage database into an RLP stream", + ArgsUsage: "", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.CacheFlag, + utils.LightModeFlag, + }, + Category: "BLOCKCHAIN COMMANDS", + Description: ` +The export-preimages command export hash preimages to an RLP encoded stream`, } copydbCommand = cli.Command{ Action: utils.MigrateFlags(copyDb), @@ -146,34 +169,6 @@ Remove blockchain and state databases`, The arguments are interpreted as block numbers or hashes. Use "ethereum dump 0" to dump the genesis block.`, } - preimageDumpCommand = cli.Command{ - Action: utils.MigrateFlags(dumpPreimage), - Name: "preimagedump", - Usage: "Dump the preimage database in json format", - ArgsUsage: "", - Flags: []cli.Flag{ - utils.DataDirFlag, - utils.CacheFlag, - utils.LightModeFlag, - }, - Category: "BLOCKCHAIN COMMANDS", - Description: ` -Dump the preimage database in json format`, - } - preimageImportCommand = cli.Command{ - Action: utils.MigrateFlags(importPreimage), - Name: "preimageimport", - Usage: "Import the preimage data from the specified file", - ArgsUsage: "", - Flags: []cli.Flag{ - utils.DataDirFlag, - utils.CacheFlag, - utils.LightModeFlag, - }, - Category: "BLOCKCHAIN COMMANDS", - Description: ` -Import the preimage data from the specified file`, - } ) // initGenesis will initialise the given JSON format genesis file and writes it as @@ -332,7 +327,39 @@ func exportChain(ctx *cli.Context) error { if err != nil { utils.Fatalf("Export error: %v\n", err) } - fmt.Printf("Export done in %v", time.Since(start)) + fmt.Printf("Export done in %v\n", time.Since(start)) + return nil +} + +// importPreimages imports preimage data from the specified file. +func importPreimages(ctx *cli.Context) error { + if len(ctx.Args()) < 1 { + utils.Fatalf("This command requires an argument.") + } + stack := makeFullNode(ctx) + diskdb := utils.MakeChainDatabase(ctx, stack).(*ethdb.LDBDatabase) + + start := time.Now() + if err := utils.ImportPreimages(diskdb, ctx.Args().First()); err != nil { + utils.Fatalf("Export error: %v\n", err) + } + fmt.Printf("Export done in %v\n", time.Since(start)) + return nil +} + +// exportPreimages dumps the preimage data to specified json file in streaming way. +func exportPreimages(ctx *cli.Context) error { + if len(ctx.Args()) < 1 { + utils.Fatalf("This command requires an argument.") + } + stack := makeFullNode(ctx) + diskdb := utils.MakeChainDatabase(ctx, stack).(*ethdb.LDBDatabase) + + start := time.Now() + if err := utils.ExportPreimages(diskdb, ctx.Args().First()); err != nil { + utils.Fatalf("Export error: %v\n", err) + } + fmt.Printf("Export done in %v\n", time.Since(start)) return nil } @@ -439,86 +466,6 @@ func dump(ctx *cli.Context) error { return nil } -// PreimageEntry represents a map between preimage and hash. -type PreimageEntry struct { - Hash string `json:"hash"` - Preimage string `json:"preimage"` -} - -// dumpPreimage dumps the preimage data to specified json file in streaming way. -func dumpPreimage(ctx *cli.Context) error { - // Make sure the export json file has been specified. - if len(ctx.Args()) < 1 { - utils.Fatalf("This command requires an argument.") - } - - // Encode preimage data to json file in streaming way. - file, err := os.Create(ctx.Args().First()) - if err != nil { - return err - } - encoder := json.NewEncoder(file) - - stack := makeFullNode(ctx) - db := utils.MakeChainDatabase(ctx, stack) - - // Dump all preimage entries. - it := db.(*ethdb.LDBDatabase).NewIteratorByPrefix(secureKeyPrefix) - for it.Next() { - hash := it.Key()[len(secureKeyPrefix):] - if err := encoder.Encode(PreimageEntry{common.Bytes2Hex(hash), common.Bytes2Hex(it.Value())}); err != nil { - return err - } - } - return nil -} - -// importPreimages imports preimage data from the specified file. -func importPreimage(ctx *cli.Context) error { - // Make sure the export json file has been specified. - if len(ctx.Args()) < 1 { - utils.Fatalf("This command requires an argument.") - } - - // Decode the preimage data in streaming way. - file, err := os.Open(ctx.Args().First()) - if err != nil { - return err - } - decoder := json.NewDecoder(file) - - stack := makeFullNode(ctx) - db := utils.MakeChainDatabase(ctx, stack) - - var ( - entry PreimageEntry - preimages = make(map[common.Hash][]byte) - ) - - for decoder.More() { - if err := decoder.Decode(&entry); err != nil { - return err - } - preimages[common.HexToHash(entry.Hash)] = common.Hex2Bytes(entry.Preimage) - // Flush to database in batch - if len(preimages) > 1024 { - err := core.WritePreimages(db, 0, preimages) - if err != nil { - return err - } - preimages = make(map[common.Hash][]byte) - } - } - // Flush the last batch preimage data - if len(preimages) > 0 { - err := core.WritePreimages(db, 0, preimages) - if err != nil { - return err - } - } - return nil -} - // hashish returns true for strings that look like hashes. func hashish(x string) bool { _, err := strconv.Atoi(x) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 6e234a704b..061384d1b3 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -155,11 +155,11 @@ func init() { initCommand, importCommand, exportCommand, + importPreimagesCommand, + exportPreimagesCommand, copydbCommand, removedbCommand, dumpCommand, - preimageDumpCommand, - preimageImportCommand, // See monitorcmd.go: monitorCommand, // See accountcmd.go: diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 186d18d8f5..c0af4c13e7 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -27,8 +27,11 @@ import ( "strings" "syscall" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/internal/debug" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -105,6 +108,8 @@ func ImportChain(chain *core.BlockChain, fn string) error { } log.Info("Importing blockchain", "file", fn) + + // Open the file handle and potentially unwrap the gzip stream fh, err := os.Open(fn) if err != nil { return err @@ -180,8 +185,12 @@ func missingBlocks(chain *core.BlockChain, blocks []*types.Block) []*types.Block return nil } +// ExportChain exports a blockchain into the specified file, truncating any data +// already present in the file. func ExportChain(blockchain *core.BlockChain, fn string) error { log.Info("Exporting blockchain", "file", fn) + + // Open the file handle and potentially wrap with a gzip stream fh, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.ModePerm) if err != nil { return err @@ -193,7 +202,7 @@ func ExportChain(blockchain *core.BlockChain, fn string) error { writer = gzip.NewWriter(writer) defer writer.(*gzip.Writer).Close() } - + // Iterate over the blocks and export them if err := blockchain.Export(writer); err != nil { return err } @@ -202,9 +211,12 @@ func ExportChain(blockchain *core.BlockChain, fn string) error { return nil } +// ExportAppendChain exports a blockchain into the specified file, appending to +// the file if data already exists in it. func ExportAppendChain(blockchain *core.BlockChain, fn string, first uint64, last uint64) error { log.Info("Exporting blockchain", "file", fn) - // TODO verify mode perms + + // Open the file handle and potentially wrap with a gzip stream fh, err := os.OpenFile(fn, os.O_CREATE|os.O_APPEND|os.O_WRONLY, os.ModePerm) if err != nil { return err @@ -216,10 +228,86 @@ func ExportAppendChain(blockchain *core.BlockChain, fn string, first uint64, las writer = gzip.NewWriter(writer) defer writer.(*gzip.Writer).Close() } - + // Iterate over the blocks and export them if err := blockchain.ExportN(writer, first, last); err != nil { return err } log.Info("Exported blockchain to", "file", fn) return nil } + +// ImportPreimages imports a batch of exported hash preimages into the database. +func ImportPreimages(db *ethdb.LDBDatabase, fn string) error { + log.Info("Importing preimages", "file", fn) + + // Open the file handle and potentially unwrap the gzip stream + fh, err := os.Open(fn) + if err != nil { + return err + } + defer fh.Close() + + var reader io.Reader = fh + if strings.HasSuffix(fn, ".gz") { + if reader, err = gzip.NewReader(reader); err != nil { + return err + } + } + stream := rlp.NewStream(reader, 0) + + // Import the preimages in batches to prevent disk trashing + preimages := make(map[common.Hash][]byte) + + for { + // Read the next entry and ensure it's not junk + var blob []byte + + if err := stream.Decode(&blob); err != nil { + if err == io.EOF { + break + } + return err + } + // Accumulate the preimages and flush when enough ws gathered + preimages[crypto.Keccak256Hash(blob)] = common.CopyBytes(blob) + if len(preimages) > 1024 { + if err := core.WritePreimages(db, 0, preimages); err != nil { + return err + } + preimages = make(map[common.Hash][]byte) + } + } + // Flush the last batch preimage data + if len(preimages) > 0 { + return core.WritePreimages(db, 0, preimages) + } + return nil +} + +// ExportPreimages exports all known hash preimages into the specified file, +// truncating any data already present in the file. +func ExportPreimages(db *ethdb.LDBDatabase, fn string) error { + log.Info("Exporting preimages", "file", fn) + + // Open the file handle and potentially wrap with a gzip stream + fh, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.ModePerm) + if err != nil { + return err + } + defer fh.Close() + + var writer io.Writer = fh + if strings.HasSuffix(fn, ".gz") { + writer = gzip.NewWriter(writer) + defer writer.(*gzip.Writer).Close() + } + // Iterate over the preimages and export them + it := db.NewIteratorWithPrefix([]byte("secure-key-")) + for it.Next() { + if err := rlp.Encode(writer, it.Value()); err != nil { + return err + } + } + log.Info("Exported preimages", "file", fn) + return nil +} diff --git a/ethdb/database.go b/ethdb/database.go index d0256c56f4..30ed37dc7b 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -122,8 +122,8 @@ func (db *LDBDatabase) NewIterator() iterator.Iterator { return db.db.NewIterator(nil, nil) } -// NewIteratorByPrefix returns a iterator to iterate over subset of database content with a particular prefix. -func (db *LDBDatabase) NewIteratorByPrefix(prefix []byte) iterator.Iterator { +// NewIteratorWithPrefix returns a iterator to iterate over subset of database content with a particular prefix. +func (db *LDBDatabase) NewIteratorWithPrefix(prefix []byte) iterator.Iterator { return db.db.NewIterator(util.BytesPrefix(prefix), nil) } From e506d384e90e709027b46c1d3dffc0390482a22c Mon Sep 17 00:00:00 2001 From: Jia Chenhui Date: Mon, 26 Mar 2018 19:45:34 +0800 Subject: [PATCH 08/19] core/state: fix typo (#16370) --- core/state/statedb.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/state/statedb.go b/core/state/statedb.go index 776693e248..5473ff8da7 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -83,7 +83,7 @@ type StateDB struct { lock sync.Mutex } -// Create a new state from a given trie +// Create a new state from a given trie. func New(root common.Hash, db Database) (*StateDB, error) { tr, err := db.OpenTrie(root) if err != nil { @@ -110,7 +110,7 @@ func (self *StateDB) Error() error { return self.dbErr } -// Reset clears out all emphemeral state objects from the state db, but keeps +// Reset clears out all ephemeral state objects from the state db, but keeps // the underlying state trie to avoid reloading data for the next operations. func (self *StateDB) Reset(root common.Hash) error { tr, err := self.db.OpenTrie(root) From e9b5e22ad121ef2251d1c72526c0994c03d2caf0 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 26 Mar 2018 19:46:37 +0800 Subject: [PATCH 09/19] rpc: limit chunked requests (#16343) --- rpc/http.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rpc/http.go b/rpc/http.go index 9805d69b63..e8f51150f4 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -169,7 +169,8 @@ func (srv *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { // All checks passed, create a codec that reads direct from the request body // untilEOF and writes the response to w and order the server to process a // single request. - codec := NewJSONCodec(&httpReadWriteNopCloser{r.Body, w}) + body := io.LimitReader(r.Body, maxRequestContentLength) + codec := NewJSONCodec(&httpReadWriteNopCloser{body, w}) defer codec.Close() w.Header().Set("content-type", contentType) From db9b2f5405643b3584eaa90b05a3dbfa19d717fb Mon Sep 17 00:00:00 2001 From: Zhenguo Niu Date: Mon, 26 Mar 2018 20:21:20 +0800 Subject: [PATCH 10/19] cmd/puppeth: add constraints to network name (#16336) * cmd/puppeth: add constraints to network name * cmd/puppeth: update usage of network arg * cmd/puppeth: avoid package dependency on utils --- cmd/puppeth/puppeth.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/puppeth/puppeth.go b/cmd/puppeth/puppeth.go index 26382dac13..f9b8fe4811 100644 --- a/cmd/puppeth/puppeth.go +++ b/cmd/puppeth/puppeth.go @@ -20,6 +20,7 @@ package main import ( "math/rand" "os" + "strings" "time" "github.com/ethereum/go-ethereum/log" @@ -34,7 +35,7 @@ func main() { app.Flags = []cli.Flag{ cli.StringFlag{ Name: "network", - Usage: "name of the network to administer", + Usage: "name of the network to administer (no spaces or hyphens, please)", }, cli.IntFlag{ Name: "loglevel", @@ -47,6 +48,10 @@ func main() { log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(c.Int("loglevel")), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) rand.Seed(time.Now().UnixNano()) + network := c.String("network") + if strings.Contains(network, " ") || strings.Contains(network, "-") { + log.Crit("No spaces or hyphens allowed in network name") + } // Start the wizard and relinquish control makeWizard(c.String("network")).run() return nil From c3dc814fea97650a9791be87576534de01666763 Mon Sep 17 00:00:00 2001 From: hydai Date: Mon, 26 Mar 2018 21:40:00 +0800 Subject: [PATCH 11/19] core/vm: Fixed typo in core/vm/evm.go --- core/vm/evm.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/vm/evm.go b/core/vm/evm.go index 46e7baff4c..96676c314a 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -111,7 +111,7 @@ type EVM struct { callGasTemp uint64 } -// NewEVM retutrns a new EVM . The returned EVM is not thread safe and should +// NewEVM returns a new EVM. The returned EVM is not thread safe and should // only ever be used *once*. func NewEVM(ctx Context, statedb StateDB, chainConfig *params.ChainConfig, vmConfig Config) *EVM { evm := &EVM{ @@ -221,7 +221,7 @@ func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte, to = AccountRef(caller.Address()) ) // initialise a new contract and set the code that is to be used by the - // E The contract is a scoped evmironment for this execution context + // EVM. The contract is a scoped environment for this execution context // only. contract := NewContract(caller, to, value, gas) contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr)) @@ -341,7 +341,7 @@ func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.I evm.Transfer(evm.StateDB, caller.Address(), contractAddr, value) // initialise a new contract and set the code that is to be used by the - // E The contract is a scoped evmironment for this execution context + // EVM. The contract is a scoped environment for this execution context // only. contract := NewContract(caller, AccountRef(contractAddr), value, gas) contract.SetCallCode(&contractAddr, crypto.Keccak256Hash(code), code) From 89cc604a50e1fe7fb1cec45085e9b37a0800e644 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 26 Mar 2018 23:02:12 +0800 Subject: [PATCH 12/19] light: new mainnet CHT (#16390) --- light/postprocess.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/light/postprocess.go b/light/postprocess.go index 384a635f76..d67c5e2c3e 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -58,10 +58,10 @@ type trustedCheckpoint struct { var ( mainnetCheckpoint = trustedCheckpoint{ name: "mainnet", - sectionIdx: 157, - sectionHead: common.HexToHash("1963c080887ca7f406c2bb114293eea83e54f783f94df24b447f7e3b6317c747"), - chtRoot: common.HexToHash("42abc436567dfb678a38fa6a9f881aa4c8a4cc8eaa2def08359292c3d0bd48ec"), - bloomTrieRoot: common.HexToHash("281c9f8fb3cb8b37ae45e9907ef8f3b19cd22c54e297c2d6c09c1db1593dce42"), + sectionIdx: 161, + sectionHead: common.HexToHash("75b0c4baa7a62cece48abdcb03b6f31601961c9bece67dcd61df87aad4fc0d8d"), + chtRoot: common.HexToHash("bbbfaa67b29716348997ec21a39c03b8d1fb973f6a43740b865595ba26ee812f"), + bloomTrieRoot: common.HexToHash("d6db6e6248354d7453391ce97830072a28ea4216be0bd95a5db9f53b1a64677b"), } ropstenCheckpoint = trustedCheckpoint{ From 329ac18ef617d0238f71637bffe78f028b0f13f7 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 26 Mar 2018 18:08:41 +0200 Subject: [PATCH 13/19] params: v1.8.3 stable --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 921d075996..e37ea2d39f 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 8 // Minor version component of the current release - VersionPatch = 3 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 8 // Minor version component of the current release + VersionPatch = 3 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From 85ea9159d03c733791b92a4b445a5a750056eb39 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 26 Mar 2018 18:09:56 +0200 Subject: [PATCH 14/19] params, VERSION: v1.8.4 unstable --- VERSION | 2 +- params/version.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/VERSION b/VERSION index a7ee35a3ea..bfa363e76e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.8.3 +1.8.4 diff --git a/params/version.go b/params/version.go index e37ea2d39f..181f84631e 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 8 // Minor version component of the current release - VersionPatch = 3 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 8 // Minor version component of the current release + VersionPatch = 4 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From 61349907099f2e7cee7ce90e0191fe3f9df2e09e Mon Sep 17 00:00:00 2001 From: hydai Date: Tue, 27 Mar 2018 12:29:04 +0800 Subject: [PATCH 15/19] core/vm: Fixed typos in core/vm/interpreter.go --- core/vm/interpreter.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 47d5e7f2fa..7090e0261f 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -42,7 +42,7 @@ type Config struct { } // Interpreter is used to run Ethereum based contracts and will utilise the -// passed evmironment to query external sources for state information. +// passed environment to query external sources for state information. // The Interpreter will run the byte code VM based on the passed // configuration. type Interpreter struct { @@ -184,7 +184,7 @@ func (in *Interpreter) Run(contract *Contract, input []byte) (ret []byte, err er } } // consume the gas and return an error if not enough gas is available. - // cost is explicitly set so that the capture state defer method cas get the proper cost + // cost is explicitly set so that the capture state defer method can get the proper cost cost, err = operation.gasCost(in.gasTable, in.evm, contract, stack, mem, memorySize) if err != nil || !contract.UseGas(cost) { return nil, ErrOutOfGas From 45bd4feddeadfbde5d1e560797155aacb0abbadf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Tue, 27 Mar 2018 09:09:15 +0200 Subject: [PATCH 16/19] light: new CHT for ropsten (#16393) --- light/postprocess.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/light/postprocess.go b/light/postprocess.go index d67c5e2c3e..a1b1d9fb04 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -66,10 +66,10 @@ var ( ropstenCheckpoint = trustedCheckpoint{ name: "ropsten", - sectionIdx: 83, - sectionHead: common.HexToHash("3ca623586bc0da35f1fc8d9b6b55950f3b1f69be9c6501846a2df672adb61236"), - chtRoot: common.HexToHash("8f08ec7783969768c6ef06e5fe3398223cbf4ae2907b676da7b6fe6c7f55b059"), - bloomTrieRoot: common.HexToHash("02d86d3c6a87f8f8a92c2a59bbba2132ff6f9f61b0915a5dc28a9d8279219fd0"), + sectionIdx: 87, + sectionHead: common.HexToHash("ebc0adcb30ed21cbe95bd77499cc1af0bada621fee3644cb80dbcf1444c123fe"), + chtRoot: common.HexToHash("d9830f4893c821ddf149b8cb9d3e3bfe3109d2eea8e3c4a4ede7c8b2ee8a7800"), + bloomTrieRoot: common.HexToHash("c76e12d713f65b84c5a36d06bc77d0c8419248ea0b36e0812a78b76aa6da0ddb"), } ) From 80449719bd664bb0ed9c599765f6313f9b8a8303 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Tue, 27 Mar 2018 17:26:08 +0200 Subject: [PATCH 17/19] whisper: fix issue in topic list copy (#16381) - Fixes #16271. What was appeneded was a pointer to an object that changes during the iteration. - The topic is allocated as a 4-byte array, fill partial topics with 0s. Partial topics are currently disabled, but would crash as they rely on the presence of byte number 3. --- whisper/whisperv6/api.go | 7 ++-- whisper/whisperv6/api_test.go | 78 +++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 3 deletions(-) create mode 100644 whisper/whisperv6/api_test.go diff --git a/whisper/whisperv6/api.go b/whisper/whisperv6/api.go index 96e2b17e7c..3f3a082afe 100644 --- a/whisper/whisperv6/api.go +++ b/whisper/whisperv6/api.go @@ -558,9 +558,10 @@ func (api *PublicWhisperAPI) NewMessageFilter(req Criteria) (string, error) { } if len(req.Topics) > 0 { - topics = make([][]byte, 0, len(req.Topics)) - for _, topic := range req.Topics { - topics = append(topics, topic[:]) + topics = make([][]byte, len(req.Topics)) + for i, topic := range req.Topics { + topics[i] = make([]byte, TopicLength) + copy(topics[i], topic[:]) } } diff --git a/whisper/whisperv6/api_test.go b/whisper/whisperv6/api_test.go new file mode 100644 index 0000000000..004a41c949 --- /dev/null +++ b/whisper/whisperv6/api_test.go @@ -0,0 +1,78 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package whisperv6 + +import ( + "bytes" + "crypto/ecdsa" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + set "gopkg.in/fatih/set.v0" +) + +func TestMultipleTopicCopyInNewMessageFilter(t *testing.T) { + w := &Whisper{ + privateKeys: make(map[string]*ecdsa.PrivateKey), + symKeys: make(map[string][]byte), + envelopes: make(map[common.Hash]*Envelope), + expirations: make(map[uint32]*set.SetNonTS), + peers: make(map[*Peer]struct{}), + messageQueue: make(chan *Envelope, messageQueueLimit), + p2pMsgQueue: make(chan *Envelope, messageQueueLimit), + quit: make(chan struct{}), + syncAllowance: DefaultSyncAllowance, + } + w.filters = NewFilters(w) + + keyID, err := w.GenerateSymKey() + if err != nil { + t.Fatalf("Error generating symmetric key: %v", err) + } + api := PublicWhisperAPI{ + w: w, + lastUsed: make(map[string]time.Time), + } + + t1 := [4]byte{0xde, 0xea, 0xbe, 0xef} + t2 := [4]byte{0xca, 0xfe, 0xde, 0xca} + + crit := Criteria{ + SymKeyID: keyID, + Topics: []TopicType{TopicType(t1), TopicType(t2)}, + } + + _, err = api.NewMessageFilter(crit) + if err != nil { + t.Fatalf("Error creating the filter: %v", err) + } + + found := false + candidates := w.filters.getWatchersByTopic(TopicType(t1)) + for _, f := range candidates { + if len(f.Topics) == 2 { + if bytes.Equal(f.Topics[0], t1[:]) && bytes.Equal(f.Topics[1], t2[:]) { + found = true + } + } + } + + if !found { + t.Fatalf("Could not find filter with both topics") + } +} From 1a8894b3d59bdf592244b91b6a629f7cf25dcdde Mon Sep 17 00:00:00 2001 From: Jia Chenhui Date: Wed, 28 Mar 2018 14:26:37 +0800 Subject: [PATCH 18/19] core/state: uniform parameter style (#16398) - Uniform code style. --- core/state/statedb.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/core/state/statedb.go b/core/state/statedb.go index 5473ff8da7..bd67e789d5 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -235,10 +235,10 @@ func (self *StateDB) GetCodeHash(addr common.Address) common.Hash { return common.BytesToHash(stateObject.CodeHash()) } -func (self *StateDB) GetState(a common.Address, b common.Hash) common.Hash { - stateObject := self.getStateObject(a) +func (self *StateDB) GetState(addr common.Address, bhash common.Hash) common.Hash { + stateObject := self.getStateObject(addr) if stateObject != nil { - return stateObject.GetState(self.db, b) + return stateObject.GetState(self.db, bhash) } return common.Hash{} } @@ -250,8 +250,8 @@ func (self *StateDB) Database() Database { // StorageTrie returns the storage trie of an account. // The return value is a copy and is nil for non-existent accounts. -func (self *StateDB) StorageTrie(a common.Address) Trie { - stateObject := self.getStateObject(a) +func (self *StateDB) StorageTrie(addr common.Address) Trie { + stateObject := self.getStateObject(addr) if stateObject == nil { return nil } @@ -271,7 +271,7 @@ func (self *StateDB) HasSuicided(addr common.Address) bool { * SETTERS */ -// AddBalance adds amount to the account associated with addr +// AddBalance adds amount to the account associated with addr. func (self *StateDB) AddBalance(addr common.Address, amount *big.Int) { stateObject := self.GetOrNewStateObject(addr) if stateObject != nil { @@ -279,7 +279,7 @@ func (self *StateDB) AddBalance(addr common.Address, amount *big.Int) { } } -// SubBalance subtracts amount from the account associated with addr +// SubBalance subtracts amount from the account associated with addr. func (self *StateDB) SubBalance(addr common.Address, amount *big.Int) { stateObject := self.GetOrNewStateObject(addr) if stateObject != nil { @@ -308,7 +308,7 @@ func (self *StateDB) SetCode(addr common.Address, code []byte) { } } -func (self *StateDB) SetState(addr common.Address, key common.Hash, value common.Hash) { +func (self *StateDB) SetState(addr common.Address, key, value common.Hash) { stateObject := self.GetOrNewStateObject(addr) if stateObject != nil { stateObject.SetState(self.db, key, value) @@ -337,7 +337,7 @@ func (self *StateDB) Suicide(addr common.Address) bool { } // -// Setting, updating & deleting state object methods +// Setting, updating & deleting state object methods. // // updateStateObject writes the given object to the trie. @@ -388,7 +388,7 @@ func (self *StateDB) setStateObject(object *stateObject) { self.stateObjects[object.Address()] = object } -// Retrieve a state object or create a new state object if nil +// Retrieve a state object or create a new state object if nil. func (self *StateDB) GetOrNewStateObject(addr common.Address) *stateObject { stateObject := self.getStateObject(addr) if stateObject == nil || stateObject.deleted { From a095b84ec5a35cb3432380cc677d4e244b4a137f Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 28 Mar 2018 14:35:41 +0200 Subject: [PATCH 19/19] travis.yml: remove sudo requirement for PPA and Azure purge builders (#16404) This is supposed to fix the FTP upload issue according to travis-ci/travis-ci#9391. --- .travis.yml | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index cade11700f..40d940de0d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,14 +47,12 @@ matrix: script: - go run build/ci.go lint - # This builder does the Ubuntu PPA and Linux Azure uploads + # This builder does the Ubuntu PPA upload - os: linux dist: trusty - sudo: required go: "1.10" env: - ubuntu-ppa - - azure-linux git: submodules: false # avoid cloning ethereum/tests addons: @@ -63,11 +61,25 @@ matrix: - devscripts - debhelper - dput - - gcc-multilib - fakeroot script: - # Build for the primary platforms that Trusty can manage - go run build/ci.go debsrc -signer "Go Ethereum Linux Builder " -upload ppa:ethereum/ethereum + + # This builder does the Linux Azure uploads + - os: linux + dist: trusty + sudo: required + go: "1.10" + env: + - azure-linux + git: + submodules: false # avoid cloning ethereum/tests + addons: + apt: + packages: + - gcc-multilib + script: + # Build for the primary platforms that Trusty can manage - go run build/ci.go install - go run build/ci.go archive -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds - go run build/ci.go install -arch 386 @@ -181,7 +193,6 @@ matrix: # This builder does the Azure archive purges to avoid accumulating junk - os: linux dist: trusty - sudo: required go: "1.10" env: - azure-purge