From a42be3b78dcb70ea3c44ba46c277cf41ef0bba23 Mon Sep 17 00:00:00 2001 From: kiel barry Date: Tue, 8 May 2018 02:48:07 -0700 Subject: [PATCH 01/46] rlp: fix some golint warnings (#16659) --- rlp/decode.go | 43 ++++++++++++++++++------------------------- rlp/encode.go | 21 +++++++++------------ 2 files changed, 27 insertions(+), 37 deletions(-) diff --git a/rlp/decode.go b/rlp/decode.go index 60d9dab2b5..dbbe599597 100644 --- a/rlp/decode.go +++ b/rlp/decode.go @@ -29,6 +29,23 @@ import ( ) var ( + // EOL is returned when the end of the current list + // has been reached during streaming. + EOL = errors.New("rlp: end of list") + + // Actual Errors + ErrExpectedString = errors.New("rlp: expected String or Byte") + ErrExpectedList = errors.New("rlp: expected List") + ErrCanonInt = errors.New("rlp: non-canonical integer format") + ErrCanonSize = errors.New("rlp: non-canonical size information") + ErrElemTooLarge = errors.New("rlp: element is larger than containing list") + ErrValueTooLarge = errors.New("rlp: value size exceeds available input length") + ErrMoreThanOneValue = errors.New("rlp: input contains more than one value") + + // internal errors + errNotInList = errors.New("rlp: call of ListEnd outside of any list") + errNotAtEOL = errors.New("rlp: call of ListEnd not positioned at EOL") + errUintOverflow = errors.New("rlp: uint overflow") errNoPointer = errors.New("rlp: interface given to Decode must be a pointer") errDecodeIntoNil = errors.New("rlp: pointer given to Decode must not be nil") ) @@ -274,9 +291,8 @@ func makeListDecoder(typ reflect.Type, tag tags) (decoder, error) { if etype.Kind() == reflect.Uint8 && !reflect.PtrTo(etype).Implements(decoderInterface) { if typ.Kind() == reflect.Array { return decodeByteArray, nil - } else { - return decodeByteSlice, nil } + return decodeByteSlice, nil } etypeinfo, err := cachedTypeInfo1(etype, tags{}) if err != nil { @@ -555,29 +571,6 @@ func (k Kind) String() string { } } -var ( - // EOL is returned when the end of the current list - // has been reached during streaming. - EOL = errors.New("rlp: end of list") - - // Actual Errors - ErrExpectedString = errors.New("rlp: expected String or Byte") - ErrExpectedList = errors.New("rlp: expected List") - ErrCanonInt = errors.New("rlp: non-canonical integer format") - ErrCanonSize = errors.New("rlp: non-canonical size information") - ErrElemTooLarge = errors.New("rlp: element is larger than containing list") - ErrValueTooLarge = errors.New("rlp: value size exceeds available input length") - - // This error is reported by DecodeBytes if the slice contains - // additional data after the first RLP value. - ErrMoreThanOneValue = errors.New("rlp: input contains more than one value") - - // internal errors - errNotInList = errors.New("rlp: call of ListEnd outside of any list") - errNotAtEOL = errors.New("rlp: call of ListEnd not positioned at EOL") - errUintOverflow = errors.New("rlp: uint overflow") -) - // ByteReader must be implemented by any input reader for a Stream. It // is implemented by e.g. bufio.Reader and bytes.Reader. type ByteReader interface { diff --git a/rlp/encode.go b/rlp/encode.go index 44592c2f53..445b4b5b21 100644 --- a/rlp/encode.go +++ b/rlp/encode.go @@ -92,7 +92,7 @@ func Encode(w io.Writer, val interface{}) error { return eb.toWriter(w) } -// EncodeBytes returns the RLP encoding of val. +// EncodeToBytes returns the RLP encoding of val. // Please see the documentation of Encode for the encoding rules. func EncodeToBytes(val interface{}) ([]byte, error) { eb := encbufPool.Get().(*encbuf) @@ -104,7 +104,7 @@ func EncodeToBytes(val interface{}) ([]byte, error) { return eb.toBytes(), nil } -// EncodeReader returns a reader from which the RLP encoding of val +// EncodeToReader returns a reader from which the RLP encoding of val // can be read. The returned size is the total size of the encoded // data. // @@ -151,11 +151,10 @@ func puthead(buf []byte, smalltag, largetag byte, size uint64) int { if size < 56 { buf[0] = smalltag + byte(size) return 1 - } else { - sizesize := putint(buf[1:], size) - buf[0] = largetag + byte(sizesize) - return sizesize + 1 } + sizesize := putint(buf[1:], size) + buf[0] = largetag + byte(sizesize) + return sizesize + 1 } // encbufs are pooled. @@ -218,7 +217,7 @@ func (w *encbuf) list() *listhead { func (w *encbuf) listEnd(lh *listhead) { lh.size = w.size() - lh.offset - lh.size if lh.size < 56 { - w.lhsize += 1 // length encoded into kind tag + w.lhsize++ // length encoded into kind tag } else { w.lhsize += 1 + intsize(uint64(lh.size)) } @@ -322,10 +321,9 @@ func (r *encReader) next() []byte { p := r.buf.str[r.strpos:head.offset] r.strpos += sizebefore return p - } else { - r.lhpos++ - return head.encode(r.buf.sizebuf) } + r.lhpos++ + return head.encode(r.buf.sizebuf) case r.strpos < len(r.buf.str): // String data at the end, after all list headers. @@ -576,9 +574,8 @@ func makePtrWriter(typ reflect.Type) (writer, error) { writer := func(val reflect.Value, w *encbuf) error { if val.IsNil() { return nilfunc(w) - } else { - return etypeinfo.writer(val.Elem(), w) } + return etypeinfo.writer(val.Elem(), w) } return writer, err } From 864e80a48fd18fda89b80b11f9116c4e11cb835d Mon Sep 17 00:00:00 2001 From: kiel barry Date: Tue, 8 May 2018 04:08:43 -0700 Subject: [PATCH 02/46] p2p: fix some golint warnings (#16577) --- p2p/discover/table_test.go | 20 +- p2p/discv5/net_test.go | 20 +- p2p/discv5/node.go | 8 +- p2p/discv5/ticket.go | 4 +- p2p/discv5/topic.go | 16 +- p2p/message.go | 24 +-- p2p/peer_error.go | 4 +- p2p/simulations/adapters/inproc.go | 98 ++++----- p2p/simulations/adapters/state.go | 8 +- p2p/simulations/network.go | 309 +++++++++++++++-------------- p2p/testing/peerpool.go | 34 ++-- p2p/testing/protocolsession.go | 26 +-- p2p/testing/protocoltester.go | 12 +- 13 files changed, 292 insertions(+), 291 deletions(-) diff --git a/p2p/discover/table_test.go b/p2p/discover/table_test.go index 3ce48d2995..f2d3f9a2ad 100644 --- a/p2p/discover/table_test.go +++ b/p2p/discover/table_test.go @@ -582,26 +582,26 @@ func (*preminedTestnet) ping(toid NodeID, toaddr *net.UDPAddr) error { return ni // mine generates a testnet struct literal with nodes at // various distances to the given target. -func (n *preminedTestnet) mine(target NodeID) { - n.target = target - n.targetSha = crypto.Keccak256Hash(n.target[:]) +func (tn *preminedTestnet) mine(target NodeID) { + tn.target = target + tn.targetSha = crypto.Keccak256Hash(tn.target[:]) found := 0 for found < bucketSize*10 { k := newkey() id := PubkeyID(&k.PublicKey) sha := crypto.Keccak256Hash(id[:]) - ld := logdist(n.targetSha, sha) - if len(n.dists[ld]) < bucketSize { - n.dists[ld] = append(n.dists[ld], id) + ld := logdist(tn.targetSha, sha) + if len(tn.dists[ld]) < bucketSize { + tn.dists[ld] = append(tn.dists[ld], id) fmt.Println("found ID with ld", ld) found++ } } fmt.Println("&preminedTestnet{") - fmt.Printf(" target: %#v,\n", n.target) - fmt.Printf(" targetSha: %#v,\n", n.targetSha) - fmt.Printf(" dists: [%d][]NodeID{\n", len(n.dists)) - for ld, ns := range n.dists { + fmt.Printf(" target: %#v,\n", tn.target) + fmt.Printf(" targetSha: %#v,\n", tn.targetSha) + fmt.Printf(" dists: [%d][]NodeID{\n", len(tn.dists)) + for ld, ns := range tn.dists { if len(ns) == 0 { continue } diff --git a/p2p/discv5/net_test.go b/p2p/discv5/net_test.go index 369282ca9c..001d193cc9 100644 --- a/p2p/discv5/net_test.go +++ b/p2p/discv5/net_test.go @@ -336,26 +336,26 @@ func (*preminedTestnet) localAddr() *net.UDPAddr { // mine generates a testnet struct literal with nodes at // various distances to the given target. -func (n *preminedTestnet) mine(target NodeID) { - n.target = target - n.targetSha = crypto.Keccak256Hash(n.target[:]) +func (tn *preminedTestnet) mine(target NodeID) { + tn.target = target + tn.targetSha = crypto.Keccak256Hash(tn.target[:]) found := 0 for found < bucketSize*10 { k := newkey() id := PubkeyID(&k.PublicKey) sha := crypto.Keccak256Hash(id[:]) - ld := logdist(n.targetSha, sha) - if len(n.dists[ld]) < bucketSize { - n.dists[ld] = append(n.dists[ld], id) + ld := logdist(tn.targetSha, sha) + if len(tn.dists[ld]) < bucketSize { + tn.dists[ld] = append(tn.dists[ld], id) fmt.Println("found ID with ld", ld) found++ } } fmt.Println("&preminedTestnet{") - fmt.Printf(" target: %#v,\n", n.target) - fmt.Printf(" targetSha: %#v,\n", n.targetSha) - fmt.Printf(" dists: [%d][]NodeID{\n", len(n.dists)) - for ld, ns := range n.dists { + fmt.Printf(" target: %#v,\n", tn.target) + fmt.Printf(" targetSha: %#v,\n", tn.targetSha) + fmt.Printf(" dists: [%d][]NodeID{\n", len(tn.dists)) + for ld, ns := range tn.dists { if len(ns) == 0 { continue } diff --git a/p2p/discv5/node.go b/p2p/discv5/node.go index fd88a55b15..3d47485122 100644 --- a/p2p/discv5/node.go +++ b/p2p/discv5/node.go @@ -315,11 +315,11 @@ func PubkeyID(pub *ecdsa.PublicKey) NodeID { // Pubkey returns the public key represented by the node ID. // It returns an error if the ID is not a point on the curve. -func (id NodeID) Pubkey() (*ecdsa.PublicKey, error) { +func (n NodeID) Pubkey() (*ecdsa.PublicKey, error) { p := &ecdsa.PublicKey{Curve: crypto.S256(), X: new(big.Int), Y: new(big.Int)} - half := len(id) / 2 - p.X.SetBytes(id[:half]) - p.Y.SetBytes(id[half:]) + half := len(n) / 2 + p.X.SetBytes(n[:half]) + p.Y.SetBytes(n[half:]) if !p.Curve.IsOnCurve(p.X, p.Y) { return nil, errors.New("id is invalid secp256k1 curve point") } diff --git a/p2p/discv5/ticket.go b/p2p/discv5/ticket.go index b3d1ac4baf..ae4b18e7cd 100644 --- a/p2p/discv5/ticket.go +++ b/p2p/discv5/ticket.go @@ -304,8 +304,8 @@ func (s ticketRefByWaitTime) Len() int { return len(s) } -func (r ticketRef) waitTime() mclock.AbsTime { - return r.t.regTime[r.idx] - r.t.issueTime +func (ref ticketRef) waitTime() mclock.AbsTime { + return ref.t.regTime[ref.idx] - ref.t.issueTime } // Less reports whether the element with diff --git a/p2p/discv5/topic.go b/p2p/discv5/topic.go index e7a7f8e02a..609a41297f 100644 --- a/p2p/discv5/topic.go +++ b/p2p/discv5/topic.go @@ -271,15 +271,15 @@ func (t *topicTable) useTicket(node *Node, serialNo uint32, topics []Topic, idx return false } -func (topictab *topicTable) getTicket(node *Node, topics []Topic) *ticket { - topictab.collectGarbage() +func (t *topicTable) getTicket(node *Node, topics []Topic) *ticket { + t.collectGarbage() now := mclock.Now() - n := topictab.getOrNewNode(node) + n := t.getOrNewNode(node) n.lastIssuedTicket++ - topictab.storeTicketCounters(node) + t.storeTicketCounters(node) - t := &ticket{ + tic := &ticket{ issueTime: now, topics: topics, serial: n.lastIssuedTicket, @@ -287,15 +287,15 @@ func (topictab *topicTable) getTicket(node *Node, topics []Topic) *ticket { } for i, topic := range topics { var waitPeriod time.Duration - if topic := topictab.topics[topic]; topic != nil { + if topic := t.topics[topic]; topic != nil { waitPeriod = topic.wcl.waitPeriod } else { waitPeriod = minWaitPeriod } - t.regTime[i] = now + mclock.AbsTime(waitPeriod) + tic.regTime[i] = now + mclock.AbsTime(waitPeriod) } - return t + return tic } const gcInterval = time.Minute diff --git a/p2p/message.go b/p2p/message.go index ffa2ef86a0..a4eac54d33 100644 --- a/p2p/message.go +++ b/p2p/message.go @@ -270,15 +270,15 @@ func newMsgEventer(rw MsgReadWriter, feed *event.Feed, peerID discover.NodeID, p // ReadMsg reads a message from the underlying MsgReadWriter and emits a // "message received" event -func (self *msgEventer) ReadMsg() (Msg, error) { - msg, err := self.MsgReadWriter.ReadMsg() +func (ev *msgEventer) ReadMsg() (Msg, error) { + msg, err := ev.MsgReadWriter.ReadMsg() if err != nil { return msg, err } - self.feed.Send(&PeerEvent{ + ev.feed.Send(&PeerEvent{ Type: PeerEventTypeMsgRecv, - Peer: self.peerID, - Protocol: self.Protocol, + Peer: ev.peerID, + Protocol: ev.Protocol, MsgCode: &msg.Code, MsgSize: &msg.Size, }) @@ -287,15 +287,15 @@ func (self *msgEventer) ReadMsg() (Msg, error) { // WriteMsg writes a message to the underlying MsgReadWriter and emits a // "message sent" event -func (self *msgEventer) WriteMsg(msg Msg) error { - err := self.MsgReadWriter.WriteMsg(msg) +func (ev *msgEventer) WriteMsg(msg Msg) error { + err := ev.MsgReadWriter.WriteMsg(msg) if err != nil { return err } - self.feed.Send(&PeerEvent{ + ev.feed.Send(&PeerEvent{ Type: PeerEventTypeMsgSend, - Peer: self.peerID, - Protocol: self.Protocol, + Peer: ev.peerID, + Protocol: ev.Protocol, MsgCode: &msg.Code, MsgSize: &msg.Size, }) @@ -304,8 +304,8 @@ func (self *msgEventer) WriteMsg(msg Msg) error { // Close closes the underlying MsgReadWriter if it implements the io.Closer // interface -func (self *msgEventer) Close() error { - if v, ok := self.MsgReadWriter.(io.Closer); ok { +func (ev *msgEventer) Close() error { + if v, ok := ev.MsgReadWriter.(io.Closer); ok { return v.Close() } return nil diff --git a/p2p/peer_error.go b/p2p/peer_error.go index a1cddb707b..ab61bfef06 100644 --- a/p2p/peer_error.go +++ b/p2p/peer_error.go @@ -48,8 +48,8 @@ func newPeerError(code int, format string, v ...interface{}) *peerError { return err } -func (self *peerError) Error() string { - return self.message +func (pe *peerError) Error() string { + return pe.message } var errProtocolReturned = errors.New("protocol returned") diff --git a/p2p/simulations/adapters/inproc.go b/p2p/simulations/adapters/inproc.go index 48d7c17301..6d90b4a9fc 100644 --- a/p2p/simulations/adapters/inproc.go +++ b/p2p/simulations/adapters/inproc.go @@ -154,30 +154,30 @@ type SimNode struct { } // Addr returns the node's discovery address -func (self *SimNode) Addr() []byte { - return []byte(self.Node().String()) +func (sn *SimNode) Addr() []byte { + return []byte(sn.Node().String()) } // Node returns a discover.Node representing the SimNode -func (self *SimNode) Node() *discover.Node { - return discover.NewNode(self.ID, net.IP{127, 0, 0, 1}, 30303, 30303) +func (sn *SimNode) Node() *discover.Node { + return discover.NewNode(sn.ID, net.IP{127, 0, 0, 1}, 30303, 30303) } // Client returns an rpc.Client which can be used to communicate with the // underlying services (it is set once the node has started) -func (self *SimNode) Client() (*rpc.Client, error) { - self.lock.RLock() - defer self.lock.RUnlock() - if self.client == nil { +func (sn *SimNode) Client() (*rpc.Client, error) { + sn.lock.RLock() + defer sn.lock.RUnlock() + if sn.client == nil { return nil, errors.New("node not started") } - return self.client, nil + return sn.client, nil } // ServeRPC serves RPC requests over the given connection by creating an // in-memory client to the node's RPC server -func (self *SimNode) ServeRPC(conn net.Conn) error { - handler, err := self.node.RPCHandler() +func (sn *SimNode) ServeRPC(conn net.Conn) error { + handler, err := sn.node.RPCHandler() if err != nil { return err } @@ -187,13 +187,13 @@ func (self *SimNode) ServeRPC(conn net.Conn) error { // Snapshots creates snapshots of the services by calling the // simulation_snapshot RPC method -func (self *SimNode) Snapshots() (map[string][]byte, error) { - self.lock.RLock() - services := make(map[string]node.Service, len(self.running)) - for name, service := range self.running { +func (sn *SimNode) Snapshots() (map[string][]byte, error) { + sn.lock.RLock() + services := make(map[string]node.Service, len(sn.running)) + for name, service := range sn.running { services[name] = service } - self.lock.RUnlock() + sn.lock.RUnlock() if len(services) == 0 { return nil, errors.New("no running services") } @@ -213,23 +213,23 @@ func (self *SimNode) Snapshots() (map[string][]byte, error) { } // Start registers the services and starts the underlying devp2p node -func (self *SimNode) Start(snapshots map[string][]byte) error { +func (sn *SimNode) Start(snapshots map[string][]byte) error { newService := func(name string) func(ctx *node.ServiceContext) (node.Service, error) { return func(nodeCtx *node.ServiceContext) (node.Service, error) { ctx := &ServiceContext{ - RPCDialer: self.adapter, + RPCDialer: sn.adapter, NodeContext: nodeCtx, - Config: self.config, + Config: sn.config, } if snapshots != nil { ctx.Snapshot = snapshots[name] } - serviceFunc := self.adapter.services[name] + serviceFunc := sn.adapter.services[name] service, err := serviceFunc(ctx) if err != nil { return nil, err } - self.running[name] = service + sn.running[name] = service return service, nil } } @@ -237,9 +237,9 @@ func (self *SimNode) Start(snapshots map[string][]byte) error { // ensure we only register the services once in the case of the node // being stopped and then started again var regErr error - self.registerOnce.Do(func() { - for _, name := range self.config.Services { - if err := self.node.Register(newService(name)); err != nil { + sn.registerOnce.Do(func() { + for _, name := range sn.config.Services { + if err := sn.node.Register(newService(name)); err != nil { regErr = err return } @@ -249,54 +249,54 @@ func (self *SimNode) Start(snapshots map[string][]byte) error { return regErr } - if err := self.node.Start(); err != nil { + if err := sn.node.Start(); err != nil { return err } // create an in-process RPC client - handler, err := self.node.RPCHandler() + handler, err := sn.node.RPCHandler() if err != nil { return err } - self.lock.Lock() - self.client = rpc.DialInProc(handler) - self.lock.Unlock() + sn.lock.Lock() + sn.client = rpc.DialInProc(handler) + sn.lock.Unlock() return nil } // Stop closes the RPC client and stops the underlying devp2p node -func (self *SimNode) Stop() error { - self.lock.Lock() - if self.client != nil { - self.client.Close() - self.client = nil +func (sn *SimNode) Stop() error { + sn.lock.Lock() + if sn.client != nil { + sn.client.Close() + sn.client = nil } - self.lock.Unlock() - return self.node.Stop() + sn.lock.Unlock() + return sn.node.Stop() } // Services returns a copy of the underlying services -func (self *SimNode) Services() []node.Service { - self.lock.RLock() - defer self.lock.RUnlock() - services := make([]node.Service, 0, len(self.running)) - for _, service := range self.running { +func (sn *SimNode) Services() []node.Service { + sn.lock.RLock() + defer sn.lock.RUnlock() + services := make([]node.Service, 0, len(sn.running)) + for _, service := range sn.running { services = append(services, service) } return services } // Server returns the underlying p2p.Server -func (self *SimNode) Server() *p2p.Server { - return self.node.Server() +func (sn *SimNode) Server() *p2p.Server { + return sn.node.Server() } // SubscribeEvents subscribes the given channel to peer events from the // underlying p2p.Server -func (self *SimNode) SubscribeEvents(ch chan *p2p.PeerEvent) event.Subscription { - srv := self.Server() +func (sn *SimNode) SubscribeEvents(ch chan *p2p.PeerEvent) event.Subscription { + srv := sn.Server() if srv == nil { panic("node not running") } @@ -304,12 +304,12 @@ func (self *SimNode) SubscribeEvents(ch chan *p2p.PeerEvent) event.Subscription } // NodeInfo returns information about the node -func (self *SimNode) NodeInfo() *p2p.NodeInfo { - server := self.Server() +func (sn *SimNode) NodeInfo() *p2p.NodeInfo { + server := sn.Server() if server == nil { return &p2p.NodeInfo{ - ID: self.ID.String(), - Enode: self.Node().String(), + ID: sn.ID.String(), + Enode: sn.Node().String(), } } return server.NodeInfo() diff --git a/p2p/simulations/adapters/state.go b/p2p/simulations/adapters/state.go index 0d4ecfb0ff..78dfb11f95 100644 --- a/p2p/simulations/adapters/state.go +++ b/p2p/simulations/adapters/state.go @@ -20,12 +20,12 @@ type SimStateStore struct { m map[string][]byte } -func (self *SimStateStore) Load(s string) ([]byte, error) { - return self.m[s], nil +func (st *SimStateStore) Load(s string) ([]byte, error) { + return st.m[s], nil } -func (self *SimStateStore) Save(s string, data []byte) error { - self.m[s] = data +func (st *SimStateStore) Save(s string, data []byte) error { + st.m[s] = data return nil } diff --git a/p2p/simulations/network.go b/p2p/simulations/network.go index 0983e0a85a..1a2c1e8ff4 100644 --- a/p2p/simulations/network.go +++ b/p2p/simulations/network.go @@ -74,22 +74,22 @@ func NewNetwork(nodeAdapter adapters.NodeAdapter, conf *NetworkConfig) *Network } // Events returns the output event feed of the Network. -func (self *Network) Events() *event.Feed { - return &self.events +func (net *Network) Events() *event.Feed { + return &net.events } // NewNode adds a new node to the network with a random ID -func (self *Network) NewNode() (*Node, error) { +func (net *Network) NewNode() (*Node, error) { conf := adapters.RandomNodeConfig() - conf.Services = []string{self.DefaultService} - return self.NewNodeWithConfig(conf) + conf.Services = []string{net.DefaultService} + return net.NewNodeWithConfig(conf) } // NewNodeWithConfig adds a new node to the network with the given config, // returning an error if a node with the same ID or name already exists -func (self *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error) { - self.lock.Lock() - defer self.lock.Unlock() +func (net *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error) { + net.lock.Lock() + defer net.lock.Unlock() // create a random ID and PrivateKey if not set if conf.ID == (discover.NodeID{}) { @@ -100,31 +100,31 @@ func (self *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error) id := conf.ID if conf.Reachable == nil { conf.Reachable = func(otherID discover.NodeID) bool { - _, err := self.InitConn(conf.ID, otherID) + _, err := net.InitConn(conf.ID, otherID) return err == nil } } // assign a name to the node if not set if conf.Name == "" { - conf.Name = fmt.Sprintf("node%02d", len(self.Nodes)+1) + conf.Name = fmt.Sprintf("node%02d", len(net.Nodes)+1) } // check the node doesn't already exist - if node := self.getNode(id); node != nil { + if node := net.getNode(id); node != nil { return nil, fmt.Errorf("node with ID %q already exists", id) } - if node := self.getNodeByName(conf.Name); node != nil { + if node := net.getNodeByName(conf.Name); node != nil { return nil, fmt.Errorf("node with name %q already exists", conf.Name) } // if no services are configured, use the default service if len(conf.Services) == 0 { - conf.Services = []string{self.DefaultService} + conf.Services = []string{net.DefaultService} } // use the NodeAdapter to create the node - adapterNode, err := self.nodeAdapter.NewNode(conf) + adapterNode, err := net.nodeAdapter.NewNode(conf) if err != nil { return nil, err } @@ -133,27 +133,27 @@ func (self *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error) Config: conf, } log.Trace(fmt.Sprintf("node %v created", id)) - self.nodeMap[id] = len(self.Nodes) - self.Nodes = append(self.Nodes, node) + net.nodeMap[id] = len(net.Nodes) + net.Nodes = append(net.Nodes, node) // emit a "control" event - self.events.Send(ControlEvent(node)) + net.events.Send(ControlEvent(node)) return node, nil } // Config returns the network configuration -func (self *Network) Config() *NetworkConfig { - return &self.NetworkConfig +func (net *Network) Config() *NetworkConfig { + return &net.NetworkConfig } // StartAll starts all nodes in the network -func (self *Network) StartAll() error { - for _, node := range self.Nodes { +func (net *Network) StartAll() error { + for _, node := range net.Nodes { if node.Up { continue } - if err := self.Start(node.ID()); err != nil { + if err := net.Start(node.ID()); err != nil { return err } } @@ -161,12 +161,12 @@ func (self *Network) StartAll() error { } // StopAll stops all nodes in the network -func (self *Network) StopAll() error { - for _, node := range self.Nodes { +func (net *Network) StopAll() error { + for _, node := range net.Nodes { if !node.Up { continue } - if err := self.Stop(node.ID()); err != nil { + if err := net.Stop(node.ID()); err != nil { return err } } @@ -174,21 +174,21 @@ func (self *Network) StopAll() error { } // Start starts the node with the given ID -func (self *Network) Start(id discover.NodeID) error { - return self.startWithSnapshots(id, nil) +func (net *Network) Start(id discover.NodeID) error { + return net.startWithSnapshots(id, nil) } // startWithSnapshots starts the node with the given ID using the give // snapshots -func (self *Network) startWithSnapshots(id discover.NodeID, snapshots map[string][]byte) error { - node := self.GetNode(id) +func (net *Network) startWithSnapshots(id discover.NodeID, snapshots map[string][]byte) error { + node := net.GetNode(id) if node == nil { return fmt.Errorf("node %v does not exist", id) } if node.Up { return fmt.Errorf("node %v already up", id) } - log.Trace(fmt.Sprintf("starting node %v: %v using %v", id, node.Up, self.nodeAdapter.Name())) + log.Trace(fmt.Sprintf("starting node %v: %v using %v", id, node.Up, net.nodeAdapter.Name())) if err := node.Start(snapshots); err != nil { log.Warn(fmt.Sprintf("start up failed: %v", err)) return err @@ -196,7 +196,7 @@ func (self *Network) startWithSnapshots(id discover.NodeID, snapshots map[string node.Up = true log.Info(fmt.Sprintf("started node %v: %v", id, node.Up)) - self.events.Send(NewEvent(node)) + net.events.Send(NewEvent(node)) // subscribe to peer events client, err := node.Client() @@ -208,22 +208,22 @@ func (self *Network) startWithSnapshots(id discover.NodeID, snapshots map[string if err != nil { return fmt.Errorf("error getting peer events for node %v: %s", id, err) } - go self.watchPeerEvents(id, events, sub) + go net.watchPeerEvents(id, events, sub) return nil } // watchPeerEvents reads peer events from the given channel and emits // corresponding network events -func (self *Network) watchPeerEvents(id discover.NodeID, events chan *p2p.PeerEvent, sub event.Subscription) { +func (net *Network) watchPeerEvents(id discover.NodeID, events chan *p2p.PeerEvent, sub event.Subscription) { defer func() { sub.Unsubscribe() // assume the node is now down - self.lock.Lock() - node := self.getNode(id) + net.lock.Lock() + node := net.getNode(id) node.Up = false - self.lock.Unlock() - self.events.Send(NewEvent(node)) + net.lock.Unlock() + net.events.Send(NewEvent(node)) }() for { select { @@ -235,16 +235,16 @@ func (self *Network) watchPeerEvents(id discover.NodeID, events chan *p2p.PeerEv switch event.Type { case p2p.PeerEventTypeAdd: - self.DidConnect(id, peer) + net.DidConnect(id, peer) case p2p.PeerEventTypeDrop: - self.DidDisconnect(id, peer) + net.DidDisconnect(id, peer) case p2p.PeerEventTypeMsgSend: - self.DidSend(id, peer, event.Protocol, *event.MsgCode) + net.DidSend(id, peer, event.Protocol, *event.MsgCode) case p2p.PeerEventTypeMsgRecv: - self.DidReceive(peer, id, event.Protocol, *event.MsgCode) + net.DidReceive(peer, id, event.Protocol, *event.MsgCode) } @@ -258,8 +258,8 @@ func (self *Network) watchPeerEvents(id discover.NodeID, events chan *p2p.PeerEv } // Stop stops the node with the given ID -func (self *Network) Stop(id discover.NodeID) error { - node := self.GetNode(id) +func (net *Network) Stop(id discover.NodeID) error { + node := net.GetNode(id) if node == nil { return fmt.Errorf("node %v does not exist", id) } @@ -272,15 +272,15 @@ func (self *Network) Stop(id discover.NodeID) error { node.Up = false log.Info(fmt.Sprintf("stop node %v: %v", id, node.Up)) - self.events.Send(ControlEvent(node)) + net.events.Send(ControlEvent(node)) return nil } // Connect connects two nodes together by calling the "admin_addPeer" RPC // method on the "one" node so that it connects to the "other" node -func (self *Network) Connect(oneID, otherID discover.NodeID) error { +func (net *Network) Connect(oneID, otherID discover.NodeID) error { log.Debug(fmt.Sprintf("connecting %s to %s", oneID, otherID)) - conn, err := self.InitConn(oneID, otherID) + conn, err := net.InitConn(oneID, otherID) if err != nil { return err } @@ -288,14 +288,14 @@ func (self *Network) Connect(oneID, otherID discover.NodeID) error { if err != nil { return err } - self.events.Send(ControlEvent(conn)) + net.events.Send(ControlEvent(conn)) return client.Call(nil, "admin_addPeer", string(conn.other.Addr())) } // Disconnect disconnects two nodes by calling the "admin_removePeer" RPC // method on the "one" node so that it disconnects from the "other" node -func (self *Network) Disconnect(oneID, otherID discover.NodeID) error { - conn := self.GetConn(oneID, otherID) +func (net *Network) Disconnect(oneID, otherID discover.NodeID) error { + conn := net.GetConn(oneID, otherID) if conn == nil { return fmt.Errorf("connection between %v and %v does not exist", oneID, otherID) } @@ -306,13 +306,13 @@ func (self *Network) Disconnect(oneID, otherID discover.NodeID) error { if err != nil { return err } - self.events.Send(ControlEvent(conn)) + net.events.Send(ControlEvent(conn)) return client.Call(nil, "admin_removePeer", string(conn.other.Addr())) } // DidConnect tracks the fact that the "one" node connected to the "other" node -func (self *Network) DidConnect(one, other discover.NodeID) error { - conn, err := self.GetOrCreateConn(one, other) +func (net *Network) DidConnect(one, other discover.NodeID) error { + conn, err := net.GetOrCreateConn(one, other) if err != nil { return fmt.Errorf("connection between %v and %v does not exist", one, other) } @@ -320,14 +320,14 @@ func (self *Network) DidConnect(one, other discover.NodeID) error { return fmt.Errorf("%v and %v already connected", one, other) } conn.Up = true - self.events.Send(NewEvent(conn)) + net.events.Send(NewEvent(conn)) return nil } // DidDisconnect tracks the fact that the "one" node disconnected from the // "other" node -func (self *Network) DidDisconnect(one, other discover.NodeID) error { - conn := self.GetConn(one, other) +func (net *Network) DidDisconnect(one, other discover.NodeID) error { + conn := net.GetConn(one, other) if conn == nil { return fmt.Errorf("connection between %v and %v does not exist", one, other) } @@ -336,12 +336,12 @@ func (self *Network) DidDisconnect(one, other discover.NodeID) error { } conn.Up = false conn.initiated = time.Now().Add(-dialBanTimeout) - self.events.Send(NewEvent(conn)) + net.events.Send(NewEvent(conn)) return nil } // DidSend tracks the fact that "sender" sent a message to "receiver" -func (self *Network) DidSend(sender, receiver discover.NodeID, proto string, code uint64) error { +func (net *Network) DidSend(sender, receiver discover.NodeID, proto string, code uint64) error { msg := &Msg{ One: sender, Other: receiver, @@ -349,12 +349,12 @@ func (self *Network) DidSend(sender, receiver discover.NodeID, proto string, cod Code: code, Received: false, } - self.events.Send(NewEvent(msg)) + net.events.Send(NewEvent(msg)) return nil } // DidReceive tracks the fact that "receiver" received a message from "sender" -func (self *Network) DidReceive(sender, receiver discover.NodeID, proto string, code uint64) error { +func (net *Network) DidReceive(sender, receiver discover.NodeID, proto string, code uint64) error { msg := &Msg{ One: sender, Other: receiver, @@ -362,36 +362,36 @@ func (self *Network) DidReceive(sender, receiver discover.NodeID, proto string, Code: code, Received: true, } - self.events.Send(NewEvent(msg)) + net.events.Send(NewEvent(msg)) return nil } // GetNode gets the node with the given ID, returning nil if the node does not // exist -func (self *Network) GetNode(id discover.NodeID) *Node { - self.lock.Lock() - defer self.lock.Unlock() - return self.getNode(id) +func (net *Network) GetNode(id discover.NodeID) *Node { + net.lock.Lock() + defer net.lock.Unlock() + return net.getNode(id) } // GetNode gets the node with the given name, returning nil if the node does // not exist -func (self *Network) GetNodeByName(name string) *Node { - self.lock.Lock() - defer self.lock.Unlock() - return self.getNodeByName(name) +func (net *Network) GetNodeByName(name string) *Node { + net.lock.Lock() + defer net.lock.Unlock() + return net.getNodeByName(name) } -func (self *Network) getNode(id discover.NodeID) *Node { - i, found := self.nodeMap[id] +func (net *Network) getNode(id discover.NodeID) *Node { + i, found := net.nodeMap[id] if !found { return nil } - return self.Nodes[i] + return net.Nodes[i] } -func (self *Network) getNodeByName(name string) *Node { - for _, node := range self.Nodes { +func (net *Network) getNodeByName(name string) *Node { + for _, node := range net.Nodes { if node.Config.Name == name { return node } @@ -400,40 +400,40 @@ func (self *Network) getNodeByName(name string) *Node { } // GetNodes returns the existing nodes -func (self *Network) GetNodes() (nodes []*Node) { - self.lock.Lock() - defer self.lock.Unlock() +func (net *Network) GetNodes() (nodes []*Node) { + net.lock.Lock() + defer net.lock.Unlock() - nodes = append(nodes, self.Nodes...) + nodes = append(nodes, net.Nodes...) return nodes } // GetConn returns the connection which exists between "one" and "other" // regardless of which node initiated the connection -func (self *Network) GetConn(oneID, otherID discover.NodeID) *Conn { - self.lock.Lock() - defer self.lock.Unlock() - return self.getConn(oneID, otherID) +func (net *Network) GetConn(oneID, otherID discover.NodeID) *Conn { + net.lock.Lock() + defer net.lock.Unlock() + return net.getConn(oneID, otherID) } // GetOrCreateConn is like GetConn but creates the connection if it doesn't // already exist -func (self *Network) GetOrCreateConn(oneID, otherID discover.NodeID) (*Conn, error) { - self.lock.Lock() - defer self.lock.Unlock() - return self.getOrCreateConn(oneID, otherID) +func (net *Network) GetOrCreateConn(oneID, otherID discover.NodeID) (*Conn, error) { + net.lock.Lock() + defer net.lock.Unlock() + return net.getOrCreateConn(oneID, otherID) } -func (self *Network) getOrCreateConn(oneID, otherID discover.NodeID) (*Conn, error) { - if conn := self.getConn(oneID, otherID); conn != nil { +func (net *Network) getOrCreateConn(oneID, otherID discover.NodeID) (*Conn, error) { + if conn := net.getConn(oneID, otherID); conn != nil { return conn, nil } - one := self.getNode(oneID) + one := net.getNode(oneID) if one == nil { return nil, fmt.Errorf("node %v does not exist", oneID) } - other := self.getNode(otherID) + other := net.getNode(otherID) if other == nil { return nil, fmt.Errorf("node %v does not exist", otherID) } @@ -444,18 +444,18 @@ func (self *Network) getOrCreateConn(oneID, otherID discover.NodeID) (*Conn, err other: other, } label := ConnLabel(oneID, otherID) - self.connMap[label] = len(self.Conns) - self.Conns = append(self.Conns, conn) + net.connMap[label] = len(net.Conns) + net.Conns = append(net.Conns, conn) return conn, nil } -func (self *Network) getConn(oneID, otherID discover.NodeID) *Conn { +func (net *Network) getConn(oneID, otherID discover.NodeID) *Conn { label := ConnLabel(oneID, otherID) - i, found := self.connMap[label] + i, found := net.connMap[label] if !found { return nil } - return self.Conns[i] + return net.Conns[i] } // InitConn(one, other) retrieves the connectiton model for the connection between @@ -466,13 +466,13 @@ func (self *Network) getConn(oneID, otherID discover.NodeID) *Conn { // it also checks whether there has been recent attempt to connect the peers // this is cheating as the simulation is used as an oracle and know about // remote peers attempt to connect to a node which will then not initiate the connection -func (self *Network) InitConn(oneID, otherID discover.NodeID) (*Conn, error) { - self.lock.Lock() - defer self.lock.Unlock() +func (net *Network) InitConn(oneID, otherID discover.NodeID) (*Conn, error) { + net.lock.Lock() + defer net.lock.Unlock() if oneID == otherID { return nil, fmt.Errorf("refusing to connect to self %v", oneID) } - conn, err := self.getOrCreateConn(oneID, otherID) + conn, err := net.getOrCreateConn(oneID, otherID) if err != nil { return nil, err } @@ -491,28 +491,28 @@ func (self *Network) InitConn(oneID, otherID discover.NodeID) (*Conn, error) { } // Shutdown stops all nodes in the network and closes the quit channel -func (self *Network) Shutdown() { - for _, node := range self.Nodes { +func (net *Network) Shutdown() { + for _, node := range net.Nodes { log.Debug(fmt.Sprintf("stopping node %s", node.ID().TerminalString())) if err := node.Stop(); err != nil { log.Warn(fmt.Sprintf("error stopping node %s", node.ID().TerminalString()), "err", err) } } - close(self.quitc) + close(net.quitc) } //Reset resets all network properties: //emtpies the nodes and the connection list -func (self *Network) Reset() { - self.lock.Lock() - defer self.lock.Unlock() +func (net *Network) Reset() { + net.lock.Lock() + defer net.lock.Unlock() //re-initialize the maps - self.connMap = make(map[string]int) - self.nodeMap = make(map[discover.NodeID]int) + net.connMap = make(map[string]int) + net.nodeMap = make(map[discover.NodeID]int) - self.Nodes = nil - self.Conns = nil + net.Nodes = nil + net.Conns = nil } // Node is a wrapper around adapters.Node which is used to track the status @@ -528,37 +528,37 @@ type Node struct { } // ID returns the ID of the node -func (self *Node) ID() discover.NodeID { - return self.Config.ID +func (n *Node) ID() discover.NodeID { + return n.Config.ID } // String returns a log-friendly string -func (self *Node) String() string { - return fmt.Sprintf("Node %v", self.ID().TerminalString()) +func (n *Node) String() string { + return fmt.Sprintf("Node %v", n.ID().TerminalString()) } // NodeInfo returns information about the node -func (self *Node) NodeInfo() *p2p.NodeInfo { +func (n *Node) NodeInfo() *p2p.NodeInfo { // avoid a panic if the node is not started yet - if self.Node == nil { + if n.Node == nil { return nil } - info := self.Node.NodeInfo() - info.Name = self.Config.Name + info := n.Node.NodeInfo() + info.Name = n.Config.Name return info } // MarshalJSON implements the json.Marshaler interface so that the encoded // JSON includes the NodeInfo -func (self *Node) MarshalJSON() ([]byte, error) { +func (n *Node) MarshalJSON() ([]byte, error) { return json.Marshal(struct { Info *p2p.NodeInfo `json:"info,omitempty"` Config *adapters.NodeConfig `json:"config,omitempty"` Up bool `json:"up"` }{ - Info: self.NodeInfo(), - Config: self.Config, - Up: self.Up, + Info: n.NodeInfo(), + Config: n.Config, + Up: n.Up, }) } @@ -580,19 +580,19 @@ type Conn struct { } // nodesUp returns whether both nodes are currently up -func (self *Conn) nodesUp() error { - if !self.one.Up { - return fmt.Errorf("one %v is not up", self.One) +func (c *Conn) nodesUp() error { + if !c.one.Up { + return fmt.Errorf("one %v is not up", c.One) } - if !self.other.Up { - return fmt.Errorf("other %v is not up", self.Other) + if !c.other.Up { + return fmt.Errorf("other %v is not up", c.Other) } return nil } // String returns a log-friendly string -func (self *Conn) String() string { - return fmt.Sprintf("Conn %v->%v", self.One.TerminalString(), self.Other.TerminalString()) +func (c *Conn) String() string { + return fmt.Sprintf("Conn %v->%v", c.One.TerminalString(), c.Other.TerminalString()) } // Msg represents a p2p message sent between two nodes in the network @@ -605,8 +605,8 @@ type Msg struct { } // String returns a log-friendly string -func (self *Msg) String() string { - return fmt.Sprintf("Msg(%d) %v->%v", self.Code, self.One.TerminalString(), self.Other.TerminalString()) +func (m *Msg) String() string { + return fmt.Sprintf("Msg(%d) %v->%v", m.Code, m.One.TerminalString(), m.Other.TerminalString()) } // ConnLabel generates a deterministic string which represents a connection @@ -640,14 +640,14 @@ type NodeSnapshot struct { } // Snapshot creates a network snapshot -func (self *Network) Snapshot() (*Snapshot, error) { - self.lock.Lock() - defer self.lock.Unlock() +func (net *Network) Snapshot() (*Snapshot, error) { + net.lock.Lock() + defer net.lock.Unlock() snap := &Snapshot{ - Nodes: make([]NodeSnapshot, len(self.Nodes)), - Conns: make([]Conn, len(self.Conns)), + Nodes: make([]NodeSnapshot, len(net.Nodes)), + Conns: make([]Conn, len(net.Conns)), } - for i, node := range self.Nodes { + for i, node := range net.Nodes { snap.Nodes[i] = NodeSnapshot{Node: *node} if !node.Up { continue @@ -658,33 +658,33 @@ func (self *Network) Snapshot() (*Snapshot, error) { } snap.Nodes[i].Snapshots = snapshots } - for i, conn := range self.Conns { + for i, conn := range net.Conns { snap.Conns[i] = *conn } return snap, nil } // Load loads a network snapshot -func (self *Network) Load(snap *Snapshot) error { +func (net *Network) Load(snap *Snapshot) error { for _, n := range snap.Nodes { - if _, err := self.NewNodeWithConfig(n.Node.Config); err != nil { + if _, err := net.NewNodeWithConfig(n.Node.Config); err != nil { return err } if !n.Node.Up { continue } - if err := self.startWithSnapshots(n.Node.Config.ID, n.Snapshots); err != nil { + if err := net.startWithSnapshots(n.Node.Config.ID, n.Snapshots); err != nil { return err } } for _, conn := range snap.Conns { - if !self.GetNode(conn.One).Up || !self.GetNode(conn.Other).Up { + if !net.GetNode(conn.One).Up || !net.GetNode(conn.Other).Up { //in this case, at least one of the nodes of a connection is not up, //so it would result in the snapshot `Load` to fail continue } - if err := self.Connect(conn.One, conn.Other); err != nil { + if err := net.Connect(conn.One, conn.Other); err != nil { return err } } @@ -692,7 +692,7 @@ func (self *Network) Load(snap *Snapshot) error { } // Subscribe reads control events from a channel and executes them -func (self *Network) Subscribe(events chan *Event) { +func (net *Network) Subscribe(events chan *Event) { for { select { case event, ok := <-events: @@ -700,23 +700,23 @@ func (self *Network) Subscribe(events chan *Event) { return } if event.Control { - self.executeControlEvent(event) + net.executeControlEvent(event) } - case <-self.quitc: + case <-net.quitc: return } } } -func (self *Network) executeControlEvent(event *Event) { +func (net *Network) executeControlEvent(event *Event) { log.Trace("execute control event", "type", event.Type, "event", event) switch event.Type { case EventTypeNode: - if err := self.executeNodeEvent(event); err != nil { + if err := net.executeNodeEvent(event); err != nil { log.Error("error executing node event", "event", event, "err", err) } case EventTypeConn: - if err := self.executeConnEvent(event); err != nil { + if err := net.executeConnEvent(event); err != nil { log.Error("error executing conn event", "event", event, "err", err) } case EventTypeMsg: @@ -724,20 +724,21 @@ func (self *Network) executeControlEvent(event *Event) { } } -func (self *Network) executeNodeEvent(e *Event) error { +func (net *Network) executeNodeEvent(e *Event) error { if !e.Node.Up { - return self.Stop(e.Node.ID()) + return net.Stop(e.Node.ID()) } - if _, err := self.NewNodeWithConfig(e.Node.Config); err != nil { + if _, err := net.NewNodeWithConfig(e.Node.Config); err != nil { return err } - return self.Start(e.Node.ID()) + return net.Start(e.Node.ID()) } -func (self *Network) executeConnEvent(e *Event) error { +func (net *Network) executeConnEvent(e *Event) error { if e.Conn.Up { - return self.Connect(e.Conn.One, e.Conn.Other) + return net.Connect(e.Conn.One, e.Conn.Other) + } else { + return net.Disconnect(e.Conn.One, e.Conn.Other) } - return self.Disconnect(e.Conn.One, e.Conn.Other) } diff --git a/p2p/testing/peerpool.go b/p2p/testing/peerpool.go index 45c6e61425..ed00396e23 100644 --- a/p2p/testing/peerpool.go +++ b/p2p/testing/peerpool.go @@ -39,29 +39,29 @@ func NewTestPeerPool() *TestPeerPool { return &TestPeerPool{peers: make(map[discover.NodeID]TestPeer)} } -func (self *TestPeerPool) Add(p TestPeer) { - self.lock.Lock() - defer self.lock.Unlock() - log.Trace(fmt.Sprintf("pp add peer %v", p.ID())) - self.peers[p.ID()] = p +func (p *TestPeerPool) Add(peer TestPeer) { + p.lock.Lock() + defer p.lock.Unlock() + log.Trace(fmt.Sprintf("pp add peer %v", peer.ID())) + p.peers[peer.ID()] = peer } -func (self *TestPeerPool) Remove(p TestPeer) { - self.lock.Lock() - defer self.lock.Unlock() - delete(self.peers, p.ID()) +func (p *TestPeerPool) Remove(peer TestPeer) { + p.lock.Lock() + defer p.lock.Unlock() + delete(p.peers, peer.ID()) } -func (self *TestPeerPool) Has(id discover.NodeID) bool { - self.lock.Lock() - defer self.lock.Unlock() - _, ok := self.peers[id] +func (p *TestPeerPool) Has(id discover.NodeID) bool { + p.lock.Lock() + defer p.lock.Unlock() + _, ok := p.peers[id] return ok } -func (self *TestPeerPool) Get(id discover.NodeID) TestPeer { - self.lock.Lock() - defer self.lock.Unlock() - return self.peers[id] +func (p *TestPeerPool) Get(id discover.NodeID) TestPeer { + p.lock.Lock() + defer p.lock.Unlock() + return p.peers[id] } diff --git a/p2p/testing/protocolsession.go b/p2p/testing/protocolsession.go index 361285f06e..8f73bfa03e 100644 --- a/p2p/testing/protocolsession.go +++ b/p2p/testing/protocolsession.go @@ -78,10 +78,10 @@ type Disconnect struct { } // trigger sends messages from peers -func (self *ProtocolSession) trigger(trig Trigger) error { - simNode, ok := self.adapter.GetNode(trig.Peer) +func (s *ProtocolSession) trigger(trig Trigger) error { + simNode, ok := s.adapter.GetNode(trig.Peer) if !ok { - return fmt.Errorf("trigger: peer %v does not exist (1- %v)", trig.Peer, len(self.IDs)) + return fmt.Errorf("trigger: peer %v does not exist (1- %v)", trig.Peer, len(s.IDs)) } mockNode, ok := simNode.Services()[0].(*mockNode) if !ok { @@ -107,7 +107,7 @@ func (self *ProtocolSession) trigger(trig Trigger) error { } // expect checks an expectation of a message sent out by the pivot node -func (self *ProtocolSession) expect(exps []Expect) error { +func (s *ProtocolSession) expect(exps []Expect) error { // construct a map of expectations for each node peerExpects := make(map[discover.NodeID][]Expect) for _, exp := range exps { @@ -120,9 +120,9 @@ func (self *ProtocolSession) expect(exps []Expect) error { // construct a map of mockNodes for each node mockNodes := make(map[discover.NodeID]*mockNode) for nodeID := range peerExpects { - simNode, ok := self.adapter.GetNode(nodeID) + simNode, ok := s.adapter.GetNode(nodeID) if !ok { - return fmt.Errorf("trigger: peer %v does not exist (1- %v)", nodeID, len(self.IDs)) + return fmt.Errorf("trigger: peer %v does not exist (1- %v)", nodeID, len(s.IDs)) } mockNode, ok := simNode.Services()[0].(*mockNode) if !ok { @@ -202,9 +202,9 @@ func (self *ProtocolSession) expect(exps []Expect) error { } // TestExchanges tests a series of exchanges against the session -func (self *ProtocolSession) TestExchanges(exchanges ...Exchange) error { +func (s *ProtocolSession) TestExchanges(exchanges ...Exchange) error { for i, e := range exchanges { - if err := self.testExchange(e); err != nil { + if err := s.testExchange(e); err != nil { return fmt.Errorf("exchange #%d %q: %v", i, e.Label, err) } log.Trace(fmt.Sprintf("exchange #%d %q: run successfully", i, e.Label)) @@ -214,14 +214,14 @@ func (self *ProtocolSession) TestExchanges(exchanges ...Exchange) error { // testExchange tests a single Exchange. // Default timeout value is 2 seconds. -func (self *ProtocolSession) testExchange(e Exchange) error { +func (s *ProtocolSession) testExchange(e Exchange) error { errc := make(chan error) done := make(chan struct{}) defer close(done) go func() { for _, trig := range e.Triggers { - err := self.trigger(trig) + err := s.trigger(trig) if err != nil { errc <- err return @@ -229,7 +229,7 @@ func (self *ProtocolSession) testExchange(e Exchange) error { } select { - case errc <- self.expect(e.Expects): + case errc <- s.expect(e.Expects): case <-done: } }() @@ -250,7 +250,7 @@ func (self *ProtocolSession) testExchange(e Exchange) error { // TestDisconnected tests the disconnections given as arguments // the disconnect structs describe what disconnect error is expected on which peer -func (self *ProtocolSession) TestDisconnected(disconnects ...*Disconnect) error { +func (s *ProtocolSession) TestDisconnected(disconnects ...*Disconnect) error { expects := make(map[discover.NodeID]error) for _, disconnect := range disconnects { expects[disconnect.Peer] = disconnect.Error @@ -259,7 +259,7 @@ func (self *ProtocolSession) TestDisconnected(disconnects ...*Disconnect) error timeout := time.After(time.Second) for len(expects) > 0 { select { - case event := <-self.events: + case event := <-s.events: if event.Type != p2p.PeerEventTypeDrop { continue } diff --git a/p2p/testing/protocoltester.go b/p2p/testing/protocoltester.go index a797412d60..636613c57a 100644 --- a/p2p/testing/protocoltester.go +++ b/p2p/testing/protocoltester.go @@ -101,24 +101,24 @@ func NewProtocolTester(t *testing.T, id discover.NodeID, n int, run func(*p2p.Pe } // Stop stops the p2p server -func (self *ProtocolTester) Stop() error { - self.Server.Stop() +func (t *ProtocolTester) Stop() error { + t.Server.Stop() return nil } // Connect brings up the remote peer node and connects it using the // p2p/simulations network connection with the in memory network adapter -func (self *ProtocolTester) Connect(selfID discover.NodeID, peers ...*adapters.NodeConfig) { +func (t *ProtocolTester) Connect(selfID discover.NodeID, peers ...*adapters.NodeConfig) { for _, peer := range peers { log.Trace(fmt.Sprintf("start node %v", peer.ID)) - if _, err := self.network.NewNodeWithConfig(peer); err != nil { + if _, err := t.network.NewNodeWithConfig(peer); err != nil { panic(fmt.Sprintf("error starting peer %v: %v", peer.ID, err)) } - if err := self.network.Start(peer.ID); err != nil { + if err := t.network.Start(peer.ID); err != nil { panic(fmt.Sprintf("error starting peer %v: %v", peer.ID, err)) } log.Trace(fmt.Sprintf("connect to %v", peer.ID)) - if err := self.network.Connect(selfID, peer.ID); err != nil { + if err := t.network.Connect(selfID, peer.ID); err != nil { panic(fmt.Sprintf("error connecting to peer %v: %v", peer.ID, err)) } } From fedae9501582cb64c850def557b1fc3742d892dc Mon Sep 17 00:00:00 2001 From: Domino Valdano Date: Tue, 8 May 2018 05:39:15 -0700 Subject: [PATCH 03/46] eth/filters: derive FilterCriteria from ethereum.FilterQuery (#16629) --- eth/filters/api.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index ec403709c5..1297b74788 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -268,14 +268,8 @@ func (api *PublicFilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc } // FilterCriteria represents a request to create a new filter. -// -// TODO(karalabe): Kill this in favor of ethereum.FilterQuery. -type FilterCriteria struct { - FromBlock *big.Int - ToBlock *big.Int - Addresses []common.Address - Topics [][]common.Hash -} +// Same as ethereum.FilterQuery but with UnmarshalJSON() method. +type FilterCriteria ethereum.FilterQuery // NewFilter creates a new filter and returns the filter id. It can be // used to retrieve logs when the state changes. This method cannot be From c4a4613d9504db43a26a3c79dda8bf6be0d1237a Mon Sep 17 00:00:00 2001 From: Ivan Daniluk Date: Tue, 8 May 2018 18:05:27 +0300 Subject: [PATCH 04/46] p2p/simulations/adapters: fix websocket log line parsing in exec adapter (#16667) --- p2p/simulations/adapters/exec.go | 26 ++------------- p2p/simulations/adapters/ws.go | 51 +++++++++++++++++++++++++++++ p2p/simulations/adapters/ws_test.go | 21 ++++++++++++ 3 files changed, 75 insertions(+), 23 deletions(-) create mode 100644 p2p/simulations/adapters/ws.go create mode 100644 p2p/simulations/adapters/ws_test.go diff --git a/p2p/simulations/adapters/exec.go b/p2p/simulations/adapters/exec.go index a566fb27d8..f381c11596 100644 --- a/p2p/simulations/adapters/exec.go +++ b/p2p/simulations/adapters/exec.go @@ -17,7 +17,6 @@ package adapters import ( - "bufio" "context" "crypto/ecdsa" "encoding/json" @@ -29,7 +28,6 @@ import ( "os/exec" "os/signal" "path/filepath" - "regexp" "strings" "sync" "syscall" @@ -150,10 +148,6 @@ func (n *ExecNode) Client() (*rpc.Client, error) { return n.client, nil } -// wsAddrPattern is a regex used to read the WebSocket address from the node's -// log -var wsAddrPattern = regexp.MustCompile(`ws://[\d.:]+`) - // Start exec's the node passing the ID and service as command line arguments // and the node config encoded as JSON in the _P2P_NODE_CONFIG environment // variable @@ -196,23 +190,9 @@ func (n *ExecNode) Start(snapshots map[string][]byte) (err error) { n.Cmd = cmd // read the WebSocket address from the stderr logs - var wsAddr string - wsAddrC := make(chan string) - go func() { - s := bufio.NewScanner(stderrR) - for s.Scan() { - if strings.Contains(s.Text(), "WebSocket endpoint opened:") { - wsAddrC <- wsAddrPattern.FindString(s.Text()) - } - } - }() - select { - case wsAddr = <-wsAddrC: - if wsAddr == "" { - return errors.New("failed to read WebSocket address from stderr") - } - case <-time.After(10 * time.Second): - return errors.New("timed out waiting for WebSocket address on stderr") + wsAddr, err := findWSAddr(stderrR, 10*time.Second) + if err != nil { + return fmt.Errorf("error getting WebSocket address: %s", err) } // create the RPC client and load the node info diff --git a/p2p/simulations/adapters/ws.go b/p2p/simulations/adapters/ws.go new file mode 100644 index 0000000000..979a21709e --- /dev/null +++ b/p2p/simulations/adapters/ws.go @@ -0,0 +1,51 @@ +package adapters + +import ( + "bufio" + "errors" + "io" + "regexp" + "strings" + "time" +) + +// wsAddrPattern is a regex used to read the WebSocket address from the node's +// log +var wsAddrPattern = regexp.MustCompile(`ws://[\d.:]+`) + +func matchWSAddr(str string) (string, bool) { + if !strings.Contains(str, "WebSocket endpoint opened") { + return "", false + } + + return wsAddrPattern.FindString(str), true +} + +// findWSAddr scans through reader r, looking for the log entry with +// WebSocket address information. +func findWSAddr(r io.Reader, timeout time.Duration) (string, error) { + ch := make(chan string) + + go func() { + s := bufio.NewScanner(r) + for s.Scan() { + addr, ok := matchWSAddr(s.Text()) + if ok { + ch <- addr + } + } + close(ch) + }() + + var wsAddr string + select { + case wsAddr = <-ch: + if wsAddr == "" { + return "", errors.New("empty result") + } + case <-time.After(timeout): + return "", errors.New("timed out") + } + + return wsAddr, nil +} diff --git a/p2p/simulations/adapters/ws_test.go b/p2p/simulations/adapters/ws_test.go new file mode 100644 index 0000000000..0bb9ed2b2b --- /dev/null +++ b/p2p/simulations/adapters/ws_test.go @@ -0,0 +1,21 @@ +package adapters + +import ( + "bytes" + "testing" + "time" +) + +func TestFindWSAddr(t *testing.T) { + line := `t=2018-05-02T19:00:45+0200 lvl=info msg="WebSocket endpoint opened" node.id=26c65a606d1125a44695bc08573190d047152b6b9a776ccbbe593e90f91444d9c1ebdadac6a775ad9fdd0923468a1d698ed3a842c1fb89c1bc0f9d4801f8c39c url=ws://127.0.0.1:59975` + buf := bytes.NewBufferString(line) + got, err := findWSAddr(buf, 10*time.Second) + if err != nil { + t.Fatalf("Failed to find addr: %v", err) + } + expected := `ws://127.0.0.1:59975` + + if got != expected { + t.Fatalf("Expected to get '%s', but got '%s'", expected, got) + } +} From eab6e5a317acf67409f82bc5c1f4d959413dfd47 Mon Sep 17 00:00:00 2001 From: ligi Date: Wed, 9 May 2018 01:13:53 +0200 Subject: [PATCH 05/46] build: specify the key to use when invoking gpg:sign-and-deploy-file (#16696) --- build/ci.go | 21 +++++++++++++-------- internal/build/pgp.go | 12 ++++++++++++ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/build/ci.go b/build/ci.go index 204c206752..79dcc146c3 100644 --- a/build/ci.go +++ b/build/ci.go @@ -755,14 +755,18 @@ func doAndroidArchive(cmdline []string) { os.Rename(archive, meta.Package+".aar") if *signer != "" && *deploy != "" { // Import the signing key into the local GPG instance - if b64key := os.Getenv(*signer); b64key != "" { - key, err := base64.StdEncoding.DecodeString(b64key) - if err != nil { - log.Fatalf("invalid base64 %s", *signer) - } - gpg := exec.Command("gpg", "--import") - gpg.Stdin = bytes.NewReader(key) - build.MustRun(gpg) + b64key := os.Getenv(*signer) + key, err := base64.StdEncoding.DecodeString(b64key) + if err != nil { + log.Fatalf("invalid base64 %s", *signer) + } + gpg := exec.Command("gpg", "--import") + gpg.Stdin = bytes.NewReader(key) + build.MustRun(gpg) + + keyID, err := build.PGPKeyID(string(key)) + if err != nil { + log.Fatal(err) } // Upload the artifacts to Sonatype and/or Maven Central repo := *deploy + "/service/local/staging/deploy/maven2" @@ -771,6 +775,7 @@ func doAndroidArchive(cmdline []string) { } build.MustRunCommand("mvn", "gpg:sign-and-deploy-file", "-e", "-X", "-settings=build/mvn.settings", "-Durl="+repo, "-DrepositoryId=ossrh", + "-Dgpg.keyname="+keyID, "-DpomFile="+meta.Package+".pom", "-Dfile="+meta.Package+".aar") } } diff --git a/internal/build/pgp.go b/internal/build/pgp.go index 79ab9c06f1..c7d0d23397 100644 --- a/internal/build/pgp.go +++ b/internal/build/pgp.go @@ -57,3 +57,15 @@ func PGPSignFile(input string, output string, pgpkey string) error { // Generate the signature and return return openpgp.ArmoredDetachSign(out, keys[0], in, nil) } + +// PGPKeyID parses an armored key and returns the key ID. +func PGPKeyID(pgpkey string) (string, error) { + keys, err := openpgp.ReadArmoredKeyRing(bytes.NewBufferString(pgpkey)) + if err != nil { + return "", err + } + if len(keys) != 1 { + return "", fmt.Errorf("key count mismatch: have %d, want %d", len(keys), 1) + } + return keys[0].PrimaryKey.KeyIdString(), nil +} From ba975dc0931b9f2962b2f163675772458ed339fd Mon Sep 17 00:00:00 2001 From: kiel barry Date: Tue, 8 May 2018 16:17:09 -0700 Subject: [PATCH 06/46] crypto: fix golint warnings (#16710) --- crypto/crypto.go | 12 ++++++------ crypto/crypto_test.go | 8 ++++---- crypto/secp256k1/curve.go | 2 +- crypto/secp256k1/secp256_test.go | 2 +- crypto/signature_nocgo.go | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/crypto/crypto.go b/crypto/crypto.go index 1c4d5a2e02..76d1ffaf64 100644 --- a/crypto/crypto.go +++ b/crypto/crypto.go @@ -35,8 +35,8 @@ import ( ) var ( - secp256k1_N, _ = new(big.Int).SetString("fffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141", 16) - secp256k1_halfN = new(big.Int).Div(secp256k1_N, big.NewInt(2)) + secp256k1N, _ = new(big.Int).SetString("fffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141", 16) + secp256k1halfN = new(big.Int).Div(secp256k1N, big.NewInt(2)) ) // Keccak256 calculates and returns the Keccak256 hash of the input data. @@ -68,7 +68,7 @@ func Keccak512(data ...[]byte) []byte { return d.Sum(nil) } -// Creates an ethereum address given the bytes and the nonce +// CreateAddress creates an ethereum address given the bytes and the nonce func CreateAddress(b common.Address, nonce uint64) common.Address { data, _ := rlp.EncodeToBytes([]interface{}{b, nonce}) return common.BytesToAddress(Keccak256(data)[12:]) @@ -99,7 +99,7 @@ func toECDSA(d []byte, strict bool) (*ecdsa.PrivateKey, error) { priv.D = new(big.Int).SetBytes(d) // The priv.D must < N - if priv.D.Cmp(secp256k1_N) >= 0 { + if priv.D.Cmp(secp256k1N) >= 0 { return nil, fmt.Errorf("invalid private key, >=N") } // The priv.D must not be zero or negative. @@ -184,11 +184,11 @@ func ValidateSignatureValues(v byte, r, s *big.Int, homestead bool) bool { } // reject upper range of s values (ECDSA malleability) // see discussion in secp256k1/libsecp256k1/include/secp256k1.h - if homestead && s.Cmp(secp256k1_halfN) > 0 { + if homestead && s.Cmp(secp256k1halfN) > 0 { return false } // Frontier: allow s to be in full N range - return r.Cmp(secp256k1_N) < 0 && s.Cmp(secp256k1_N) < 0 && (v == 0 || v == 1) + return r.Cmp(secp256k1N) < 0 && s.Cmp(secp256k1N) < 0 && (v == 0 || v == 1) } func PubkeyToAddress(p ecdsa.PublicKey) common.Address { diff --git a/crypto/crypto_test.go b/crypto/crypto_test.go index 8350354623..804de3fe22 100644 --- a/crypto/crypto_test.go +++ b/crypto/crypto_test.go @@ -154,7 +154,7 @@ func TestValidateSignatureValues(t *testing.T) { minusOne := big.NewInt(-1) one := common.Big1 zero := common.Big0 - secp256k1nMinus1 := new(big.Int).Sub(secp256k1_N, common.Big1) + secp256k1nMinus1 := new(big.Int).Sub(secp256k1N, common.Big1) // correct v,r,s check(true, 0, one, one) @@ -181,9 +181,9 @@ func TestValidateSignatureValues(t *testing.T) { // correct sig with max r,s check(true, 0, secp256k1nMinus1, secp256k1nMinus1) // correct v, combinations of incorrect r,s at upper limit - check(false, 0, secp256k1_N, secp256k1nMinus1) - check(false, 0, secp256k1nMinus1, secp256k1_N) - check(false, 0, secp256k1_N, secp256k1_N) + check(false, 0, secp256k1N, secp256k1nMinus1) + check(false, 0, secp256k1nMinus1, secp256k1N) + check(false, 0, secp256k1N, secp256k1N) // current callers ensures r,s cannot be negative, but let's test for that too // as crypto package could be used stand-alone diff --git a/crypto/secp256k1/curve.go b/crypto/secp256k1/curve.go index f51be5e353..6fdf2be6a4 100644 --- a/crypto/secp256k1/curve.go +++ b/crypto/secp256k1/curve.go @@ -77,7 +77,7 @@ func (BitCurve *BitCurve) Params() *elliptic.CurveParams { } } -// IsOnBitCurve returns true if the given (x,y) lies on the BitCurve. +// IsOnCurve returns true if the given (x,y) lies on the BitCurve. func (BitCurve *BitCurve) IsOnCurve(x, y *big.Int) bool { // y² = x³ + b y2 := new(big.Int).Mul(y, y) //y² diff --git a/crypto/secp256k1/secp256_test.go b/crypto/secp256k1/secp256_test.go index f6582ecd54..b608bcfcf1 100644 --- a/crypto/secp256k1/secp256_test.go +++ b/crypto/secp256k1/secp256_test.go @@ -49,7 +49,7 @@ func randSig() []byte { // tests for malleability // highest bit of signature ECDSA s value must be 0, in the 33th byte func compactSigCheck(t *testing.T, sig []byte) { - var b int = int(sig[32]) + var b = int(sig[32]) if b < 0 { t.Errorf("highest bit is negative: %d", b) } diff --git a/crypto/signature_nocgo.go b/crypto/signature_nocgo.go index f636b23772..e8fa18ed47 100644 --- a/crypto/signature_nocgo.go +++ b/crypto/signature_nocgo.go @@ -88,7 +88,7 @@ func VerifySignature(pubkey, hash, signature []byte) bool { return false } // Reject malleable signatures. libsecp256k1 does this check but btcec doesn't. - if sig.S.Cmp(secp256k1_halfN) > 0 { + if sig.S.Cmp(secp256k1halfN) > 0 { return false } return sig.Verify(hash, key) From c60f6f62147048f80a0e092b483318af88b5d78b Mon Sep 17 00:00:00 2001 From: Guilherme Salgado Date: Wed, 9 May 2018 00:20:20 +0100 Subject: [PATCH 07/46] p2p: don't discard reason set by Disconnect (#16559) Peer.run was discarding the reason for disconnection sent to the disc channel by Disconnect. --- p2p/peer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/p2p/peer.go b/p2p/peer.go index 73e33418ed..c3907349fc 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -220,6 +220,7 @@ loop: reason = discReasonForError(err) break loop case err = <-p.disc: + reason = discReasonForError(err) break loop } } From 4ea493e7eb9fa3b3695a22487de2605e8da1d2fa Mon Sep 17 00:00:00 2001 From: kiel barry Date: Wed, 9 May 2018 00:38:03 -0700 Subject: [PATCH 08/46] cmd: various golint fixes (#16700) * cmd: various golint fixes * cmd: update to pr change request * cmd: update to pr change request --- cmd/clef/main.go | 38 +++++++++++++++++++------------------- cmd/evm/json_logger.go | 2 ++ cmd/evm/staterunner.go | 2 ++ cmd/geth/accountcmd.go | 4 ++-- cmd/geth/bugcmd.go | 6 +++--- 5 files changed, 28 insertions(+), 24 deletions(-) diff --git a/cmd/clef/main.go b/cmd/clef/main.go index 9a1ae91acb..348bcb22f6 100644 --- a/cmd/clef/main.go +++ b/cmd/clef/main.go @@ -47,11 +47,11 @@ import ( "gopkg.in/urfave/cli.v1" ) -// ExternalApiVersion -- see extapi_changelog.md -const ExternalApiVersion = "2.0.0" +// ExternalAPIVersion -- see extapi_changelog.md +const ExternalAPIVersion = "2.0.0" -// InternalApiVersion -- see intapi_changelog.md -const InternalApiVersion = "2.0.0" +// InternalAPIVersion -- see intapi_changelog.md +const InternalAPIVersion = "2.0.0" const legalWarning = ` WARNING! @@ -398,10 +398,10 @@ func signer(c *cli.Context) error { } // register signer API with server var ( - extapiUrl = "n/a" - ipcApiUrl = "n/a" + extapiURL = "n/a" + ipcapiURL = "n/a" ) - rpcApi := []rpc.API{ + rpcAPI := []rpc.API{ { Namespace: "account", Public: true, @@ -415,12 +415,12 @@ func signer(c *cli.Context) error { // start http server httpEndpoint := fmt.Sprintf("%s:%d", c.String(utils.RPCListenAddrFlag.Name), c.Int(rpcPortFlag.Name)) - listener, _, err := rpc.StartHTTPEndpoint(httpEndpoint, rpcApi, []string{"account"}, cors, vhosts) + listener, _, err := rpc.StartHTTPEndpoint(httpEndpoint, rpcAPI, []string{"account"}, cors, vhosts) if err != nil { utils.Fatalf("Could not start RPC api: %v", err) } - extapiUrl = fmt.Sprintf("http://%s", httpEndpoint) - log.Info("HTTP endpoint opened", "url", extapiUrl) + extapiURL = fmt.Sprintf("http://%s", httpEndpoint) + log.Info("HTTP endpoint opened", "url", extapiURL) defer func() { listener.Close() @@ -430,19 +430,19 @@ func signer(c *cli.Context) error { } if !c.Bool(utils.IPCDisabledFlag.Name) { if c.IsSet(utils.IPCPathFlag.Name) { - ipcApiUrl = c.String(utils.IPCPathFlag.Name) + ipcapiURL = c.String(utils.IPCPathFlag.Name) } else { - ipcApiUrl = filepath.Join(configDir, "clef.ipc") + ipcapiURL = filepath.Join(configDir, "clef.ipc") } - listener, _, err := rpc.StartIPCEndpoint(ipcApiUrl, rpcApi) + listener, _, err := rpc.StartIPCEndpoint(ipcapiURL, rpcAPI) if err != nil { utils.Fatalf("Could not start IPC api: %v", err) } - log.Info("IPC endpoint opened", "url", ipcApiUrl) + log.Info("IPC endpoint opened", "url", ipcapiURL) defer func() { listener.Close() - log.Info("IPC endpoint closed", "url", ipcApiUrl) + log.Info("IPC endpoint closed", "url", ipcapiURL) }() } @@ -453,10 +453,10 @@ func signer(c *cli.Context) error { } ui.OnSignerStartup(core.StartupInfo{ Info: map[string]interface{}{ - "extapi_version": ExternalApiVersion, - "intapi_version": InternalApiVersion, - "extapi_http": extapiUrl, - "extapi_ipc": ipcApiUrl, + "extapi_version": ExternalAPIVersion, + "intapi_version": InternalAPIVersion, + "extapi_http": extapiURL, + "extapi_ipc": ipcapiURL, }, }) diff --git a/cmd/evm/json_logger.go b/cmd/evm/json_logger.go index 0e7a911896..f16424fbe6 100644 --- a/cmd/evm/json_logger.go +++ b/cmd/evm/json_logger.go @@ -32,6 +32,8 @@ type JSONLogger struct { cfg *vm.LogConfig } +// NewJSONLogger creates a new EVM tracer that prints execution steps as JSON objects +// into the provided stream. func NewJSONLogger(cfg *vm.LogConfig, writer io.Writer) *JSONLogger { return &JSONLogger{json.NewEncoder(writer), cfg} } diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index 071ea94ad0..6d5ff069f6 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -38,6 +38,8 @@ var stateTestCommand = cli.Command{ ArgsUsage: "", } +// StatetestResult contains the execution status after running a state test, any +// error that might have occurred and a dump of the final state if requested. type StatetestResult struct { Name string `json:"name"` Pass bool `json:"pass"` diff --git a/cmd/geth/accountcmd.go b/cmd/geth/accountcmd.go index 0db5c4ce0f..071be85397 100644 --- a/cmd/geth/accountcmd.go +++ b/cmd/geth/accountcmd.go @@ -340,7 +340,7 @@ func importWallet(ctx *cli.Context) error { if len(keyfile) == 0 { utils.Fatalf("keyfile must be given as argument") } - keyJson, err := ioutil.ReadFile(keyfile) + keyJSON, err := ioutil.ReadFile(keyfile) if err != nil { utils.Fatalf("Could not read wallet file: %v", err) } @@ -349,7 +349,7 @@ func importWallet(ctx *cli.Context) error { passphrase := getPassPhrase("", false, 0, utils.MakePasswordList(ctx)) ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore) - acct, err := ks.ImportPreSaleKey(keyJson, passphrase) + acct, err := ks.ImportPreSaleKey(keyJSON, passphrase) if err != nil { utils.Fatalf("%v", err) } diff --git a/cmd/geth/bugcmd.go b/cmd/geth/bugcmd.go index 51187ac90e..7e9a8ccc70 100644 --- a/cmd/geth/bugcmd.go +++ b/cmd/geth/bugcmd.go @@ -41,7 +41,7 @@ var bugCommand = cli.Command{ Category: "MISCELLANEOUS COMMANDS", } -const issueUrl = "https://github.com/ethereum/go-ethereum/issues/new" +const issueURL = "https://github.com/ethereum/go-ethereum/issues/new" // reportBug reports a bug by opening a new URL to the go-ethereum GH issue // tracker and setting default values as the issue body. @@ -58,8 +58,8 @@ func reportBug(ctx *cli.Context) error { fmt.Fprintln(&buff, header) // open a new GH issue - if !browser.Open(issueUrl + "?body=" + url.QueryEscape(buff.String())) { - fmt.Printf("Please file a new issue at %s using this template:\n\n%s", issueUrl, buff.String()) + if !browser.Open(issueURL + "?body=" + url.QueryEscape(buff.String())) { + fmt.Printf("Please file a new issue at %s using this template:\n\n%s", issueURL, buff.String()) } return nil } From 4747aad160ed3721a70a133105c474cb3b66519e Mon Sep 17 00:00:00 2001 From: kiel barry Date: Wed, 9 May 2018 00:59:00 -0700 Subject: [PATCH 09/46] eth: golint fixes to variable names (#16711) --- eth/api_backend.go | 66 ++++++++++++++++++++++---------------------- eth/backend.go | 10 +++---- eth/protocol.go | 6 ++-- ethstats/ethstats.go | 2 +- 4 files changed, 42 insertions(+), 42 deletions(-) diff --git a/eth/api_backend.go b/eth/api_backend.go index 532cd5c334..4ace9b5945 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -37,26 +37,26 @@ import ( "github.com/ethereum/go-ethereum/rpc" ) -// EthApiBackend implements ethapi.Backend for full nodes -type EthApiBackend struct { +// EthAPIBackend implements ethapi.Backend for full nodes +type EthAPIBackend struct { eth *Ethereum gpo *gasprice.Oracle } -func (b *EthApiBackend) ChainConfig() *params.ChainConfig { +func (b *EthAPIBackend) ChainConfig() *params.ChainConfig { return b.eth.chainConfig } -func (b *EthApiBackend) CurrentBlock() *types.Block { +func (b *EthAPIBackend) CurrentBlock() *types.Block { return b.eth.blockchain.CurrentBlock() } -func (b *EthApiBackend) SetHead(number uint64) { +func (b *EthAPIBackend) SetHead(number uint64) { b.eth.protocolManager.downloader.Cancel() b.eth.blockchain.SetHead(number) } -func (b *EthApiBackend) HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) { +func (b *EthAPIBackend) HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) { // Pending block is only known by the miner if blockNr == rpc.PendingBlockNumber { block := b.eth.miner.PendingBlock() @@ -69,7 +69,7 @@ func (b *EthApiBackend) HeaderByNumber(ctx context.Context, blockNr rpc.BlockNum return b.eth.blockchain.GetHeaderByNumber(uint64(blockNr)), nil } -func (b *EthApiBackend) BlockByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Block, error) { +func (b *EthAPIBackend) BlockByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Block, error) { // Pending block is only known by the miner if blockNr == rpc.PendingBlockNumber { block := b.eth.miner.PendingBlock() @@ -82,7 +82,7 @@ func (b *EthApiBackend) BlockByNumber(ctx context.Context, blockNr rpc.BlockNumb return b.eth.blockchain.GetBlockByNumber(uint64(blockNr)), nil } -func (b *EthApiBackend) StateAndHeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*state.StateDB, *types.Header, error) { +func (b *EthAPIBackend) StateAndHeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*state.StateDB, *types.Header, error) { // Pending state is only known by the miner if blockNr == rpc.PendingBlockNumber { block, state := b.eth.miner.Pending() @@ -97,18 +97,18 @@ func (b *EthApiBackend) StateAndHeaderByNumber(ctx context.Context, blockNr rpc. return stateDb, header, err } -func (b *EthApiBackend) GetBlock(ctx context.Context, hash common.Hash) (*types.Block, error) { +func (b *EthAPIBackend) GetBlock(ctx context.Context, hash common.Hash) (*types.Block, error) { return b.eth.blockchain.GetBlockByHash(hash), nil } -func (b *EthApiBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { +func (b *EthAPIBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { if number := rawdb.ReadHeaderNumber(b.eth.chainDb, hash); number != nil { return rawdb.ReadReceipts(b.eth.chainDb, hash, *number), nil } return nil, nil } -func (b *EthApiBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { +func (b *EthAPIBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { number := rawdb.ReadHeaderNumber(b.eth.chainDb, hash) if number == nil { return nil, nil @@ -124,11 +124,11 @@ func (b *EthApiBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*typ return logs, nil } -func (b *EthApiBackend) GetTd(blockHash common.Hash) *big.Int { +func (b *EthAPIBackend) GetTd(blockHash common.Hash) *big.Int { return b.eth.blockchain.GetTdByHash(blockHash) } -func (b *EthApiBackend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) { +func (b *EthAPIBackend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) { state.SetBalance(msg.From(), math.MaxBig256) vmError := func() error { return nil } @@ -136,31 +136,31 @@ func (b *EthApiBackend) GetEVM(ctx context.Context, msg core.Message, state *sta return vm.NewEVM(context, state, b.eth.chainConfig, vmCfg), vmError, nil } -func (b *EthApiBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { +func (b *EthAPIBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { return b.eth.BlockChain().SubscribeRemovedLogsEvent(ch) } -func (b *EthApiBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { +func (b *EthAPIBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { return b.eth.BlockChain().SubscribeChainEvent(ch) } -func (b *EthApiBackend) SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription { +func (b *EthAPIBackend) SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription { return b.eth.BlockChain().SubscribeChainHeadEvent(ch) } -func (b *EthApiBackend) SubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription { +func (b *EthAPIBackend) SubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription { return b.eth.BlockChain().SubscribeChainSideEvent(ch) } -func (b *EthApiBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { +func (b *EthAPIBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { return b.eth.BlockChain().SubscribeLogsEvent(ch) } -func (b *EthApiBackend) SendTx(ctx context.Context, signedTx *types.Transaction) error { +func (b *EthAPIBackend) SendTx(ctx context.Context, signedTx *types.Transaction) error { return b.eth.txPool.AddLocal(signedTx) } -func (b *EthApiBackend) GetPoolTransactions() (types.Transactions, error) { +func (b *EthAPIBackend) GetPoolTransactions() (types.Transactions, error) { pending, err := b.eth.txPool.Pending() if err != nil { return nil, err @@ -172,56 +172,56 @@ func (b *EthApiBackend) GetPoolTransactions() (types.Transactions, error) { return txs, nil } -func (b *EthApiBackend) GetPoolTransaction(hash common.Hash) *types.Transaction { +func (b *EthAPIBackend) GetPoolTransaction(hash common.Hash) *types.Transaction { return b.eth.txPool.Get(hash) } -func (b *EthApiBackend) GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) { +func (b *EthAPIBackend) GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) { return b.eth.txPool.State().GetNonce(addr), nil } -func (b *EthApiBackend) Stats() (pending int, queued int) { +func (b *EthAPIBackend) Stats() (pending int, queued int) { return b.eth.txPool.Stats() } -func (b *EthApiBackend) TxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions) { +func (b *EthAPIBackend) TxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions) { return b.eth.TxPool().Content() } -func (b *EthApiBackend) SubscribeTxPreEvent(ch chan<- core.TxPreEvent) event.Subscription { +func (b *EthAPIBackend) SubscribeTxPreEvent(ch chan<- core.TxPreEvent) event.Subscription { return b.eth.TxPool().SubscribeTxPreEvent(ch) } -func (b *EthApiBackend) Downloader() *downloader.Downloader { +func (b *EthAPIBackend) Downloader() *downloader.Downloader { return b.eth.Downloader() } -func (b *EthApiBackend) ProtocolVersion() int { +func (b *EthAPIBackend) ProtocolVersion() int { return b.eth.EthVersion() } -func (b *EthApiBackend) SuggestPrice(ctx context.Context) (*big.Int, error) { +func (b *EthAPIBackend) SuggestPrice(ctx context.Context) (*big.Int, error) { return b.gpo.SuggestPrice(ctx) } -func (b *EthApiBackend) ChainDb() ethdb.Database { +func (b *EthAPIBackend) ChainDb() ethdb.Database { return b.eth.ChainDb() } -func (b *EthApiBackend) EventMux() *event.TypeMux { +func (b *EthAPIBackend) EventMux() *event.TypeMux { return b.eth.EventMux() } -func (b *EthApiBackend) AccountManager() *accounts.Manager { +func (b *EthAPIBackend) AccountManager() *accounts.Manager { return b.eth.AccountManager() } -func (b *EthApiBackend) BloomStatus() (uint64, uint64) { +func (b *EthAPIBackend) BloomStatus() (uint64, uint64) { sections, _, _ := b.eth.bloomIndexer.Sections() return params.BloomBitsBlocks, sections } -func (b *EthApiBackend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) { +func (b *EthAPIBackend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) { for i := 0; i < bloomFilterThreads; i++ { go session.Multiplex(bloomRetrievalBatch, bloomRetrievalWait, b.eth.bloomRequests) } diff --git a/eth/backend.go b/eth/backend.go index e4c5fef3e4..ea70e3826c 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -82,7 +82,7 @@ type Ethereum struct { bloomRequests chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests bloomIndexer *core.ChainIndexer // Bloom indexer operating during block imports - ApiBackend *EthApiBackend + APIBackend *EthAPIBackend miner *miner.Miner gasPrice *big.Int @@ -169,12 +169,12 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { eth.miner = miner.New(eth, eth.chainConfig, eth.EventMux(), eth.engine) eth.miner.SetExtra(makeExtraData(config.ExtraData)) - eth.ApiBackend = &EthApiBackend{eth, nil} + eth.APIBackend = &EthAPIBackend{eth, nil} gpoParams := config.GPO if gpoParams.Default == nil { gpoParams.Default = config.GasPrice } - eth.ApiBackend.gpo = gasprice.NewOracle(eth.ApiBackend, gpoParams) + eth.APIBackend.gpo = gasprice.NewOracle(eth.APIBackend, gpoParams) return eth, nil } @@ -242,7 +242,7 @@ func CreateConsensusEngine(ctx *node.ServiceContext, config *ethash.Config, chai // APIs returns the collection of RPC services the ethereum package offers. // NOTE, some of these services probably need to be moved to somewhere else. func (s *Ethereum) APIs() []rpc.API { - apis := ethapi.GetAPIs(s.ApiBackend) + apis := ethapi.GetAPIs(s.APIBackend) // Append any APIs exposed explicitly by the consensus engine apis = append(apis, s.engine.APIs(s.BlockChain())...) @@ -272,7 +272,7 @@ func (s *Ethereum) APIs() []rpc.API { }, { Namespace: "eth", Version: "1.0", - Service: filters.NewPublicFilterAPI(s.ApiBackend, false), + Service: filters.NewPublicFilterAPI(s.APIBackend, false), Public: true, }, { Namespace: "admin", diff --git a/eth/protocol.go b/eth/protocol.go index cd7db57f23..328d5b9939 100644 --- a/eth/protocol.go +++ b/eth/protocol.go @@ -34,13 +34,13 @@ const ( eth63 = 63 ) -// Official short name of the protocol used during capability negotiation. +// ProtocolName is the official short name of the protocol used during capability negotiation. var ProtocolName = "eth" -// Supported versions of the eth protocol (first is primary). +// ProtocolVersions are the upported versions of the eth protocol (first is primary). var ProtocolVersions = []uint{eth63, eth62} -// Number of implemented message corresponding to different protocol versions. +// ProtocolLengths are the number of implemented message corresponding to different protocol versions. var ProtocolLengths = []uint64{17, 8} const ProtocolMaxMsgSize = 10 * 1024 * 1024 // Maximum cap on the size of a protocol message diff --git a/ethstats/ethstats.go b/ethstats/ethstats.go index ae7e252654..a15d846150 100644 --- a/ethstats/ethstats.go +++ b/ethstats/ethstats.go @@ -689,7 +689,7 @@ func (s *Service) reportStats(conn *websocket.Conn) error { sync := s.eth.Downloader().Progress() syncing = s.eth.BlockChain().CurrentHeader().Number.Uint64() >= sync.HighestBlock - price, _ := s.eth.ApiBackend.SuggestPrice(context.Background()) + price, _ := s.eth.APIBackend.SuggestPrice(context.Background()) gasprice = int(price.Uint64()) } else { sync := s.les.Downloader().Progress() From 4e7dc34ff1a7469b95eb16f5b4084c26a0ab3662 Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 9 May 2018 16:29:25 +0800 Subject: [PATCH 10/46] eth/filter: check nil pointer when unsubscribe (#16682) * eth/filter: check nil pointer when unsubscribe * eth/filters, accounts, rpc: abort system if subscribe failed * eth/filter: add crit log before exit * eth/filter, event: minor fixes --- eth/filters/filter_system.go | 99 +++++++++++++++++++++--------------- event/event.go | 6 +++ 2 files changed, 64 insertions(+), 41 deletions(-) diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index 6d16f572d0..bb11734a76 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" ) @@ -92,8 +93,21 @@ type EventSystem struct { backend Backend lightMode bool lastHead *types.Header - install chan *subscription // install filter for event notification - uninstall chan *subscription // remove filter for event notification + + // Subscriptions + txSub event.Subscription // Subscription for new transaction event + logsSub event.Subscription // Subscription for new log event + rmLogsSub event.Subscription // Subscription for removed log event + chainSub event.Subscription // Subscription for new chain event + pendingLogSub *event.TypeMuxSubscription // Subscription for pending log event + + // Channels + install chan *subscription // install filter for event notification + uninstall chan *subscription // remove filter for event notification + txCh chan core.TxPreEvent // Channel to receive new transaction event + logsCh chan []*types.Log // Channel to receive new log event + rmLogsCh chan core.RemovedLogsEvent // Channel to receive removed log event + chainCh chan core.ChainEvent // Channel to receive new chain event } // NewEventSystem creates a new manager that listens for event on the given mux, @@ -109,10 +123,27 @@ func NewEventSystem(mux *event.TypeMux, backend Backend, lightMode bool) *EventS lightMode: lightMode, install: make(chan *subscription), uninstall: make(chan *subscription), + txCh: make(chan core.TxPreEvent, txChanSize), + logsCh: make(chan []*types.Log, logsChanSize), + rmLogsCh: make(chan core.RemovedLogsEvent, rmLogsChanSize), + chainCh: make(chan core.ChainEvent, chainEvChanSize), + } + + // Subscribe events + m.txSub = m.backend.SubscribeTxPreEvent(m.txCh) + m.logsSub = m.backend.SubscribeLogsEvent(m.logsCh) + m.rmLogsSub = m.backend.SubscribeRemovedLogsEvent(m.rmLogsCh) + m.chainSub = m.backend.SubscribeChainEvent(m.chainCh) + // TODO(rjl493456442): use feed to subscribe pending log event + m.pendingLogSub = m.mux.Subscribe(core.PendingLogsEvent{}) + + // Make sure none of the subscriptions are empty + if m.txSub == nil || m.logsSub == nil || m.rmLogsSub == nil || m.chainSub == nil || + m.pendingLogSub.Closed() { + log.Crit("Subscribe for event system failed") } go m.eventLoop() - return m } @@ -412,52 +443,37 @@ func (es *EventSystem) lightFilterLogs(header *types.Header, addresses []common. // eventLoop (un)installs filters and processes mux events. func (es *EventSystem) eventLoop() { - var ( - index = make(filterIndex) - sub = es.mux.Subscribe(core.PendingLogsEvent{}) - // Subscribe TxPreEvent form txpool - txCh = make(chan core.TxPreEvent, txChanSize) - txSub = es.backend.SubscribeTxPreEvent(txCh) - // Subscribe RemovedLogsEvent - rmLogsCh = make(chan core.RemovedLogsEvent, rmLogsChanSize) - rmLogsSub = es.backend.SubscribeRemovedLogsEvent(rmLogsCh) - // Subscribe []*types.Log - logsCh = make(chan []*types.Log, logsChanSize) - logsSub = es.backend.SubscribeLogsEvent(logsCh) - // Subscribe ChainEvent - chainEvCh = make(chan core.ChainEvent, chainEvChanSize) - chainEvSub = es.backend.SubscribeChainEvent(chainEvCh) - ) - - // Unsubscribe all events - defer sub.Unsubscribe() - defer txSub.Unsubscribe() - defer rmLogsSub.Unsubscribe() - defer logsSub.Unsubscribe() - defer chainEvSub.Unsubscribe() + // Ensure all subscriptions get cleaned up + defer func() { + es.pendingLogSub.Unsubscribe() + es.txSub.Unsubscribe() + es.logsSub.Unsubscribe() + es.rmLogsSub.Unsubscribe() + es.chainSub.Unsubscribe() + }() + index := make(filterIndex) for i := UnknownSubscription; i < LastIndexSubscription; i++ { index[i] = make(map[rpc.ID]*subscription) } for { select { - case ev, active := <-sub.Chan(): + // Handle subscribed events + case ev := <-es.txCh: + es.broadcast(index, ev) + case ev := <-es.logsCh: + es.broadcast(index, ev) + case ev := <-es.rmLogsCh: + es.broadcast(index, ev) + case ev := <-es.chainCh: + es.broadcast(index, ev) + case ev, active := <-es.pendingLogSub.Chan(): if !active { // system stopped return } es.broadcast(index, ev) - // Handle subscribed events - case ev := <-txCh: - es.broadcast(index, ev) - case ev := <-rmLogsCh: - es.broadcast(index, ev) - case ev := <-logsCh: - es.broadcast(index, ev) - case ev := <-chainEvCh: - es.broadcast(index, ev) - case f := <-es.install: if f.typ == MinedAndPendingLogsSubscription { // the type are logs and pending logs subscriptions @@ -467,6 +483,7 @@ func (es *EventSystem) eventLoop() { index[f.typ][f.id] = f } close(f.installed) + case f := <-es.uninstall: if f.typ == MinedAndPendingLogsSubscription { // the type are logs and pending logs subscriptions @@ -478,13 +495,13 @@ func (es *EventSystem) eventLoop() { close(f.err) // System stopped - case <-txSub.Err(): + case <-es.txSub.Err(): return - case <-rmLogsSub.Err(): + case <-es.logsSub.Err(): return - case <-logsSub.Err(): + case <-es.rmLogsSub.Err(): return - case <-chainEvSub.Err(): + case <-es.chainSub.Err(): return } } diff --git a/event/event.go b/event/event.go index 20d20d1f57..4232787314 100644 --- a/event/event.go +++ b/event/event.go @@ -180,6 +180,12 @@ func (s *TypeMuxSubscription) Unsubscribe() { s.closewait() } +func (s *TypeMuxSubscription) Closed() bool { + s.closeMu.Lock() + defer s.closeMu.Unlock() + return s.closed +} + func (s *TypeMuxSubscription) closewait() { s.closeMu.Lock() defer s.closeMu.Unlock() From 5dbd8b42a90779bd4269012c1336679fd4ca9824 Mon Sep 17 00:00:00 2001 From: Andrea Franz Date: Wed, 9 May 2018 13:40:59 +0200 Subject: [PATCH 11/46] whisper/shhclient: update call to shh_generateSymKeyFromPassword to pass a string (#16668) --- whisper/shhclient/client.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/whisper/shhclient/client.go b/whisper/shhclient/client.go index 7b25e739e1..8e7085a0a6 100644 --- a/whisper/shhclient/client.go +++ b/whisper/shhclient/client.go @@ -135,9 +135,9 @@ func (sc *Client) AddSymmetricKey(ctx context.Context, key []byte) (string, erro } // GenerateSymmetricKeyFromPassword generates the key from password, stores it, and returns its identifier. -func (sc *Client) GenerateSymmetricKeyFromPassword(ctx context.Context, passwd []byte) (string, error) { +func (sc *Client) GenerateSymmetricKeyFromPassword(ctx context.Context, passwd string) (string, error) { var id string - return id, sc.c.CallContext(ctx, &id, "shh_generateSymKeyFromPassword", hexutil.Bytes(passwd)) + return id, sc.c.CallContext(ctx, &id, "shh_generateSymKeyFromPassword", passwd) } // HasSymmetricKey returns an indication if the key associated with the given id is stored in the node. From 7beccb29becf439df7bf4c033a94c019ad25bead Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 9 May 2018 20:24:25 +0800 Subject: [PATCH 12/46] all: get rid of error when creating memory database (#16716) * all: get rid of error when create mdb * core: clean up variables definition * all: inline mdb definition --- accounts/abi/bind/backends/simulated.go | 2 +- cmd/evm/runner.go | 5 +-- consensus/clique/snapshot_test.go | 2 +- core/bench_test.go | 2 +- core/block_validator_test.go | 6 +-- core/blockchain_test.go | 58 ++++++++++++------------- core/chain_indexer_test.go | 2 +- core/chain_makers.go | 9 ++-- core/chain_makers_test.go | 2 +- core/dao_test.go | 14 +++--- core/genesis.go | 2 +- core/genesis_test.go | 2 +- core/helper_test.go | 10 +---- core/rawdb/accessors_chain_test.go | 16 +++---- core/rawdb/accessors_indexes_test.go | 2 +- core/state/managed_state_test.go | 3 +- core/state/state_test.go | 5 +-- core/state/statedb_test.go | 15 +++---- core/state/sync_test.go | 16 +++---- core/tx_pool_test.go | 54 ++++++++--------------- core/vm/runtime/runtime.go | 6 +-- core/vm/runtime/runtime_test.go | 3 +- eth/api_test.go | 3 +- eth/downloader/downloader_test.go | 4 +- eth/fetcher/fetcher_test.go | 2 +- eth/filters/filter_system_test.go | 12 ++--- eth/handler_test.go | 4 +- eth/helper_test.go | 2 +- eth/tracers/tracers_test.go | 3 +- ethdb/database_test.go | 6 +-- ethdb/memory_database.go | 8 ++-- les/handler_test.go | 19 ++++---- les/odr_test.go | 4 +- les/request_test.go | 4 +- light/lightchain_test.go | 4 +- light/odr_test.go | 4 +- light/trie_test.go | 8 ++-- light/txpool_test.go | 4 +- node/node.go | 2 +- node/service.go | 2 +- tests/block_test_util.go | 2 +- tests/state_test_util.go | 3 +- tests/vm_test_util.go | 3 +- trie/iterator_test.go | 4 +- trie/proof_test.go | 10 ++--- trie/secure_trie_test.go | 8 +--- trie/sync_test.go | 30 +++++-------- trie/trie_test.go | 11 ++--- 48 files changed, 172 insertions(+), 230 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 76c51a40c1..7b605e1c1e 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -66,7 +66,7 @@ type SimulatedBackend struct { // NewSimulatedBackend creates a new binding backend using a simulated blockchain // for testing purposes. func NewSimulatedBackend(alloc core.GenesisAlloc) *SimulatedBackend { - database, _ := ethdb.NewMemDatabase() + database := ethdb.NewMemDatabase() genesis := core.Genesis{Config: params.AllEthashProtocolChanges, Alloc: alloc} genesis.MustCommit(database) blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}) diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index 99919304ae..7138a9ddd4 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -98,14 +98,13 @@ func runCmd(ctx *cli.Context) error { } if ctx.GlobalString(GenesisFlag.Name) != "" { gen := readGenesis(ctx.GlobalString(GenesisFlag.Name)) - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() genesis := gen.ToBlock(db) statedb, _ = state.New(genesis.Root(), state.NewDatabase(db)) chainConfig = gen.Config blockNumber = gen.Number } else { - db, _ := ethdb.NewMemDatabase() - statedb, _ = state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) } if ctx.GlobalString(SenderFlag.Name) != "" { sender = common.HexToAddress(ctx.GlobalString(SenderFlag.Name)) diff --git a/consensus/clique/snapshot_test.go b/consensus/clique/snapshot_test.go index 91fafcbe07..29a8379832 100644 --- a/consensus/clique/snapshot_test.go +++ b/consensus/clique/snapshot_test.go @@ -352,7 +352,7 @@ func TestVoting(t *testing.T) { copy(genesis.ExtraData[extraVanity+j*common.AddressLength:], signer[:]) } // Create a pristine blockchain with the genesis injected - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() genesis.Commit(db) // Assemble a chain of headers from the cast votes diff --git a/core/bench_test.go b/core/bench_test.go index ee30cfed0e..748aebe407 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -149,7 +149,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) { // Create the database in memory or in a temporary directory. var db ethdb.Database if !disk { - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() } else { dir, err := ioutil.TempDir("", "eth-core-bench") if err != nil { diff --git a/core/block_validator_test.go b/core/block_validator_test.go index e334b3c3cd..2a171218e3 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -32,7 +32,7 @@ import ( func TestHeaderVerification(t *testing.T) { // Create a simple chain to verify var ( - testdb, _ = ethdb.NewMemDatabase() + testdb = ethdb.NewMemDatabase() gspec = &Genesis{Config: params.TestChainConfig} genesis = gspec.MustCommit(testdb) blocks, _ = GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), testdb, 8, nil) @@ -84,7 +84,7 @@ func TestHeaderConcurrentVerification32(t *testing.T) { testHeaderConcurrentVeri func testHeaderConcurrentVerification(t *testing.T, threads int) { // Create a simple chain to verify var ( - testdb, _ = ethdb.NewMemDatabase() + testdb = ethdb.NewMemDatabase() gspec = &Genesis{Config: params.TestChainConfig} genesis = gspec.MustCommit(testdb) blocks, _ = GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), testdb, 8, nil) @@ -156,7 +156,7 @@ func TestHeaderConcurrentAbortion32(t *testing.T) { testHeaderConcurrentAbortion func testHeaderConcurrentAbortion(t *testing.T, threads int) { // Create a simple chain to verify var ( - testdb, _ = ethdb.NewMemDatabase() + testdb = ethdb.NewMemDatabase() gspec = &Genesis{Config: params.TestChainConfig} genesis = gspec.MustCommit(testdb) blocks, _ = GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), testdb, 1024, nil) diff --git a/core/blockchain_test.go b/core/blockchain_test.go index c1638c31f8..89c071174f 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -569,11 +569,11 @@ func testInsertNonceError(t *testing.T, full bool) { func TestFastVsFullChains(t *testing.T) { // Configure and generate a sample block chain var ( - gendb, _ = ethdb.NewMemDatabase() - key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - address = crypto.PubkeyToAddress(key.PublicKey) - funds = big.NewInt(1000000000) - gspec = &Genesis{ + gendb = ethdb.NewMemDatabase() + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + funds = big.NewInt(1000000000) + gspec = &Genesis{ Config: params.TestChainConfig, Alloc: GenesisAlloc{address: {Balance: funds}}, } @@ -599,7 +599,7 @@ func TestFastVsFullChains(t *testing.T) { } }) // Import the chain as an archive node for the comparison baseline - archiveDb, _ := ethdb.NewMemDatabase() + archiveDb := ethdb.NewMemDatabase() gspec.MustCommit(archiveDb) archive, _ := NewBlockChain(archiveDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) defer archive.Stop() @@ -608,7 +608,7 @@ func TestFastVsFullChains(t *testing.T) { t.Fatalf("failed to process block %d: %v", n, err) } // Fast import the chain as a non-archive node to test - fastDb, _ := ethdb.NewMemDatabase() + fastDb := ethdb.NewMemDatabase() gspec.MustCommit(fastDb) fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) defer fast.Stop() @@ -657,12 +657,12 @@ func TestFastVsFullChains(t *testing.T) { func TestLightVsFastVsFullChainHeads(t *testing.T) { // Configure and generate a sample block chain var ( - gendb, _ = ethdb.NewMemDatabase() - key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - address = crypto.PubkeyToAddress(key.PublicKey) - funds = big.NewInt(1000000000) - gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{address: {Balance: funds}}} - genesis = gspec.MustCommit(gendb) + gendb = ethdb.NewMemDatabase() + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + funds = big.NewInt(1000000000) + gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{address: {Balance: funds}}} + genesis = gspec.MustCommit(gendb) ) height := uint64(1024) blocks, receipts := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), gendb, int(height), nil) @@ -685,7 +685,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { } } // Import the chain as an archive node and ensure all pointers are updated - archiveDb, _ := ethdb.NewMemDatabase() + archiveDb := ethdb.NewMemDatabase() gspec.MustCommit(archiveDb) archive, _ := NewBlockChain(archiveDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) @@ -699,7 +699,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { assert(t, "archive", archive, height/2, height/2, height/2) // Import the chain as a non-archive node and ensure all pointers are updated - fastDb, _ := ethdb.NewMemDatabase() + fastDb := ethdb.NewMemDatabase() gspec.MustCommit(fastDb) fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) defer fast.Stop() @@ -719,7 +719,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { assert(t, "fast", fast, height/2, height/2, 0) // Import the chain as a light node and ensure all pointers are updated - lightDb, _ := ethdb.NewMemDatabase() + lightDb := ethdb.NewMemDatabase() gspec.MustCommit(lightDb) light, _ := NewBlockChain(lightDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) @@ -742,7 +742,7 @@ func TestChainTxReorgs(t *testing.T) { addr1 = crypto.PubkeyToAddress(key1.PublicKey) addr2 = crypto.PubkeyToAddress(key2.PublicKey) addr3 = crypto.PubkeyToAddress(key3.PublicKey) - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() gspec = &Genesis{ Config: params.TestChainConfig, GasLimit: 3141592, @@ -854,7 +854,7 @@ func TestLogReorgs(t *testing.T) { var ( key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() // this code generates a log code = common.Hex2Bytes("60606040525b7f24ec1d3ff24c2f6ff210738839dbc339cd45a5294d85c79361016243157aae7b60405180905060405180910390a15b600a8060416000396000f360606040526008565b00") gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000)}}} @@ -898,7 +898,7 @@ func TestLogReorgs(t *testing.T) { func TestReorgSideEvent(t *testing.T) { var ( - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) gspec = &Genesis{ @@ -1026,7 +1026,7 @@ func TestCanonicalBlockRetrieval(t *testing.T) { func TestEIP155Transition(t *testing.T) { // Configure and generate a sample block chain var ( - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") address = crypto.PubkeyToAddress(key.PublicKey) funds = big.NewInt(1000000000) @@ -1130,7 +1130,7 @@ func TestEIP155Transition(t *testing.T) { func TestEIP161AccountRemoval(t *testing.T) { // Configure and generate a sample block chain var ( - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") address = crypto.PubkeyToAddress(key.PublicKey) funds = big.NewInt(1000000000) @@ -1202,7 +1202,7 @@ func TestBlockchainHeaderchainReorgConsistency(t *testing.T) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() genesis := new(Genesis).MustCommit(db) blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 64, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) @@ -1218,7 +1218,7 @@ func TestBlockchainHeaderchainReorgConsistency(t *testing.T) { } // Import the canonical and fork chain side by side, verifying the current block // and current header consistency - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() new(Genesis).MustCommit(diskdb) chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}) @@ -1247,7 +1247,7 @@ func TestTrieForkGC(t *testing.T) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() genesis := new(Genesis).MustCommit(db) blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 2*triesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) @@ -1262,7 +1262,7 @@ func TestTrieForkGC(t *testing.T) { forks[i] = fork[0] } // Import the canonical and fork chain side by side, forcing the trie cache to cache both - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() new(Genesis).MustCommit(diskdb) chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}) @@ -1293,7 +1293,7 @@ func TestLargeReorgTrieGC(t *testing.T) { // Generate the original common chain segment and the two competing forks engine := ethash.NewFaker() - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() genesis := new(Genesis).MustCommit(db) shared, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 64, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) @@ -1301,7 +1301,7 @@ func TestLargeReorgTrieGC(t *testing.T) { competitor, _ := GenerateChain(params.TestChainConfig, shared[len(shared)-1], engine, db, 2*triesInMemory+1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{3}) }) // Import the shared chain and the original canonical one - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() new(Genesis).MustCommit(diskdb) chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}) @@ -1361,7 +1361,7 @@ func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks in ) // Generate the original common chain segment and the two competing forks engine := ethash.NewFaker() - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() genesis := gspec.MustCommit(db) blockGenerator := func(i int, block *BlockGen) { @@ -1383,7 +1383,7 @@ func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks in b.ResetTimer() for i := 0; i < b.N; i++ { // Import the shared chain and the original canonical one - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() gspec.MustCommit(diskdb) chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}) diff --git a/core/chain_indexer_test.go b/core/chain_indexer_test.go index 3205616e8d..550caf5567 100644 --- a/core/chain_indexer_test.go +++ b/core/chain_indexer_test.go @@ -48,7 +48,7 @@ func TestChainIndexerWithChildren(t *testing.T) { // multiple backends. The section size and required confirmation count parameters // are randomized. func testChainIndexer(t *testing.T, count int) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() defer db.Close() // Create a chain of indexers and ensure they all report empty diff --git a/core/chain_makers.go b/core/chain_makers.go index 31c9e3fb77..fcba90bb87 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -256,11 +256,12 @@ func makeHeader(chain consensus.ChainReader, parent *types.Block, state *state.S // chain. Depending on the full flag, if creates either a full block chain or a // header only chain. func newCanonical(engine consensus.Engine, n int, full bool) (ethdb.Database, *BlockChain, error) { - // Initialize a fresh chain with only a genesis block - gspec := new(Genesis) - db, _ := ethdb.NewMemDatabase() - genesis := gspec.MustCommit(db) + var ( + db = ethdb.NewMemDatabase() + genesis = new(Genesis).MustCommit(db) + ) + // Initialize a fresh chain with only a genesis block blockchain, _ := NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}) // Create and inject the requested chain if n == 0 { diff --git a/core/chain_makers_test.go b/core/chain_makers_test.go index 93be43ddce..5015d1f48e 100644 --- a/core/chain_makers_test.go +++ b/core/chain_makers_test.go @@ -36,7 +36,7 @@ func ExampleGenerateChain() { addr1 = crypto.PubkeyToAddress(key1.PublicKey) addr2 = crypto.PubkeyToAddress(key2.PublicKey) addr3 = crypto.PubkeyToAddress(key3.PublicKey) - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() ) // Ensure that key1 has some funds in the genesis block. diff --git a/core/dao_test.go b/core/dao_test.go index e0a3e3ff37..284b1d98bd 100644 --- a/core/dao_test.go +++ b/core/dao_test.go @@ -32,13 +32,13 @@ func TestDAOForkRangeExtradata(t *testing.T) { forkBlock := big.NewInt(32) // Generate a common prefix for both pro-forkers and non-forkers - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() gspec := new(Genesis) genesis := gspec.MustCommit(db) prefix, _ := GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, int(forkBlock.Int64()-1), func(i int, gen *BlockGen) {}) // Create the concurrent, conflicting two nodes - proDb, _ := ethdb.NewMemDatabase() + proDb := ethdb.NewMemDatabase() gspec.MustCommit(proDb) proConf := *params.TestChainConfig @@ -48,7 +48,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { proBc, _ := NewBlockChain(proDb, nil, &proConf, ethash.NewFaker(), vm.Config{}) defer proBc.Stop() - conDb, _ := ethdb.NewMemDatabase() + conDb := ethdb.NewMemDatabase() gspec.MustCommit(conDb) conConf := *params.TestChainConfig @@ -67,7 +67,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Try to expand both pro-fork and non-fork chains iteratively with other camp's blocks for i := int64(0); i < params.DAOForkExtraRange.Int64(); i++ { // Create a pro-fork block, and try to feed into the no-fork chain - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() gspec.MustCommit(db) bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{}) defer bc.Stop() @@ -92,7 +92,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { t.Fatalf("contra-fork chain didn't accepted no-fork block: %v", err) } // Create a no-fork block, and try to feed into the pro-fork chain - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() gspec.MustCommit(db) bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{}) defer bc.Stop() @@ -118,7 +118,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { } } // Verify that contra-forkers accept pro-fork extra-datas after forking finishes - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() gspec.MustCommit(db) bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{}) defer bc.Stop() @@ -138,7 +138,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { t.Fatalf("contra-fork chain didn't accept pro-fork block post-fork: %v", err) } // Verify that pro-forkers accept contra-fork extra-datas after forking finishes - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() gspec.MustCommit(db) bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{}) defer bc.Stop() diff --git a/core/genesis.go b/core/genesis.go index c0a636ab2d..9190e2ba22 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -222,7 +222,7 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig { // to the given database (or discards it if nil). func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { if db == nil { - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() } statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) for addr, account := range g.Alloc { diff --git a/core/genesis_test.go b/core/genesis_test.go index 613434e208..2d7f94f8f8 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -141,7 +141,7 @@ func TestSetupGenesis(t *testing.T) { } for _, test := range tests { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() config, hash, err := test.fn(db) // Check the return values. if !reflect.DeepEqual(err, test.wantErr) { diff --git a/core/helper_test.go b/core/helper_test.go index 698a2924cb..051384d854 100644 --- a/core/helper_test.go +++ b/core/helper_test.go @@ -18,7 +18,6 @@ package core import ( "container/list" - "fmt" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" @@ -77,18 +76,11 @@ func (tm *TestManager) Db() ethdb.Database { } func NewTestManager() *TestManager { - db, err := ethdb.NewMemDatabase() - if err != nil { - fmt.Println("Could not create mem-db, failing") - return nil - } - testManager := &TestManager{} testManager.eventMux = new(event.TypeMux) - testManager.db = db + testManager.db = ethdb.NewMemDatabase() // testManager.txPool = NewTxPool(testManager) // testManager.blockChain = NewBlockChain(testManager) // testManager.stateManager = NewStateManager(testManager) - return testManager } diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index 84c9c9aeb2..9ddae6e2b5 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -30,7 +30,7 @@ import ( // Tests block header storage and retrieval operations. func TestHeaderStorage(t *testing.T) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() // Create a test header to move around the database and make sure it's really new header := &types.Header{Number: big.NewInt(42), Extra: []byte("test header")} @@ -63,7 +63,7 @@ func TestHeaderStorage(t *testing.T) { // Tests block body storage and retrieval operations. func TestBodyStorage(t *testing.T) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() // Create a test body to move around the database and make sure it's really new body := &types.Body{Uncles: []*types.Header{{Extra: []byte("test header")}}} @@ -101,7 +101,7 @@ func TestBodyStorage(t *testing.T) { // Tests block storage and retrieval operations. func TestBlockStorage(t *testing.T) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() // Create a test block to move around the database and make sure it's really new block := types.NewBlockWithHeader(&types.Header{ @@ -151,7 +151,7 @@ func TestBlockStorage(t *testing.T) { // Tests that partial block contents don't get reassembled into full blocks. func TestPartialBlockStorage(t *testing.T) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() block := types.NewBlockWithHeader(&types.Header{ Extra: []byte("test block"), UncleHash: types.EmptyUncleHash, @@ -185,7 +185,7 @@ func TestPartialBlockStorage(t *testing.T) { // Tests block total difficulty storage and retrieval operations. func TestTdStorage(t *testing.T) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() // Create a test TD to move around the database and make sure it's really new hash, td := common.Hash{}, big.NewInt(314) @@ -208,7 +208,7 @@ func TestTdStorage(t *testing.T) { // Tests that canonical numbers can be mapped to hashes and retrieved. func TestCanonicalMappingStorage(t *testing.T) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() // Create a test canonical number and assinged hash to move around hash, number := common.Hash{0: 0xff}, uint64(314) @@ -231,7 +231,7 @@ func TestCanonicalMappingStorage(t *testing.T) { // Tests that head headers and head blocks can be assigned, individually. func TestHeadStorage(t *testing.T) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() blockHead := types.NewBlockWithHeader(&types.Header{Extra: []byte("test block header")}) blockFull := types.NewBlockWithHeader(&types.Header{Extra: []byte("test block full")}) @@ -266,7 +266,7 @@ func TestHeadStorage(t *testing.T) { // Tests that receipts associated with a single block can be stored and retrieved. func TestBlockReceiptStorage(t *testing.T) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() receipt1 := &types.Receipt{ Status: types.ReceiptStatusFailed, diff --git a/core/rawdb/accessors_indexes_test.go b/core/rawdb/accessors_indexes_test.go index d9c1291634..d9c10e1490 100644 --- a/core/rawdb/accessors_indexes_test.go +++ b/core/rawdb/accessors_indexes_test.go @@ -27,7 +27,7 @@ import ( // Tests that positional lookup metadata can be stored and retrieved. func TestLookupStorage(t *testing.T) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() tx1 := types.NewTransaction(1, common.BytesToAddress([]byte{0x11}), big.NewInt(111), 1111, big.NewInt(11111), []byte{0x11, 0x11, 0x11}) tx2 := types.NewTransaction(2, common.BytesToAddress([]byte{0x22}), big.NewInt(222), 2222, big.NewInt(22222), []byte{0x22, 0x22, 0x22}) diff --git a/core/state/managed_state_test.go b/core/state/managed_state_test.go index 1cfdd3a890..3d9c4e8676 100644 --- a/core/state/managed_state_test.go +++ b/core/state/managed_state_test.go @@ -26,8 +26,7 @@ import ( var addr = common.BytesToAddress([]byte("test")) func create() (*ManagedState, *account) { - db, _ := ethdb.NewMemDatabase() - statedb, _ := New(common.Hash{}, NewDatabase(db)) + statedb, _ := New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase())) ms := ManageState(statedb) ms.StateDB.SetNonce(addr, 100) ms.accounts[addr] = newAccount(ms.StateDB.getStateObject(addr)) diff --git a/core/state/state_test.go b/core/state/state_test.go index 6d42d63d82..12778f6f12 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -87,7 +87,7 @@ func (s *StateSuite) TestDump(c *checker.C) { } func (s *StateSuite) SetUpTest(c *checker.C) { - s.db, _ = ethdb.NewMemDatabase() + s.db = ethdb.NewMemDatabase() s.state, _ = New(common.Hash{}, NewDatabase(s.db)) } @@ -133,8 +133,7 @@ func (s *StateSuite) TestSnapshotEmpty(c *checker.C) { // use testing instead of checker because checker does not support // printing/logging in tests (-check.vv does not work) func TestSnapshot2(t *testing.T) { - db, _ := ethdb.NewMemDatabase() - state, _ := New(common.Hash{}, NewDatabase(db)) + state, _ := New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase())) stateobjaddr0 := toAddr([]byte("so0")) stateobjaddr1 := toAddr([]byte("so1")) diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 420ca745cc..e2b349de86 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -39,7 +39,7 @@ import ( // actually committing the state. func TestUpdateLeaks(t *testing.T) { // Create an empty state database - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() state, _ := New(common.Hash{}, NewDatabase(db)) // Update it with some accounts @@ -66,8 +66,8 @@ func TestUpdateLeaks(t *testing.T) { // only the one right before the commit. func TestIntermediateLeaks(t *testing.T) { // Create two state databases, one transitioning to the final state, the other final from the beginning - transDb, _ := ethdb.NewMemDatabase() - finalDb, _ := ethdb.NewMemDatabase() + transDb := ethdb.NewMemDatabase() + finalDb := ethdb.NewMemDatabase() transState, _ := New(common.Hash{}, NewDatabase(transDb)) finalState, _ := New(common.Hash{}, NewDatabase(finalDb)) @@ -122,8 +122,7 @@ func TestIntermediateLeaks(t *testing.T) { // https://github.com/ethereum/go-ethereum/pull/15549. func TestCopy(t *testing.T) { // Create a random state test to copy and modify "independently" - db, _ := ethdb.NewMemDatabase() - orig, _ := New(common.Hash{}, NewDatabase(db)) + orig, _ := New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase())) for i := byte(0); i < 255; i++ { obj := orig.GetOrNewStateObject(common.BytesToAddress([]byte{i})) @@ -334,8 +333,7 @@ func (test *snapshotTest) String() string { func (test *snapshotTest) run() bool { // Run all actions and create snapshots. var ( - db, _ = ethdb.NewMemDatabase() - state, _ = New(common.Hash{}, NewDatabase(db)) + state, _ = New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase())) snapshotRevs = make([]int, len(test.snapshots)) sindex = 0 ) @@ -426,8 +424,7 @@ func (s *StateSuite) TestTouchDelete(c *check.C) { // TestCopyOfCopy tests that modified objects are carried over to the copy, and the copy of the copy. // See https://github.com/ethereum/go-ethereum/pull/15225#issuecomment-380191512 func TestCopyOfCopy(t *testing.T) { - db, _ := ethdb.NewMemDatabase() - sdb, _ := New(common.Hash{}, NewDatabase(db)) + sdb, _ := New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase())) addr := common.HexToAddress("aaaa") sdb.SetBalance(addr, big.NewInt(42)) diff --git a/core/state/sync_test.go b/core/state/sync_test.go index 8f14a44e7a..3177401608 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -38,8 +38,7 @@ type testAccount struct { // makeTestState create a sample test state to test node-wise reconstruction. func makeTestState() (Database, common.Hash, []*testAccount) { // Create an empty state - diskdb, _ := ethdb.NewMemDatabase() - db := NewDatabase(diskdb) + db := NewDatabase(ethdb.NewMemDatabase()) state, _ := New(common.Hash{}, db) // Fill it with some arbitrary data @@ -125,8 +124,7 @@ func checkStateConsistency(db ethdb.Database, root common.Hash) error { // Tests that an empty state is not scheduled for syncing. func TestEmptyStateSync(t *testing.T) { empty := common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") - db, _ := ethdb.NewMemDatabase() - if req := NewStateSync(empty, db).Missing(1); len(req) != 0 { + if req := NewStateSync(empty, ethdb.NewMemDatabase()).Missing(1); len(req) != 0 { t.Errorf("content requested for empty state: %v", req) } } @@ -141,7 +139,7 @@ func testIterativeStateSync(t *testing.T, batch int) { srcDb, srcRoot, srcAccounts := makeTestState() // Create a destination state and sync with the scheduler - dstDb, _ := ethdb.NewMemDatabase() + dstDb := ethdb.NewMemDatabase() sched := NewStateSync(srcRoot, dstDb) queue := append([]common.Hash{}, sched.Missing(batch)...) @@ -173,7 +171,7 @@ func TestIterativeDelayedStateSync(t *testing.T) { srcDb, srcRoot, srcAccounts := makeTestState() // Create a destination state and sync with the scheduler - dstDb, _ := ethdb.NewMemDatabase() + dstDb := ethdb.NewMemDatabase() sched := NewStateSync(srcRoot, dstDb) queue := append([]common.Hash{}, sched.Missing(0)...) @@ -210,7 +208,7 @@ func testIterativeRandomStateSync(t *testing.T, batch int) { srcDb, srcRoot, srcAccounts := makeTestState() // Create a destination state and sync with the scheduler - dstDb, _ := ethdb.NewMemDatabase() + dstDb := ethdb.NewMemDatabase() sched := NewStateSync(srcRoot, dstDb) queue := make(map[common.Hash]struct{}) @@ -250,7 +248,7 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) { srcDb, srcRoot, srcAccounts := makeTestState() // Create a destination state and sync with the scheduler - dstDb, _ := ethdb.NewMemDatabase() + dstDb := ethdb.NewMemDatabase() sched := NewStateSync(srcRoot, dstDb) queue := make(map[common.Hash]struct{}) @@ -297,7 +295,7 @@ func TestIncompleteStateSync(t *testing.T) { checkTrieConsistency(srcDb.TrieDB().DiskDB().(ethdb.Database), srcRoot) // Create a destination state and sync with the scheduler - dstDb, _ := ethdb.NewMemDatabase() + dstDb := ethdb.NewMemDatabase() sched := NewStateSync(srcRoot, dstDb) added := []common.Hash{} diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index f0f86415d4..e7f52075e5 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -78,8 +78,7 @@ func pricedTransaction(nonce uint64, gaslimit uint64, gasprice *big.Int, key *ec } func setupTxPool() (*TxPool, *ecdsa.PrivateKey) { - diskdb, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(diskdb)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} key, _ := crypto.GenerateKey() @@ -158,8 +157,7 @@ func (c *testChain) State() (*state.StateDB, error) { // a state change between those fetches. stdb := c.statedb if *c.trigger { - db, _ := ethdb.NewMemDatabase() - c.statedb, _ = state.New(common.Hash{}, state.NewDatabase(db)) + c.statedb, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) // simulate that the new head block included tx0 and tx1 c.statedb.SetNonce(c.address, 2) c.statedb.SetBalance(c.address, new(big.Int).SetUint64(params.Ether)) @@ -175,10 +173,9 @@ func TestStateChangeDuringTransactionPoolReset(t *testing.T) { t.Parallel() var ( - db, _ = ethdb.NewMemDatabase() key, _ = crypto.GenerateKey() address = crypto.PubkeyToAddress(key.PublicKey) - statedb, _ = state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) trigger = false ) @@ -332,8 +329,7 @@ func TestTransactionChainFork(t *testing.T) { addr := crypto.PubkeyToAddress(key.PublicKey) resetState := func() { - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) statedb.AddBalance(addr, big.NewInt(100000000000000)) pool.chain = &testBlockChain{statedb, 1000000, new(event.Feed)} @@ -362,8 +358,7 @@ func TestTransactionDoubleNonce(t *testing.T) { addr := crypto.PubkeyToAddress(key.PublicKey) resetState := func() { - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) statedb.AddBalance(addr, big.NewInt(100000000000000)) pool.chain = &testBlockChain{statedb, 1000000, new(event.Feed)} @@ -553,8 +548,7 @@ func TestTransactionPostponing(t *testing.T) { t.Parallel() // Create the pool to test the postponing with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) @@ -769,8 +763,7 @@ func testTransactionQueueGlobalLimiting(t *testing.T, nolocals bool) { t.Parallel() // Create the pool to test the limit enforcement with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -858,8 +851,7 @@ func testTransactionQueueTimeLimiting(t *testing.T, nolocals bool) { evictionInterval = time.Second // Create the pool to test the non-expiration enforcement - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1013,8 +1005,7 @@ func TestTransactionPendingGlobalLimiting(t *testing.T) { t.Parallel() // Create the pool to test the limit enforcement with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1060,8 +1051,7 @@ func TestTransactionCapClearsFromAll(t *testing.T) { t.Parallel() // Create the pool to test the limit enforcement with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1095,8 +1085,7 @@ func TestTransactionPendingMinimumAllowance(t *testing.T) { t.Parallel() // Create the pool to test the limit enforcement with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1144,8 +1133,7 @@ func TestTransactionPoolRepricing(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) @@ -1266,8 +1254,7 @@ func TestTransactionPoolRepricingKeepsLocals(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) @@ -1329,8 +1316,7 @@ func TestTransactionPoolUnderpricing(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1436,8 +1422,7 @@ func TestTransactionPoolStableUnderpricing(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1503,8 +1488,7 @@ func TestTransactionReplacement(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) @@ -1598,8 +1582,7 @@ func testTransactionJournaling(t *testing.T, nolocals bool) { os.Remove(journal) // Create the original pool to inject transaction into the journal - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1697,8 +1680,7 @@ func TestTransactionStatusCheck(t *testing.T) { t.Parallel() // Create the pool to test the status retrievals with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) diff --git a/core/vm/runtime/runtime.go b/core/vm/runtime/runtime.go index 5ac5464064..c8c5d34d9d 100644 --- a/core/vm/runtime/runtime.go +++ b/core/vm/runtime/runtime.go @@ -99,8 +99,7 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) { setDefaults(cfg) if cfg.State == nil { - db, _ := ethdb.NewMemDatabase() - cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(db)) + cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) } var ( address = common.BytesToAddress([]byte("contract")) @@ -130,8 +129,7 @@ func Create(input []byte, cfg *Config) ([]byte, common.Address, uint64, error) { setDefaults(cfg) if cfg.State == nil { - db, _ := ethdb.NewMemDatabase() - cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(db)) + cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) } var ( vmenv = NewEnv(cfg) diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index 2c4dc50265..ef664bda30 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -94,8 +94,7 @@ func TestExecute(t *testing.T) { } func TestCall(t *testing.T) { - db, _ := ethdb.NewMemDatabase() - state, _ := state.New(common.Hash{}, state.NewDatabase(db)) + state, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) address := common.HexToAddress("0x0a") state.SetCode(address, []byte{ byte(vm.PUSH1), 10, diff --git a/eth/api_test.go b/eth/api_test.go index 900a82bb6a..47b062a40c 100644 --- a/eth/api_test.go +++ b/eth/api_test.go @@ -31,8 +31,7 @@ var dumper = spew.ConfigState{Indent: " "} func TestStorageRangeAt(t *testing.T) { // Create a state where account 0x010000... has a few storage entries. var ( - db, _ = ethdb.NewMemDatabase() - state, _ = state.New(common.Hash{}, state.NewDatabase(db)) + state, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) addr = common.Address{0x01} keys = []common.Hash{ // hashes of Keys of storage common.HexToHash("340dd630ad21bf010b4e676dbfa9ba9a02175262d1fa356232cfde6cb5b47ef2"), diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index e85e234c0e..d1a9a8694b 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -75,7 +75,7 @@ type downloadTester struct { // newTester creates a new downloader test mocker. func newTester() *downloadTester { - testdb, _ := ethdb.NewMemDatabase() + testdb := ethdb.NewMemDatabase() genesis := core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000)) tester := &downloadTester{ @@ -93,7 +93,7 @@ func newTester() *downloadTester { peerChainTds: make(map[string]map[common.Hash]*big.Int), peerMissingStates: make(map[string]map[common.Hash]bool), } - tester.stateDb, _ = ethdb.NewMemDatabase() + tester.stateDb = ethdb.NewMemDatabase() tester.stateDb.Put(genesis.Root().Bytes(), []byte{0x00}) tester.downloader = New(FullSync, tester.stateDb, new(event.TypeMux), tester, nil, tester.dropPeer) diff --git a/eth/fetcher/fetcher_test.go b/eth/fetcher/fetcher_test.go index 1e536fcacb..3d4f0d1e59 100644 --- a/eth/fetcher/fetcher_test.go +++ b/eth/fetcher/fetcher_test.go @@ -34,7 +34,7 @@ import ( ) var ( - testdb, _ = ethdb.NewMemDatabase() + testdb = ethdb.NewMemDatabase() testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") testAddress = crypto.PubkeyToAddress(testKey.PublicKey) genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000)) diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 73819eadc0..b4df24b471 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -153,7 +153,7 @@ func TestBlockSubscription(t *testing.T) { var ( mux = new(event.TypeMux) - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() txFeed = new(event.Feed) rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) @@ -210,7 +210,7 @@ func TestPendingTxFilter(t *testing.T) { var ( mux = new(event.TypeMux) - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() txFeed = new(event.Feed) rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) @@ -273,7 +273,7 @@ func TestPendingTxFilter(t *testing.T) { func TestLogFilterCreation(t *testing.T) { var ( mux = new(event.TypeMux) - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() txFeed = new(event.Feed) rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) @@ -322,7 +322,7 @@ func TestInvalidLogFilterCreation(t *testing.T) { var ( mux = new(event.TypeMux) - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() txFeed = new(event.Feed) rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) @@ -352,7 +352,7 @@ func TestLogFilter(t *testing.T) { var ( mux = new(event.TypeMux) - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() txFeed = new(event.Feed) rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) @@ -471,7 +471,7 @@ func TestPendingLogsSubscription(t *testing.T) { var ( mux = new(event.TypeMux) - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() txFeed = new(event.Feed) rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) diff --git a/eth/handler_test.go b/eth/handler_test.go index e336dfa285..fee4114eb6 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -366,7 +366,7 @@ func testGetNodeData(t *testing.T, protocol int) { t.Errorf("data hash mismatch: have %x, want %x", hash, want) } } - statedb, _ := ethdb.NewMemDatabase() + statedb := ethdb.NewMemDatabase() for i := 0; i < len(data); i++ { statedb.Put(hashes[i].Bytes(), data[i]) } @@ -468,7 +468,7 @@ func testDAOChallenge(t *testing.T, localForked, remoteForked bool, timeout bool var ( evmux = new(event.TypeMux) pow = ethash.NewFaker() - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() config = ¶ms.ChainConfig{DAOForkBlock: big.NewInt(1), DAOForkSupport: localForked} gspec = &core.Genesis{Config: config} genesis = gspec.MustCommit(db) diff --git a/eth/helper_test.go b/eth/helper_test.go index 2b05cea801..8a0260fc95 100644 --- a/eth/helper_test.go +++ b/eth/helper_test.go @@ -53,7 +53,7 @@ func newTestProtocolManager(mode downloader.SyncMode, blocks int, generator func var ( evmux = new(event.TypeMux) engine = ethash.NewFaker() - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() gspec = &core.Genesis{ Config: params.TestChainConfig, Alloc: core.GenesisAlloc{testBank: {Balance: big.NewInt(1000000)}}, diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index bf8120228f..d25fc459a1 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -159,8 +159,7 @@ func TestCallTracer(t *testing.T) { GasLimit: uint64(test.Context.GasLimit), GasPrice: tx.GasPrice(), } - db, _ := ethdb.NewMemDatabase() - statedb := tests.MakePreState(db, test.Genesis.Alloc) + statedb := tests.MakePreState(ethdb.NewMemDatabase(), test.Genesis.Alloc) // Create the tracer, the EVM environment and run it tracer, err := New("callTracer") diff --git a/ethdb/database_test.go b/ethdb/database_test.go index 5e4a3ca34a..2deb50988c 100644 --- a/ethdb/database_test.go +++ b/ethdb/database_test.go @@ -53,8 +53,7 @@ func TestLDB_PutGet(t *testing.T) { } func TestMemoryDB_PutGet(t *testing.T) { - db, _ := ethdb.NewMemDatabase() - testPutGet(db, t) + testPutGet(ethdb.NewMemDatabase(), t) } func testPutGet(db ethdb.Database, t *testing.T) { @@ -131,8 +130,7 @@ func TestLDB_ParallelPutGet(t *testing.T) { } func TestMemoryDB_ParallelPutGet(t *testing.T) { - db, _ := ethdb.NewMemDatabase() - testParallelPutGet(db, t) + testParallelPutGet(ethdb.NewMemDatabase(), t) } func testParallelPutGet(db ethdb.Database, t *testing.T) { diff --git a/ethdb/memory_database.go b/ethdb/memory_database.go index 8efd7bf845..c57042920a 100644 --- a/ethdb/memory_database.go +++ b/ethdb/memory_database.go @@ -31,16 +31,16 @@ type MemDatabase struct { lock sync.RWMutex } -func NewMemDatabase() (*MemDatabase, error) { +func NewMemDatabase() *MemDatabase { return &MemDatabase{ db: make(map[string][]byte), - }, nil + } } -func NewMemDatabaseWithCap(size int) (*MemDatabase, error) { +func NewMemDatabaseWithCap(size int) *MemDatabase { return &MemDatabase{ db: make(map[string][]byte, size), - }, nil + } } func (db *MemDatabase) Put(key []byte, value []byte) error { diff --git a/les/handler_test.go b/les/handler_test.go index fa38d1d4f7..31aad3ed45 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -51,8 +51,7 @@ func TestGetBlockHeadersLes1(t *testing.T) { testGetBlockHeaders(t, 1) } func TestGetBlockHeadersLes2(t *testing.T) { testGetBlockHeaders(t, 2) } func testGetBlockHeaders(t *testing.T, protocol int) { - db, _ := ethdb.NewMemDatabase() - pm := newTestProtocolManagerMust(t, false, downloader.MaxHashFetch+15, nil, nil, nil, db) + pm := newTestProtocolManagerMust(t, false, downloader.MaxHashFetch+15, nil, nil, nil, ethdb.NewMemDatabase()) bc := pm.blockchain.(*core.BlockChain) peer, _ := newTestPeer(t, "peer", protocol, pm, true) defer peer.close() @@ -181,8 +180,7 @@ func TestGetBlockBodiesLes1(t *testing.T) { testGetBlockBodies(t, 1) } func TestGetBlockBodiesLes2(t *testing.T) { testGetBlockBodies(t, 2) } func testGetBlockBodies(t *testing.T, protocol int) { - db, _ := ethdb.NewMemDatabase() - pm := newTestProtocolManagerMust(t, false, downloader.MaxBlockFetch+15, nil, nil, nil, db) + pm := newTestProtocolManagerMust(t, false, downloader.MaxBlockFetch+15, nil, nil, nil, ethdb.NewMemDatabase()) bc := pm.blockchain.(*core.BlockChain) peer, _ := newTestPeer(t, "peer", protocol, pm, true) defer peer.close() @@ -259,8 +257,7 @@ func TestGetCodeLes2(t *testing.T) { testGetCode(t, 2) } func testGetCode(t *testing.T, protocol int) { // Assemble the test environment - db, _ := ethdb.NewMemDatabase() - pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, db) + pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, ethdb.NewMemDatabase()) bc := pm.blockchain.(*core.BlockChain) peer, _ := newTestPeer(t, "peer", protocol, pm, true) defer peer.close() @@ -293,7 +290,7 @@ func TestGetReceiptLes2(t *testing.T) { testGetReceipt(t, 2) } func testGetReceipt(t *testing.T, protocol int) { // Assemble the test environment - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, db) bc := pm.blockchain.(*core.BlockChain) peer, _ := newTestPeer(t, "peer", protocol, pm, true) @@ -321,7 +318,7 @@ func TestGetProofsLes2(t *testing.T) { testGetProofs(t, 2) } func testGetProofs(t *testing.T, protocol int) { // Assemble the test environment - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, db) bc := pm.blockchain.(*core.BlockChain) peer, _ := newTestPeer(t, "peer", protocol, pm, true) @@ -384,7 +381,7 @@ func testGetCHTProofs(t *testing.T, protocol int) { frequency = uint64(light.CHTFrequencyServer) } // Assemble the test environment - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() pm := newTestProtocolManagerMust(t, false, int(frequency)+light.HelperTrieProcessConfirmations, testChainGen, nil, nil, db) bc := pm.blockchain.(*core.BlockChain) peer, _ := newTestPeer(t, "peer", protocol, pm, true) @@ -452,7 +449,7 @@ func testGetCHTProofs(t *testing.T, protocol int) { // Tests that bloombits proofs can be correctly retrieved. func TestGetBloombitsProofs(t *testing.T) { // Assemble the test environment - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() pm := newTestProtocolManagerMust(t, false, light.BloomTrieFrequency+256, testChainGen, nil, nil, db) bc := pm.blockchain.(*core.BlockChain) peer, _ := newTestPeer(t, "peer", 2, pm, true) @@ -491,7 +488,7 @@ func TestGetBloombitsProofs(t *testing.T) { } func TestTransactionStatusLes2(t *testing.T) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() pm := newTestProtocolManagerMust(t, false, 0, nil, nil, nil, db) chain := pm.blockchain.(*core.BlockChain) config := core.DefaultTxPoolConfig diff --git a/les/odr_test.go b/les/odr_test.go index 9973df0430..983f7262b0 100644 --- a/les/odr_test.go +++ b/les/odr_test.go @@ -165,8 +165,8 @@ func testOdr(t *testing.T, protocol int, expFail uint64, fn odrTestFn) { peers := newPeerSet() dist := newRequestDistributor(peers, make(chan struct{})) rm := newRetrieveManager(peers, dist, nil) - db, _ := ethdb.NewMemDatabase() - ldb, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() + ldb := ethdb.NewMemDatabase() odr := NewLesOdr(ldb, light.NewChtIndexer(db, true), light.NewBloomTrieIndexer(db, true), eth.NewBloomIndexer(db, light.BloomTrieFrequency), rm) pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, db) lpm := newTestProtocolManagerMust(t, true, 0, nil, peers, odr, ldb) diff --git a/les/request_test.go b/les/request_test.go index d4282a5927..ba2f603d8b 100644 --- a/les/request_test.go +++ b/les/request_test.go @@ -87,8 +87,8 @@ func testAccess(t *testing.T, protocol int, fn accessTestFn) { peers := newPeerSet() dist := newRequestDistributor(peers, make(chan struct{})) rm := newRetrieveManager(peers, dist, nil) - db, _ := ethdb.NewMemDatabase() - ldb, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() + ldb := ethdb.NewMemDatabase() odr := NewLesOdr(ldb, light.NewChtIndexer(db, true), light.NewBloomTrieIndexer(db, true), eth.NewBloomIndexer(db, light.BloomTrieFrequency), rm) pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, db) diff --git a/light/lightchain_test.go b/light/lightchain_test.go index b8ab1c51cf..c0aa51da29 100644 --- a/light/lightchain_test.go +++ b/light/lightchain_test.go @@ -52,7 +52,7 @@ func makeHeaderChain(parent *types.Header, n int, db ethdb.Database, seed int) [ // chain. Depending on the full flag, if creates either a full block chain or a // header only chain. func newCanonical(n int) (ethdb.Database, *LightChain, error) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() gspec := core.Genesis{Config: params.TestChainConfig} genesis := gspec.MustCommit(db) blockchain, _ := NewLightChain(&dummyOdr{db: db}, gspec.Config, ethash.NewFaker()) @@ -69,7 +69,7 @@ func newCanonical(n int) (ethdb.Database, *LightChain, error) { // newTestLightChain creates a LightChain that doesn't validate anything. func newTestLightChain() *LightChain { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() gspec := &core.Genesis{ Difficulty: big.NewInt(1), Config: params.TestChainConfig, diff --git a/light/odr_test.go b/light/odr_test.go index cc7475df38..3e7ac10118 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -245,8 +245,8 @@ func testChainGen(i int, block *core.BlockGen) { func testChainOdr(t *testing.T, protocol int, fn odrTestFn) { var ( - sdb, _ = ethdb.NewMemDatabase() - ldb, _ = ethdb.NewMemDatabase() + sdb = ethdb.NewMemDatabase() + ldb = ethdb.NewMemDatabase() gspec = core.Genesis{Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}} genesis = gspec.MustCommit(sdb) ) diff --git a/light/trie_test.go b/light/trie_test.go index 0d6b2cc1d8..84c6f162fb 100644 --- a/light/trie_test.go +++ b/light/trie_test.go @@ -34,10 +34,10 @@ import ( func TestNodeIterator(t *testing.T) { var ( - fulldb, _ = ethdb.NewMemDatabase() - lightdb, _ = ethdb.NewMemDatabase() - gspec = core.Genesis{Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}} - genesis = gspec.MustCommit(fulldb) + fulldb = ethdb.NewMemDatabase() + lightdb = ethdb.NewMemDatabase() + gspec = core.Genesis{Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}} + genesis = gspec.MustCommit(fulldb) ) gspec.MustCommit(lightdb) blockchain, _ := core.NewBlockChain(fulldb, nil, params.TestChainConfig, ethash.NewFullFaker(), vm.Config{}) diff --git a/light/txpool_test.go b/light/txpool_test.go index 13d7d3cebb..ccbd83a943 100644 --- a/light/txpool_test.go +++ b/light/txpool_test.go @@ -81,8 +81,8 @@ func TestTxPool(t *testing.T) { } var ( - sdb, _ = ethdb.NewMemDatabase() - ldb, _ = ethdb.NewMemDatabase() + sdb = ethdb.NewMemDatabase() + ldb = ethdb.NewMemDatabase() gspec = core.Genesis{Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}} genesis = gspec.MustCommit(sdb) ) diff --git a/node/node.go b/node/node.go index 83b6c4c07e..c4368189f7 100644 --- a/node/node.go +++ b/node/node.go @@ -568,7 +568,7 @@ func (n *Node) EventMux() *event.TypeMux { // ephemeral, a memory database is returned. func (n *Node) OpenDatabase(name string, cache, handles int) (ethdb.Database, error) { if n.config.DataDir == "" { - return ethdb.NewMemDatabase() + return ethdb.NewMemDatabase(), nil } return ethdb.NewLDBDatabase(n.config.resolvePath(name), cache, handles) } diff --git a/node/service.go b/node/service.go index 55062a5004..afc43e8486 100644 --- a/node/service.go +++ b/node/service.go @@ -41,7 +41,7 @@ type ServiceContext struct { // node is an ephemeral one, a memory database is returned. func (ctx *ServiceContext) OpenDatabase(name string, cache int, handles int) (ethdb.Database, error) { if ctx.config.DataDir == "" { - return ethdb.NewMemDatabase() + return ethdb.NewMemDatabase(), nil } db, err := ethdb.NewLDBDatabase(ctx.config.resolvePath(name), cache, handles) if err != nil { diff --git a/tests/block_test_util.go b/tests/block_test_util.go index 579e783b10..a72799f6e2 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -98,7 +98,7 @@ func (t *BlockTest) Run() error { } // import pre accounts & construct test genesis block & state root - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() gblock, err := t.genesis(config).Commit(db) if err != nil { return err diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 3b761bd771..84581fae18 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -126,8 +126,7 @@ func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config) (*state.StateD return nil, UnsupportedForkError{subtest.Fork} } block := t.genesis(config).ToBlock(nil) - db, _ := ethdb.NewMemDatabase() - statedb := MakePreState(db, t.json.Pre) + statedb := MakePreState(ethdb.NewMemDatabase(), t.json.Pre) post := t.json.Post[subtest.Fork][subtest.Index] msg, err := t.json.Tx.toMessage(post) diff --git a/tests/vm_test_util.go b/tests/vm_test_util.go index b365167a69..cb81c5b94e 100644 --- a/tests/vm_test_util.go +++ b/tests/vm_test_util.go @@ -79,8 +79,7 @@ type vmExecMarshaling struct { } func (t *VMTest) Run(vmconfig vm.Config) error { - db, _ := ethdb.NewMemDatabase() - statedb := MakePreState(db, t.json.Pre) + statedb := MakePreState(ethdb.NewMemDatabase(), t.json.Pre) ret, gasRemaining, err := t.exec(statedb, vmconfig) if t.json.GasRemaining == nil { diff --git a/trie/iterator_test.go b/trie/iterator_test.go index dce1c78b5d..2a510b1c2d 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -289,7 +289,7 @@ func TestIteratorContinueAfterErrorDisk(t *testing.T) { testIteratorContinueA func TestIteratorContinueAfterErrorMemonly(t *testing.T) { testIteratorContinueAfterError(t, true) } func testIteratorContinueAfterError(t *testing.T, memonly bool) { - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) tr, _ := New(common.Hash{}, triedb) @@ -376,7 +376,7 @@ func TestIteratorContinueAfterSeekErrorMemonly(t *testing.T) { func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) { // Commit test trie to db, then remove the node containing "bars". - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) ctr, _ := New(common.Hash{}, triedb) diff --git a/trie/proof_test.go b/trie/proof_test.go index fff313d7fd..a3537787cc 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -36,7 +36,7 @@ func TestProof(t *testing.T) { trie, vals := randomTrie(500) root := trie.Hash() for _, kv := range vals { - proofs, _ := ethdb.NewMemDatabase() + proofs := ethdb.NewMemDatabase() if trie.Prove(kv.k, 0, proofs) != nil { t.Fatalf("missing key %x while constructing proof", kv.k) } @@ -53,7 +53,7 @@ func TestProof(t *testing.T) { func TestOneElementProof(t *testing.T) { trie := new(Trie) updateString(trie, "k", "v") - proofs, _ := ethdb.NewMemDatabase() + proofs := ethdb.NewMemDatabase() trie.Prove([]byte("k"), 0, proofs) if len(proofs.Keys()) != 1 { t.Error("proof should have one element") @@ -71,7 +71,7 @@ func TestVerifyBadProof(t *testing.T) { trie, vals := randomTrie(800) root := trie.Hash() for _, kv := range vals { - proofs, _ := ethdb.NewMemDatabase() + proofs := ethdb.NewMemDatabase() trie.Prove(kv.k, 0, proofs) if len(proofs.Keys()) == 0 { t.Fatal("zero length proof") @@ -109,7 +109,7 @@ func BenchmarkProve(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { kv := vals[keys[i%len(keys)]] - proofs, _ := ethdb.NewMemDatabase() + proofs := ethdb.NewMemDatabase() if trie.Prove(kv.k, 0, proofs); len(proofs.Keys()) == 0 { b.Fatalf("zero length proof for %x", kv.k) } @@ -123,7 +123,7 @@ func BenchmarkVerifyProof(b *testing.B) { var proofs []*ethdb.MemDatabase for k := range vals { keys = append(keys, k) - proof, _ := ethdb.NewMemDatabase() + proof := ethdb.NewMemDatabase() trie.Prove([]byte(k), 0, proof) proofs = append(proofs, proof) } diff --git a/trie/secure_trie_test.go b/trie/secure_trie_test.go index aedf5a1cde..d16d999684 100644 --- a/trie/secure_trie_test.go +++ b/trie/secure_trie_test.go @@ -28,18 +28,14 @@ import ( ) func newEmptySecure() *SecureTrie { - diskdb, _ := ethdb.NewMemDatabase() - triedb := NewDatabase(diskdb) - - trie, _ := NewSecure(common.Hash{}, triedb, 0) + trie, _ := NewSecure(common.Hash{}, NewDatabase(ethdb.NewMemDatabase()), 0) return trie } // makeTestSecureTrie creates a large enough secure trie for testing. func makeTestSecureTrie() (*Database, *SecureTrie, map[string][]byte) { // Create an empty trie - diskdb, _ := ethdb.NewMemDatabase() - triedb := NewDatabase(diskdb) + triedb := NewDatabase(ethdb.NewMemDatabase()) trie, _ := NewSecure(common.Hash{}, triedb, 0) diff --git a/trie/sync_test.go b/trie/sync_test.go index 4a720612b6..142a6f5b1a 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -27,8 +27,7 @@ import ( // makeTestTrie create a sample test trie to test node-wise reconstruction. func makeTestTrie() (*Database, *Trie, map[string][]byte) { // Create an empty trie - diskdb, _ := ethdb.NewMemDatabase() - triedb := NewDatabase(diskdb) + triedb := NewDatabase(ethdb.NewMemDatabase()) trie, _ := New(common.Hash{}, triedb) // Fill it with some arbitrary data @@ -89,18 +88,13 @@ func checkTrieConsistency(db *Database, root common.Hash) error { // Tests that an empty trie is not scheduled for syncing. func TestEmptyTrieSync(t *testing.T) { - diskdbA, _ := ethdb.NewMemDatabase() - triedbA := NewDatabase(diskdbA) - - diskdbB, _ := ethdb.NewMemDatabase() - triedbB := NewDatabase(diskdbB) - - emptyA, _ := New(common.Hash{}, triedbA) - emptyB, _ := New(emptyRoot, triedbB) + dbA := NewDatabase(ethdb.NewMemDatabase()) + dbB := NewDatabase(ethdb.NewMemDatabase()) + emptyA, _ := New(common.Hash{}, dbA) + emptyB, _ := New(emptyRoot, dbB) for i, trie := range []*Trie{emptyA, emptyB} { - diskdb, _ := ethdb.NewMemDatabase() - if req := NewTrieSync(trie.Hash(), diskdb, nil).Missing(1); len(req) != 0 { + if req := NewTrieSync(trie.Hash(), ethdb.NewMemDatabase(), nil).Missing(1); len(req) != 0 { t.Errorf("test %d: content requested for empty trie: %v", i, req) } } @@ -116,7 +110,7 @@ func testIterativeTrieSync(t *testing.T, batch int) { srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) sched := NewTrieSync(srcTrie.Hash(), diskdb, nil) @@ -149,7 +143,7 @@ func TestIterativeDelayedTrieSync(t *testing.T) { srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) sched := NewTrieSync(srcTrie.Hash(), diskdb, nil) @@ -187,7 +181,7 @@ func testIterativeRandomTrieSync(t *testing.T, batch int) { srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) sched := NewTrieSync(srcTrie.Hash(), diskdb, nil) @@ -228,7 +222,7 @@ func TestIterativeRandomDelayedTrieSync(t *testing.T) { srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) sched := NewTrieSync(srcTrie.Hash(), diskdb, nil) @@ -275,7 +269,7 @@ func TestDuplicateAvoidanceTrieSync(t *testing.T) { srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) sched := NewTrieSync(srcTrie.Hash(), diskdb, nil) @@ -315,7 +309,7 @@ func TestIncompleteTrieSync(t *testing.T) { srcDb, srcTrie, _ := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) sched := NewTrieSync(srcTrie.Hash(), diskdb, nil) diff --git a/trie/trie_test.go b/trie/trie_test.go index 9972226288..f8e5fd12a1 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -43,8 +43,7 @@ func init() { // Used for testing func newEmpty() *Trie { - diskdb, _ := ethdb.NewMemDatabase() - trie, _ := New(common.Hash{}, NewDatabase(diskdb)) + trie, _ := New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase())) return trie } @@ -68,8 +67,7 @@ func TestNull(t *testing.T) { } func TestMissingRoot(t *testing.T) { - diskdb, _ := ethdb.NewMemDatabase() - trie, err := New(common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), NewDatabase(diskdb)) + trie, err := New(common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), NewDatabase(ethdb.NewMemDatabase())) if trie != nil { t.Error("New returned non-nil trie for invalid root") } @@ -82,7 +80,7 @@ func TestMissingNodeDisk(t *testing.T) { testMissingNode(t, false) } func TestMissingNodeMemonly(t *testing.T) { testMissingNode(t, true) } func testMissingNode(t *testing.T, memonly bool) { - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) trie, _ := New(common.Hash{}, triedb) @@ -413,8 +411,7 @@ func (randTest) Generate(r *rand.Rand, size int) reflect.Value { } func runRandTest(rt randTest) bool { - diskdb, _ := ethdb.NewMemDatabase() - triedb := NewDatabase(diskdb) + triedb := NewDatabase(ethdb.NewMemDatabase()) tr, _ := New(common.Hash{}, triedb) values := make(map[string]string) // tracks content of the trie From 53a18d2e2734d078200ec607055ae551245ae74b Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 10 May 2018 12:26:36 +0200 Subject: [PATCH 13/46] event: document select case slice use and add edge case test (#16680) Feed keeps active subscription channels in a slice called 'f.sendCases'. The Send method tracks the active cases in a local variable 'cases' whose value is f.sendCases initially. 'cases' shrinks to a shorter prefix of f.sendCases every time a send succeeds, moving the successful case out of range of the active case list. This can be confusing because the two slices share a backing array. Add more comments to document what is going on. Also add a test for removing a case that is in 'f.sentCases' but not 'cases'. --- event/feed.go | 5 ++++- event/feed_test.go | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/event/feed.go b/event/feed.go index 78fa3d98d8..f578f00c10 100644 --- a/event/feed.go +++ b/event/feed.go @@ -148,7 +148,9 @@ func (f *Feed) Send(value interface{}) (nsent int) { f.sendCases[i].Send = rvalue } - // Send until all channels except removeSub have been chosen. + // Send until all channels except removeSub have been chosen. 'cases' tracks a prefix + // of sendCases. When a send succeeds, the corresponding case moves to the end of + // 'cases' and it shrinks by one element. cases := f.sendCases for { // Fast path: try sending without blocking before adding to the select set. @@ -170,6 +172,7 @@ func (f *Feed) Send(value interface{}) (nsent int) { index := f.sendCases.find(recv.Interface()) f.sendCases = f.sendCases.delete(index) if index >= 0 && index < len(cases) { + // Shrink 'cases' too because the removed case was still active. cases = f.sendCases[:len(cases)-1] } } else { diff --git a/event/feed_test.go b/event/feed_test.go index a82c103033..be8876932c 100644 --- a/event/feed_test.go +++ b/event/feed_test.go @@ -235,6 +235,45 @@ func TestFeedUnsubscribeBlockedPost(t *testing.T) { wg.Wait() } +// Checks that unsubscribing a channel during Send works even if that +// channel has already been sent on. +func TestFeedUnsubscribeSentChan(t *testing.T) { + var ( + feed Feed + ch1 = make(chan int) + ch2 = make(chan int) + sub1 = feed.Subscribe(ch1) + sub2 = feed.Subscribe(ch2) + wg sync.WaitGroup + ) + defer sub2.Unsubscribe() + + wg.Add(1) + go func() { + feed.Send(0) + wg.Done() + }() + + // Wait for the value on ch1. + <-ch1 + // Unsubscribe ch1, removing it from the send cases. + sub1.Unsubscribe() + + // Receive ch2, finishing Send. + <-ch2 + wg.Wait() + + // Send again. This should send to ch2 only, so the wait group will unblock + // as soon as a value is received on ch2. + wg.Add(1) + go func() { + feed.Send(0) + wg.Done() + }() + <-ch2 + wg.Wait() +} + func TestFeedUnsubscribeFromInbox(t *testing.T) { var ( feed Feed From fcc18f4c80a5a7e6cc39315fb1eeb2222c811701 Mon Sep 17 00:00:00 2001 From: ligi Date: Thu, 10 May 2018 12:33:13 +0200 Subject: [PATCH 14/46] travis: use Android NDK 16b (#16562) --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 194ca63969..ce6500c46a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -152,10 +152,10 @@ matrix: - export GOPATH=$HOME/go script: # Build the Android archive and upload it to Maven Central and Azure - - curl https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip -o android-ndk-r15c.zip - - unzip -q android-ndk-r15c.zip && rm android-ndk-r15c.zip - - mv android-ndk-r15c $HOME - - export ANDROID_NDK=$HOME/android-ndk-r15c + - curl https://dl.google.com/android/repository/android-ndk-r16b-linux-x86_64.zip -o android-ndk-r16b.zip + - unzip -q android-ndk-r16b.zip && rm android-ndk-r16b.zip + - mv android-ndk-r16b $HOME + - export ANDROID_NDK=$HOME/android-ndk-r16b - mkdir -p $GOPATH/src/github.com/ethereum - ln -s `pwd` $GOPATH/src/github.com/ethereum From 784aa83942e3dbc9bab0385475dbf3755a9892ac Mon Sep 17 00:00:00 2001 From: kiel barry Date: Thu, 10 May 2018 03:36:01 -0700 Subject: [PATCH 15/46] bmt: golint updates for this or self warning (#16628) * bmt/*: golint updates for this or self warning * Update bmt.go --- bmt/bmt.go | 178 ++++++++++++++++++++++++++--------------------------- 1 file changed, 89 insertions(+), 89 deletions(-) diff --git a/bmt/bmt.go b/bmt/bmt.go index 3408758675..c290223452 100644 --- a/bmt/bmt.go +++ b/bmt/bmt.go @@ -150,29 +150,29 @@ func NewTreePool(hasher BaseHasher, segmentCount, capacity int) *TreePool { } // Drain drains the pool until it has no more than n resources -func (self *TreePool) Drain(n int) { - self.lock.Lock() - defer self.lock.Unlock() - for len(self.c) > n { - <-self.c - self.count-- +func (p *TreePool) Drain(n int) { + p.lock.Lock() + defer p.lock.Unlock() + for len(p.c) > n { + <-p.c + p.count-- } } // Reserve is blocking until it returns an available Tree // it reuses free Trees or creates a new one if size is not reached -func (self *TreePool) Reserve() *Tree { - self.lock.Lock() - defer self.lock.Unlock() +func (p *TreePool) Reserve() *Tree { + p.lock.Lock() + defer p.lock.Unlock() var t *Tree - if self.count == self.Capacity { - return <-self.c + if p.count == p.Capacity { + return <-p.c } select { - case t = <-self.c: + case t = <-p.c: default: - t = NewTree(self.hasher, self.SegmentSize, self.SegmentCount) - self.count++ + t = NewTree(p.hasher, p.SegmentSize, p.SegmentCount) + p.count++ } return t } @@ -180,8 +180,8 @@ func (self *TreePool) Reserve() *Tree { // Release gives back a Tree to the pool. // This Tree is guaranteed to be in reusable state // does not need locking -func (self *TreePool) Release(t *Tree) { - self.c <- t // can never fail but... +func (p *TreePool) Release(t *Tree) { + p.c <- t // can never fail but... } // Tree is a reusable control structure representing a BMT @@ -193,17 +193,17 @@ type Tree struct { } // Draw draws the BMT (badly) -func (self *Tree) Draw(hash []byte, d int) string { +func (t *Tree) Draw(hash []byte, d int) string { var left, right []string var anc []*Node - for i, n := range self.leaves { + for i, n := range t.leaves { left = append(left, fmt.Sprintf("%v", hashstr(n.left))) if i%2 == 0 { anc = append(anc, n.parent) } right = append(right, fmt.Sprintf("%v", hashstr(n.right))) } - anc = self.leaves + anc = t.leaves var hashes [][]string for l := 0; len(anc) > 0; l++ { var nodes []*Node @@ -277,42 +277,42 @@ func NewTree(hasher BaseHasher, segmentSize, segmentCount int) *Tree { // methods needed by hash.Hash // Size returns the size -func (self *Hasher) Size() int { - return self.size +func (h *Hasher) Size() int { + return h.size } // BlockSize returns the block size -func (self *Hasher) BlockSize() int { - return self.blocksize +func (h *Hasher) BlockSize() int { + return h.blocksize } // Sum returns the hash of the buffer // hash.Hash interface Sum method appends the byte slice to the underlying // data before it calculates and returns the hash of the chunk -func (self *Hasher) Sum(b []byte) (r []byte) { - t := self.bmt - i := self.cur +func (h *Hasher) Sum(b []byte) (r []byte) { + t := h.bmt + i := h.cur n := t.leaves[i] j := i // must run strictly before all nodes calculate // datanodes are guaranteed to have a parent - if len(self.segment) > self.size && i > 0 && n.parent != nil { + if len(h.segment) > h.size && i > 0 && n.parent != nil { n = n.parent } else { i *= 2 } - d := self.finalise(n, i) - self.writeSegment(j, self.segment, d) - c := <-self.result - self.releaseTree() + d := h.finalise(n, i) + h.writeSegment(j, h.segment, d) + c := <-h.result + h.releaseTree() // sha3(length + BMT(pure_chunk)) - if self.blockLength == nil { + if h.blockLength == nil { return c } - res := self.pool.hasher() + res := h.pool.hasher() res.Reset() - res.Write(self.blockLength) + res.Write(h.blockLength) res.Write(c) return res.Sum(nil) } @@ -321,8 +321,8 @@ func (self *Hasher) Sum(b []byte) (r []byte) { // Hash waits for the hasher result and returns it // caller must call this on a BMT Hasher being written to -func (self *Hasher) Hash() []byte { - return <-self.result +func (h *Hasher) Hash() []byte { + return <-h.result } // Hasher implements the io.Writer interface @@ -330,16 +330,16 @@ func (self *Hasher) Hash() []byte { // Write fills the buffer to hash // with every full segment complete launches a hasher go routine // that shoots up the BMT -func (self *Hasher) Write(b []byte) (int, error) { +func (h *Hasher) Write(b []byte) (int, error) { l := len(b) if l <= 0 { return 0, nil } - s := self.segment - i := self.cur - count := (self.count + 1) / 2 - need := self.count*self.size - self.cur*2*self.size - size := self.size + s := h.segment + i := h.cur + count := (h.count + 1) / 2 + need := h.count*h.size - h.cur*2*h.size + size := h.size if need > size { size *= 2 } @@ -356,7 +356,7 @@ func (self *Hasher) Write(b []byte) (int, error) { // read full segments and the last possibly partial segment for need > 0 && i < count-1 { // push all finished chunks we read - self.writeSegment(i, s, self.depth) + h.writeSegment(i, s, h.depth) need -= size if need < 0 { size += need @@ -365,8 +365,8 @@ func (self *Hasher) Write(b []byte) (int, error) { rest += size i++ } - self.segment = s - self.cur = i + h.segment = s + h.cur = i // otherwise, we can assume len(s) == 0, so all buffer is read and chunk is not yet full return l, nil } @@ -376,8 +376,8 @@ func (self *Hasher) Write(b []byte) (int, error) { // ReadFrom reads from io.Reader and appends to the data to hash using Write // it reads so that chunk to hash is maximum length or reader reaches EOF // caller must Reset the hasher prior to call -func (self *Hasher) ReadFrom(r io.Reader) (m int64, err error) { - bufsize := self.size*self.count - self.size*self.cur - len(self.segment) +func (h *Hasher) ReadFrom(r io.Reader) (m int64, err error) { + bufsize := h.size*h.count - h.size*h.cur - len(h.segment) buf := make([]byte, bufsize) var read int for { @@ -385,7 +385,7 @@ func (self *Hasher) ReadFrom(r io.Reader) (m int64, err error) { n, err = r.Read(buf) read += n if err == io.EOF || read == len(buf) { - hash := self.Sum(buf[:n]) + hash := h.Sum(buf[:n]) if read == len(buf) { err = NewEOC(hash) } @@ -394,7 +394,7 @@ func (self *Hasher) ReadFrom(r io.Reader) (m int64, err error) { if err != nil { break } - n, err = self.Write(buf[:n]) + n, err = h.Write(buf[:n]) if err != nil { break } @@ -403,9 +403,9 @@ func (self *Hasher) ReadFrom(r io.Reader) (m int64, err error) { } // Reset needs to be called before writing to the hasher -func (self *Hasher) Reset() { - self.getTree() - self.blockLength = nil +func (h *Hasher) Reset() { + h.getTree() + h.blockLength = nil } // Hasher implements the SwarmHash interface @@ -413,52 +413,52 @@ func (self *Hasher) Reset() { // ResetWithLength needs to be called before writing to the hasher // the argument is supposed to be the byte slice binary representation of // the length of the data subsumed under the hash -func (self *Hasher) ResetWithLength(l []byte) { - self.Reset() - self.blockLength = l +func (h *Hasher) ResetWithLength(l []byte) { + h.Reset() + h.blockLength = l } // Release gives back the Tree to the pool whereby it unlocks // it resets tree, segment and index -func (self *Hasher) releaseTree() { - if self.bmt != nil { - n := self.bmt.leaves[self.cur] +func (h *Hasher) releaseTree() { + if h.bmt != nil { + n := h.bmt.leaves[h.cur] for ; n != nil; n = n.parent { n.unbalanced = false if n.parent != nil { n.root = false } } - self.pool.Release(self.bmt) - self.bmt = nil + h.pool.Release(h.bmt) + h.bmt = nil } - self.cur = 0 - self.segment = nil + h.cur = 0 + h.segment = nil } -func (self *Hasher) writeSegment(i int, s []byte, d int) { - h := self.pool.hasher() - n := self.bmt.leaves[i] +func (h *Hasher) writeSegment(i int, s []byte, d int) { + hash := h.pool.hasher() + n := h.bmt.leaves[i] - if len(s) > self.size && n.parent != nil { + if len(s) > h.size && n.parent != nil { go func() { - h.Reset() - h.Write(s) - s = h.Sum(nil) + hash.Reset() + hash.Write(s) + s = hash.Sum(nil) if n.root { - self.result <- s + h.result <- s return } - self.run(n.parent, h, d, n.index, s) + h.run(n.parent, hash, d, n.index, s) }() return } - go self.run(n, h, d, i*2, s) + go h.run(n, hash, d, i*2, s) } -func (self *Hasher) run(n *Node, h hash.Hash, d int, i int, s []byte) { +func (h *Hasher) run(n *Node, hash hash.Hash, d int, i int, s []byte) { isLeft := i%2 == 0 for { if isLeft { @@ -470,18 +470,18 @@ func (self *Hasher) run(n *Node, h hash.Hash, d int, i int, s []byte) { return } if !n.unbalanced || !isLeft || i == 0 && d == 0 { - h.Reset() - h.Write(n.left) - h.Write(n.right) - s = h.Sum(nil) + hash.Reset() + hash.Write(n.left) + hash.Write(n.right) + s = hash.Sum(nil) } else { s = append(n.left, n.right...) } - self.hash = s + h.hash = s if n.root { - self.result <- s + h.result <- s return } @@ -492,20 +492,20 @@ func (self *Hasher) run(n *Node, h hash.Hash, d int, i int, s []byte) { } // getTree obtains a BMT resource by reserving one from the pool -func (self *Hasher) getTree() *Tree { - if self.bmt != nil { - return self.bmt +func (h *Hasher) getTree() *Tree { + if h.bmt != nil { + return h.bmt } - t := self.pool.Reserve() - self.bmt = t + t := h.pool.Reserve() + h.bmt = t return t } // atomic bool toggle implementing a concurrent reusable 2-state object // atomic addint with %2 implements atomic bool toggle // it returns true if the toggler just put it in the active/waiting state -func (self *Node) toggle() bool { - return atomic.AddInt32(&self.state, 1)%2 == 1 +func (n *Node) toggle() bool { + return atomic.AddInt32(&n.state, 1)%2 == 1 } func hashstr(b []byte) string { @@ -525,7 +525,7 @@ func depth(n int) (d int) { // finalise is following the zigzags on the tree belonging // to the final datasegment -func (self *Hasher) finalise(n *Node, i int) (d int) { +func (h *Hasher) finalise(n *Node, i int) (d int) { isLeft := i%2 == 0 for { // when the final segment's path is going via left segments @@ -550,8 +550,8 @@ type EOC struct { } // Error returns the error string -func (self *EOC) Error() string { - return fmt.Sprintf("hasher limit reached, chunk hash: %x", self.Hash) +func (e *EOC) Error() string { + return fmt.Sprintf("hasher limit reached, chunk hash: %x", e.Hash) } // NewEOC creates new end of chunk error with the hash From 595b47e5359568a49d36891f7ca60737c720cfc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Mon, 14 May 2018 11:23:58 +0200 Subject: [PATCH 16/46] light: new CHT for mainnet and ropsten (#16736) --- light/postprocess.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/light/postprocess.go b/light/postprocess.go index 8b9a029112..c06c18027e 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -59,18 +59,18 @@ type trustedCheckpoint struct { var ( mainnetCheckpoint = trustedCheckpoint{ name: "mainnet", - sectionIdx: 165, - sectionHead: common.HexToHash("21028acf9cd9ce80257221adc437c3c58ce046c4d43c21c3e9b1d1349059ec73"), - chtRoot: common.HexToHash("26b2458cb7d0080d3a39311c914be92c368777a65ec074e1893b8bdc79e3910a"), - bloomTrieRoot: common.HexToHash("5d06908769179186165a72db7fc3473b25c28ed27efe78a392a9ff2c3fa67f84"), + sectionIdx: 170, + sectionHead: common.HexToHash("3bb2c28bcce463d57968f14f56cdb3fbf35349ab7a701f44c1afb57349c9a356"), + chtRoot: common.HexToHash("d92b6d0853455f8439086292338e87f69781921680dd7aa072fb71547b87415e"), + bloomTrieRoot: common.HexToHash("e4e8250a2fefddead7ae42daecd848cbf9b66d748a8270f8bbd4370b764bb9e9"), } ropstenCheckpoint = trustedCheckpoint{ name: "ropsten", - sectionIdx: 92, - sectionHead: common.HexToHash("21a158f9cc643da13a237cafceb37381072649f7278cf98c5820bfbced7cfcec"), - chtRoot: common.HexToHash("1a8ddb8b086d7a33ca90eea90730225948fa504ae0283b15aff3c15c0e089bf9"), - bloomTrieRoot: common.HexToHash("fd192f92afbcdd0020c81ca0625116b5995509659653b10123bd986fe5129cc1"), + sectionIdx: 97, + sectionHead: common.HexToHash("719448c67c01eb5b9f27833a36a4e34612f66801316d7ff37daf9e77fb4cd095"), + chtRoot: common.HexToHash("a7857afc15930ca6e583b6c3d563a025144011655843d52d28e2fdaadd417bea"), + bloomTrieRoot: common.HexToHash("9c71d4b50cbec86dfeaa8e08992de8a4667b81d13c54d6522b17ce2fc5d36416"), } ) From 2688dab48c9b8ae71b8d95c7678817eaa17f2b53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 14 May 2018 13:14:17 +0300 Subject: [PATCH 17/46] params: release go-ethereum v1.8.8 --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 8b96f25167..5d0ab7c031 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 8 // Minor version component of the current release - VersionPatch = 8 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 8 // Minor version component of the current release + VersionPatch = 8 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From 49ec4f0cd1f4d4c84c13ccd8d920d56112d7268a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 14 May 2018 13:16:55 +0300 Subject: [PATCH 18/46] VERSION, params: start 1.8.9 release cycle --- VERSION | 2 +- params/version.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/VERSION b/VERSION index 1790d35988..53ed4ba51e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.8.8 +1.8.9 diff --git a/params/version.go b/params/version.go index 5d0ab7c031..1e8c43bf81 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 8 // Minor version component of the current release - VersionPatch = 8 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 8 // Minor version component of the current release + VersionPatch = 9 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From 247b5f03690b47fdcb862c9a1173365cbdd9d279 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Mon, 14 May 2018 14:47:31 +0200 Subject: [PATCH 19/46] accounts/abi: allow abi: tags when unpacking structs Go code users can now tag event struct members with `abi:` to specify in what fields the event will be de-serialized. See PR #16648 for details. --- accounts/abi/argument.go | 44 +++++++-------- accounts/abi/event_test.go | 81 +++++++++++++++++++++++++++- accounts/abi/reflect.go | 106 +++++++++++++++++++++++++++++++++---- 3 files changed, 195 insertions(+), 36 deletions(-) diff --git a/accounts/abi/argument.go b/accounts/abi/argument.go index 512d8fdfa7..93b513c346 100644 --- a/accounts/abi/argument.go +++ b/accounts/abi/argument.go @@ -111,9 +111,14 @@ func (arguments Arguments) unpackTuple(v interface{}, marshalledValues []interfa if err := requireUnpackKind(value, typ, kind, arguments); err != nil { return err } - // If the output interface is a struct, make sure names don't collide + + // If the interface is a struct, get of abi->struct_field mapping + + var abi2struct map[string]string if kind == reflect.Struct { - if err := requireUniqueStructFieldNames(arguments); err != nil { + var err error + abi2struct, err = mapAbiToStructFields(arguments, value) + if err != nil { return err } } @@ -123,9 +128,10 @@ func (arguments Arguments) unpackTuple(v interface{}, marshalledValues []interfa switch kind { case reflect.Struct: - err := unpackStruct(value, reflectValue, arg) - if err != nil { - return err + if structField, ok := abi2struct[arg.Name]; ok { + if err := set(value.FieldByName(structField), reflectValue, arg); err != nil { + return err + } } case reflect.Slice, reflect.Array: if value.Len() < i { @@ -151,17 +157,22 @@ func (arguments Arguments) unpackAtomic(v interface{}, marshalledValues []interf if len(marshalledValues) != 1 { return fmt.Errorf("abi: wrong length, expected single value, got %d", len(marshalledValues)) } + elem := reflect.ValueOf(v).Elem() kind := elem.Kind() reflectValue := reflect.ValueOf(marshalledValues[0]) + var abi2struct map[string]string if kind == reflect.Struct { - //make sure names don't collide - if err := requireUniqueStructFieldNames(arguments); err != nil { + var err error + if abi2struct, err = mapAbiToStructFields(arguments, elem); err != nil { return err } - - return unpackStruct(elem, reflectValue, arguments[0]) + arg := arguments.NonIndexed()[0] + if structField, ok := abi2struct[arg.Name]; ok { + return set(elem.FieldByName(structField), reflectValue, arg) + } + return nil } return set(elem, reflectValue, arguments.NonIndexed()[0]) @@ -277,18 +288,3 @@ func capitalise(input string) string { } return strings.ToUpper(input[:1]) + input[1:] } - -//unpackStruct extracts each argument into its corresponding struct field -func unpackStruct(value, reflectValue reflect.Value, arg Argument) error { - name := capitalise(arg.Name) - typ := value.Type() - for j := 0; j < typ.NumField(); j++ { - // TODO read tags: `abi:"fieldName"` - if typ.Field(j).Name == name { - if err := set(value.Field(j), reflectValue, arg); err != nil { - return err - } - } - } - return nil -} diff --git a/accounts/abi/event_test.go b/accounts/abi/event_test.go index cca61e433d..3bfdd7c0ab 100644 --- a/accounts/abi/event_test.go +++ b/accounts/abi/event_test.go @@ -58,12 +58,28 @@ var jsonEventPledge = []byte(`{ "type": "event" }`) +var jsonEventMixedCase = []byte(`{ + "anonymous": false, + "inputs": [{ + "indexed": false, "name": "value", "type": "uint256" + }, { + "indexed": false, "name": "_value", "type": "uint256" + }, { + "indexed": false, "name": "Value", "type": "uint256" + }], + "name": "MixedCase", + "type": "event" + }`) + // 1000000 var transferData1 = "00000000000000000000000000000000000000000000000000000000000f4240" // "0x00Ce0d46d924CC8437c806721496599FC3FFA268", 2218516807680, "usd" var pledgeData1 = "00000000000000000000000000ce0d46d924cc8437c806721496599fc3ffa2680000000000000000000000000000000000000000000000000000020489e800007573640000000000000000000000000000000000000000000000000000000000" +// 1000000,2218516807680,1000001 +var mixedCaseData1 = "00000000000000000000000000000000000000000000000000000000000f42400000000000000000000000000000000000000000000000000000020489e8000000000000000000000000000000000000000000000000000000000000000f4241" + func TestEventId(t *testing.T) { var table = []struct { definition string @@ -121,6 +137,27 @@ func TestEventTupleUnpack(t *testing.T) { Value *big.Int } + type EventTransferWithTag struct { + // this is valid because `value` is not exportable, + // so value is only unmarshalled into `Value1`. + value *big.Int + Value1 *big.Int `abi:"value"` + } + + type BadEventTransferWithSameFieldAndTag struct { + Value *big.Int + Value1 *big.Int `abi:"value"` + } + + type BadEventTransferWithDuplicatedTag struct { + Value1 *big.Int `abi:"value"` + Value2 *big.Int `abi:"value"` + } + + type BadEventTransferWithEmptyTag struct { + Value *big.Int `abi:""` + } + type EventPledge struct { Who common.Address Wad *big.Int @@ -133,9 +170,16 @@ func TestEventTupleUnpack(t *testing.T) { Currency [3]byte } + type EventMixedCase struct { + Value1 *big.Int `abi:"value"` + Value2 *big.Int `abi:"_value"` + Value3 *big.Int `abi:"Value"` + } + bigint := new(big.Int) bigintExpected := big.NewInt(1000000) bigintExpected2 := big.NewInt(2218516807680) + bigintExpected3 := big.NewInt(1000001) addr := common.HexToAddress("0x00Ce0d46d924CC8437c806721496599FC3FFA268") var testCases = []struct { data string @@ -158,6 +202,34 @@ func TestEventTupleUnpack(t *testing.T) { jsonEventTransfer, "", "Can unpack ERC20 Transfer event into slice", + }, { + transferData1, + &EventTransferWithTag{}, + &EventTransferWithTag{Value1: bigintExpected}, + jsonEventTransfer, + "", + "Can unpack ERC20 Transfer event into structure with abi: tag", + }, { + transferData1, + &BadEventTransferWithDuplicatedTag{}, + &BadEventTransferWithDuplicatedTag{}, + jsonEventTransfer, + "struct: abi tag in 'Value2' already mapped", + "Can not unpack ERC20 Transfer event with duplicated abi tag", + }, { + transferData1, + &BadEventTransferWithSameFieldAndTag{}, + &BadEventTransferWithSameFieldAndTag{}, + jsonEventTransfer, + "abi: multiple variables maps to the same abi field 'value'", + "Can not unpack ERC20 Transfer event with a field and a tag mapping to the same abi variable", + }, { + transferData1, + &BadEventTransferWithEmptyTag{}, + &BadEventTransferWithEmptyTag{}, + jsonEventTransfer, + "struct: abi tag in 'Value' is empty", + "Can not unpack ERC20 Transfer event with an empty tag", }, { pledgeData1, &EventPledge{}, @@ -216,6 +288,13 @@ func TestEventTupleUnpack(t *testing.T) { jsonEventPledge, "abi: cannot unmarshal tuple into map[string]interface {}", "Can not unpack Pledge event into map", + }, { + mixedCaseData1, + &EventMixedCase{}, + &EventMixedCase{Value1: bigintExpected, Value2: bigintExpected2, Value3: bigintExpected3}, + jsonEventMixedCase, + "", + "Can unpack abi variables with mixed case", }} for _, tc := range testCases { @@ -227,7 +306,7 @@ func TestEventTupleUnpack(t *testing.T) { assert.Nil(err, "Should be able to unpack event data.") assert.Equal(tc.expected, tc.dest, tc.name) } else { - assert.EqualError(err, tc.error) + assert.EqualError(err, tc.error, tc.name) } }) } diff --git a/accounts/abi/reflect.go b/accounts/abi/reflect.go index 5620a70845..0193517a42 100644 --- a/accounts/abi/reflect.go +++ b/accounts/abi/reflect.go @@ -19,6 +19,7 @@ package abi import ( "fmt" "reflect" + "strings" ) // indirect recursively dereferences the value until it either gets the value @@ -111,18 +112,101 @@ func requireUnpackKind(v reflect.Value, t reflect.Type, k reflect.Kind, return nil } -// requireUniqueStructFieldNames makes sure field names don't collide -func requireUniqueStructFieldNames(args Arguments) error { - exists := make(map[string]bool) - for _, arg := range args { - field := capitalise(arg.Name) - if field == "" { - return fmt.Errorf("abi: purely underscored output cannot unpack to struct") +// mapAbiToStringField maps abi to struct fields. +// first round: for each Exportable field that contains a `abi:""` tag +// and this field name exists in the arguments, pair them together. +// second round: for each argument field that has not been already linked, +// find what variable is expected to be mapped into, if it exists and has not been +// used, pair them. +func mapAbiToStructFields(args Arguments, value reflect.Value) (map[string]string, error) { + + typ := value.Type() + + abi2struct := make(map[string]string) + struct2abi := make(map[string]string) + + // first round ~~~ + for i := 0; i < typ.NumField(); i++ { + structFieldName := typ.Field(i).Name + + // skip private struct fields. + if structFieldName[:1] != strings.ToUpper(structFieldName[:1]) { + continue } - if exists[field] { - return fmt.Errorf("abi: multiple outputs mapping to the same struct field '%s'", field) + + // skip fields that have no abi:"" tag. + var ok bool + var tagName string + if tagName, ok = typ.Field(i).Tag.Lookup("abi"); !ok { + continue } - exists[field] = true + + // check if tag is empty. + if tagName == "" { + return nil, fmt.Errorf("struct: abi tag in '%s' is empty", structFieldName) + } + + // check which argument field matches with the abi tag. + found := false + for _, abiField := range args.NonIndexed() { + if abiField.Name == tagName { + if abi2struct[abiField.Name] != "" { + return nil, fmt.Errorf("struct: abi tag in '%s' already mapped", structFieldName) + } + // pair them + abi2struct[abiField.Name] = structFieldName + struct2abi[structFieldName] = abiField.Name + found = true + } + } + + // check if this tag has been mapped. + if !found { + return nil, fmt.Errorf("struct: abi tag '%s' defined but not found in abi", tagName) + } + } - return nil + + // second round ~~~ + for _, arg := range args { + + abiFieldName := arg.Name + structFieldName := capitalise(abiFieldName) + + if structFieldName == "" { + return nil, fmt.Errorf("abi: purely underscored output cannot unpack to struct") + } + + // this abi has already been paired, skip it... unless there exists another, yet unassigned + // struct field with the same field name. If so, raise an error: + // abi: [ { "name": "value" } ] + // struct { Value *big.Int , Value1 *big.Int `abi:"value"`} + if abi2struct[abiFieldName] != "" { + if abi2struct[abiFieldName] != structFieldName && + struct2abi[structFieldName] == "" && + value.FieldByName(structFieldName).IsValid() { + return nil, fmt.Errorf("abi: multiple variables maps to the same abi field '%s'", abiFieldName) + } + continue + } + + // return an error if this struct field has already been paired. + if struct2abi[structFieldName] != "" { + return nil, fmt.Errorf("abi: multiple outputs mapping to the same struct field '%s'", structFieldName) + } + + if value.FieldByName(structFieldName).IsValid() { + // pair them + abi2struct[abiFieldName] = structFieldName + struct2abi[structFieldName] = abiFieldName + } else { + // not paired, but annotate as used, to detect cases like + // abi : [ { "name": "value" }, { "name": "_value" } ] + // struct { Value *big.Int } + struct2abi[structFieldName] = abiFieldName + } + + } + + return abi2struct, nil } From ff8a033f187150f8fea8a55b93d4d6e6dab71013 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 14 May 2018 16:42:26 +0300 Subject: [PATCH 20/46] travis: try to upgrade android builder to trusty --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ce6500c46a..09b77d09d7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -126,7 +126,7 @@ matrix: # This builder does the Android Maven and Azure uploads - os: linux - dist: precise # Needed for the android tools + dist: trusty addons: apt: packages: From 6286c255f16a914b39ffd3389cba154a53e66a13 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 17 May 2018 15:11:27 +0200 Subject: [PATCH 21/46] p2p/enr: updates for discovery v4 compatibility (#16679) This applies spec changes from ethereum/EIPs#1049 and adds support for pluggable identity schemes. Some care has been taken to make the "v4" scheme standalone. It uses public APIs only and could be moved out of package enr at any time. A couple of minor changes were needed to make identity schemes work: - The sequence number is now updated in Set instead of when signing. - Record is now copy-safe, i.e. calling Set on a shallow copy doesn't modify the record it was copied from. --- p2p/enr/enr.go | 159 ++++++++++++++++++--------------------- p2p/enr/enr_test.go | 66 ++++++++-------- p2p/enr/entries.go | 56 +++++--------- p2p/enr/idscheme.go | 114 ++++++++++++++++++++++++++++ p2p/enr/idscheme_test.go | 36 +++++++++ 5 files changed, 277 insertions(+), 154 deletions(-) create mode 100644 p2p/enr/idscheme.go create mode 100644 p2p/enr/idscheme_test.go diff --git a/p2p/enr/enr.go b/p2p/enr/enr.go index c018895cc0..48683471d2 100644 --- a/p2p/enr/enr.go +++ b/p2p/enr/enr.go @@ -29,21 +29,16 @@ package enr import ( "bytes" - "crypto/ecdsa" "errors" "fmt" "io" "sort" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/rlp" ) const SizeLimit = 300 // maximum encoded size of a node record in bytes -const ID_SECP256k1_KECCAK = ID("secp256k1-keccak") // the default identity scheme - var ( errNoID = errors.New("unknown or unspecified identity scheme") errInvalidSig = errors.New("invalid signature") @@ -80,8 +75,8 @@ func (r *Record) Seq() uint64 { } // SetSeq updates the record sequence number. This invalidates any signature on the record. -// Calling SetSeq is usually not required because signing the redord increments the -// sequence number. +// Calling SetSeq is usually not required because setting any key in a signed record +// increments the sequence number. func (r *Record) SetSeq(s uint64) { r.signature = nil r.raw = nil @@ -104,33 +99,42 @@ func (r *Record) Load(e Entry) error { return &KeyError{Key: e.ENRKey(), Err: errNotFound} } -// Set adds or updates the given entry in the record. -// It panics if the value can't be encoded. +// Set adds or updates the given entry in the record. It panics if the value can't be +// encoded. If the record is signed, Set increments the sequence number and invalidates +// the sequence number. func (r *Record) Set(e Entry) { - r.signature = nil - r.raw = nil blob, err := rlp.EncodeToBytes(e) if err != nil { panic(fmt.Errorf("enr: can't encode %s: %v", e.ENRKey(), err)) } + r.invalidate() - i := sort.Search(len(r.pairs), func(i int) bool { return r.pairs[i].k >= e.ENRKey() }) - - if i < len(r.pairs) && r.pairs[i].k == e.ENRKey() { + pairs := make([]pair, len(r.pairs)) + copy(pairs, r.pairs) + i := sort.Search(len(pairs), func(i int) bool { return pairs[i].k >= e.ENRKey() }) + switch { + case i < len(pairs) && pairs[i].k == e.ENRKey(): // element is present at r.pairs[i] - r.pairs[i].v = blob - return - } else if i < len(r.pairs) { + pairs[i].v = blob + case i < len(r.pairs): // insert pair before i-th elem el := pair{e.ENRKey(), blob} - r.pairs = append(r.pairs, pair{}) - copy(r.pairs[i+1:], r.pairs[i:]) - r.pairs[i] = el - return + pairs = append(pairs, pair{}) + copy(pairs[i+1:], pairs[i:]) + pairs[i] = el + default: + // element should be placed at the end of r.pairs + pairs = append(pairs, pair{e.ENRKey(), blob}) } + r.pairs = pairs +} - // element should be placed at the end of r.pairs - r.pairs = append(r.pairs, pair{e.ENRKey(), blob}) +func (r *Record) invalidate() { + if r.signature == nil { + r.seq++ + } + r.signature = nil + r.raw = nil } // EncodeRLP implements rlp.Encoder. Encoding fails if @@ -196,39 +200,55 @@ func (r *Record) DecodeRLP(s *rlp.Stream) error { return err } - // Verify signature. - if err = dec.verifySignature(); err != nil { + _, scheme := dec.idScheme() + if scheme == nil { + return errNoID + } + if err := scheme.Verify(&dec, dec.signature); err != nil { return err } *r = dec return nil } -type s256raw []byte - -func (s256raw) ENRKey() string { return "secp256k1" } - // NodeAddr returns the node address. The return value will be nil if the record is -// unsigned. +// unsigned or uses an unknown identity scheme. func (r *Record) NodeAddr() []byte { - var entry s256raw - if r.Load(&entry) != nil { + _, scheme := r.idScheme() + if scheme == nil { return nil } - return crypto.Keccak256(entry) + return scheme.NodeAddr(r) } -// Sign signs the record with the given private key. It updates the record's identity -// scheme, public key and increments the sequence number. Sign returns an error if the -// encoded record is larger than the size limit. -func (r *Record) Sign(privkey *ecdsa.PrivateKey) error { - r.seq = r.seq + 1 - r.Set(ID_SECP256k1_KECCAK) - r.Set(Secp256k1(privkey.PublicKey)) - return r.signAndEncode(privkey) +// SetSig sets the record signature. It returns an error if the encoded record is larger +// than the size limit or if the signature is invalid according to the passed scheme. +func (r *Record) SetSig(idscheme string, sig []byte) error { + // Check that "id" is set and matches the given scheme. This panics because + // inconsitencies here are always implementation bugs in the signing function calling + // this method. + id, s := r.idScheme() + if s == nil { + panic(errNoID) + } + if id != idscheme { + panic(fmt.Errorf("identity scheme mismatch in Sign: record has %s, want %s", id, idscheme)) + } + + // Verify against the scheme. + if err := s.Verify(r, sig); err != nil { + return err + } + raw, err := r.encode(sig) + if err != nil { + return err + } + r.signature, r.raw = sig, raw + return nil } -func (r *Record) appendPairs(list []interface{}) []interface{} { +// AppendElements appends the sequence number and entries to the given slice. +func (r *Record) AppendElements(list []interface{}) []interface{} { list = append(list, r.seq) for _, p := range r.pairs { list = append(list, p.k, p.v) @@ -236,54 +256,23 @@ func (r *Record) appendPairs(list []interface{}) []interface{} { return list } -func (r *Record) signAndEncode(privkey *ecdsa.PrivateKey) error { - // Put record elements into a flat list. Leave room for the signature. - list := make([]interface{}, 1, len(r.pairs)*2+2) - list = r.appendPairs(list) - - // Sign the tail of the list. - h := sha3.NewKeccak256() - rlp.Encode(h, list[1:]) - sig, err := crypto.Sign(h.Sum(nil), privkey) - if err != nil { - return err +func (r *Record) encode(sig []byte) (raw []byte, err error) { + list := make([]interface{}, 1, 2*len(r.pairs)+1) + list[0] = sig + list = r.AppendElements(list) + if raw, err = rlp.EncodeToBytes(list); err != nil { + return nil, err } - sig = sig[:len(sig)-1] // remove v - - // Put signature in front. - r.signature, list[0] = sig, sig - r.raw, err = rlp.EncodeToBytes(list) - if err != nil { - return err + if len(raw) > SizeLimit { + return nil, errTooBig } - if len(r.raw) > SizeLimit { - return errTooBig - } - return nil + return raw, nil } -func (r *Record) verifySignature() error { - // Get identity scheme, public key, signature. +func (r *Record) idScheme() (string, IdentityScheme) { var id ID - var entry s256raw if err := r.Load(&id); err != nil { - return err - } else if id != ID_SECP256k1_KECCAK { - return errNoID + return "", nil } - if err := r.Load(&entry); err != nil { - return err - } else if len(entry) != 33 { - return fmt.Errorf("invalid public key") - } - - // Verify the signature. - list := make([]interface{}, 0, len(r.pairs)*2+1) - list = r.appendPairs(list) - h := sha3.NewKeccak256() - rlp.Encode(h, list) - if !crypto.VerifySignature(entry, h.Sum(nil), r.signature) { - return errInvalidSig - } - return nil + return string(id), FindIdentityScheme(string(id)) } diff --git a/p2p/enr/enr_test.go b/p2p/enr/enr_test.go index ce7767d105..d1d0887563 100644 --- a/p2p/enr/enr_test.go +++ b/p2p/enr/enr_test.go @@ -54,35 +54,35 @@ func TestGetSetID(t *testing.T) { assert.Equal(t, id, id2) } -// TestGetSetIP4 tests encoding/decoding and setting/getting of the IP4 key. +// TestGetSetIP4 tests encoding/decoding and setting/getting of the IP key. func TestGetSetIP4(t *testing.T) { - ip := IP4{192, 168, 0, 3} + ip := IP{192, 168, 0, 3} var r Record r.Set(ip) - var ip2 IP4 + var ip2 IP require.NoError(t, r.Load(&ip2)) assert.Equal(t, ip, ip2) } -// TestGetSetIP6 tests encoding/decoding and setting/getting of the IP6 key. +// TestGetSetIP6 tests encoding/decoding and setting/getting of the IP key. func TestGetSetIP6(t *testing.T) { - ip := IP6{0x20, 0x01, 0x48, 0x60, 0, 0, 0x20, 0x01, 0, 0, 0, 0, 0, 0, 0x00, 0x68} + ip := IP{0x20, 0x01, 0x48, 0x60, 0, 0, 0x20, 0x01, 0, 0, 0, 0, 0, 0, 0x00, 0x68} var r Record r.Set(ip) - var ip2 IP6 + var ip2 IP require.NoError(t, r.Load(&ip2)) assert.Equal(t, ip, ip2) } // TestGetSetDiscPort tests encoding/decoding and setting/getting of the DiscPort key. -func TestGetSetDiscPort(t *testing.T) { - port := DiscPort(30309) +func TestGetSetUDP(t *testing.T) { + port := UDP(30309) var r Record r.Set(port) - var port2 DiscPort + var port2 UDP require.NoError(t, r.Load(&port2)) assert.Equal(t, port, port2) } @@ -90,7 +90,7 @@ func TestGetSetDiscPort(t *testing.T) { // TestGetSetSecp256k1 tests encoding/decoding and setting/getting of the Secp256k1 key. func TestGetSetSecp256k1(t *testing.T) { var r Record - if err := r.Sign(privkey); err != nil { + if err := SignV4(&r, privkey); err != nil { t.Fatal(err) } @@ -101,16 +101,16 @@ func TestGetSetSecp256k1(t *testing.T) { func TestLoadErrors(t *testing.T) { var r Record - ip4 := IP4{127, 0, 0, 1} + ip4 := IP{127, 0, 0, 1} r.Set(ip4) // Check error for missing keys. - var ip6 IP6 - err := r.Load(&ip6) + var udp UDP + err := r.Load(&udp) if !IsNotFound(err) { t.Error("IsNotFound should return true for missing key") } - assert.Equal(t, &KeyError{Key: ip6.ENRKey(), Err: errNotFound}, err) + assert.Equal(t, &KeyError{Key: udp.ENRKey(), Err: errNotFound}, err) // Check error for invalid keys. var list []uint @@ -174,7 +174,7 @@ func TestDirty(t *testing.T) { t.Errorf("expected errEncodeUnsigned, got %#v", err) } - require.NoError(t, r.Sign(privkey)) + require.NoError(t, SignV4(&r, privkey)) if !r.Signed() { t.Error("Signed return false for signed record") } @@ -194,13 +194,13 @@ func TestDirty(t *testing.T) { func TestGetSetOverwrite(t *testing.T) { var r Record - ip := IP4{192, 168, 0, 3} + ip := IP{192, 168, 0, 3} r.Set(ip) - ip2 := IP4{192, 168, 0, 4} + ip2 := IP{192, 168, 0, 4} r.Set(ip2) - var ip3 IP4 + var ip3 IP require.NoError(t, r.Load(&ip3)) assert.Equal(t, ip2, ip3) } @@ -208,9 +208,9 @@ func TestGetSetOverwrite(t *testing.T) { // TestSignEncodeAndDecode tests signing, RLP encoding and RLP decoding of a record. func TestSignEncodeAndDecode(t *testing.T) { var r Record - r.Set(DiscPort(30303)) - r.Set(IP4{127, 0, 0, 1}) - require.NoError(t, r.Sign(privkey)) + r.Set(UDP(30303)) + r.Set(IP{127, 0, 0, 1}) + require.NoError(t, SignV4(&r, privkey)) blob, err := rlp.EncodeToBytes(r) require.NoError(t, err) @@ -230,12 +230,12 @@ func TestNodeAddr(t *testing.T) { t.Errorf("wrong address on empty record: got %v, want %v", addr, nil) } - require.NoError(t, r.Sign(privkey)) - expected := "caaa1485d83b18b32ed9ad666026151bf0cae8a0a88c857ae2d4c5be2daa6726" + require.NoError(t, SignV4(&r, privkey)) + expected := "a448f24c6d18e575453db13171562b71999873db5b286df957af199ec94617f7" assert.Equal(t, expected, hex.EncodeToString(r.NodeAddr())) } -var pyRecord, _ = hex.DecodeString("f896b840954dc36583c1f4b69ab59b1375f362f06ee99f3723cd77e64b6de6d211c27d7870642a79d4516997f94091325d2a7ca6215376971455fb221d34f35b277149a1018664697363763582765f82696490736563703235366b312d6b656363616b83697034847f00000189736563703235366b31a103ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd3138") +var pyRecord, _ = hex.DecodeString("f884b8407098ad865b00a582051940cb9cf36836572411a47278783077011599ed5cd16b76f2635f4e234738f30813a89eb9137e3e3df5266e3a1f11df72ecf1145ccb9c01826964827634826970847f00000189736563703235366b31a103ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd31388375647082765f") // TestPythonInterop checks that we can decode and verify a record produced by the Python // implementation. @@ -246,10 +246,10 @@ func TestPythonInterop(t *testing.T) { } var ( - wantAddr, _ = hex.DecodeString("caaa1485d83b18b32ed9ad666026151bf0cae8a0a88c857ae2d4c5be2daa6726") - wantSeq = uint64(1) - wantIP = IP4{127, 0, 0, 1} - wantDiscport = DiscPort(30303) + wantAddr, _ = hex.DecodeString("a448f24c6d18e575453db13171562b71999873db5b286df957af199ec94617f7") + wantSeq = uint64(1) + wantIP = IP{127, 0, 0, 1} + wantUDP = UDP(30303) ) if r.Seq() != wantSeq { t.Errorf("wrong seq: got %d, want %d", r.Seq(), wantSeq) @@ -257,7 +257,7 @@ func TestPythonInterop(t *testing.T) { if addr := r.NodeAddr(); !bytes.Equal(addr, wantAddr) { t.Errorf("wrong addr: got %x, want %x", addr, wantAddr) } - want := map[Entry]interface{}{new(IP4): &wantIP, new(DiscPort): &wantDiscport} + want := map[Entry]interface{}{new(IP): &wantIP, new(UDP): &wantUDP} for k, v := range want { desc := fmt.Sprintf("loading key %q", k.ENRKey()) if assert.NoError(t, r.Load(k), desc) { @@ -272,14 +272,14 @@ func TestRecordTooBig(t *testing.T) { key := randomString(10) // set a big value for random key, expect error - r.Set(WithEntry(key, randomString(300))) - if err := r.Sign(privkey); err != errTooBig { + r.Set(WithEntry(key, randomString(SizeLimit))) + if err := SignV4(&r, privkey); err != errTooBig { t.Fatalf("expected to get errTooBig, got %#v", err) } // set an acceptable value for random key, expect no error r.Set(WithEntry(key, randomString(100))) - require.NoError(t, r.Sign(privkey)) + require.NoError(t, SignV4(&r, privkey)) } // TestSignEncodeAndDecodeRandom tests encoding/decoding of records containing random key/value pairs. @@ -295,7 +295,7 @@ func TestSignEncodeAndDecodeRandom(t *testing.T) { r.Set(WithEntry(key, &value)) } - require.NoError(t, r.Sign(privkey)) + require.NoError(t, SignV4(&r, privkey)) _, err := rlp.EncodeToBytes(r) require.NoError(t, err) diff --git a/p2p/enr/entries.go b/p2p/enr/entries.go index 7591e6effb..71c7653a28 100644 --- a/p2p/enr/entries.go +++ b/p2p/enr/entries.go @@ -57,59 +57,43 @@ func WithEntry(k string, v interface{}) Entry { return &generic{key: k, value: v} } -// DiscPort is the "discv5" key, which holds the UDP port for discovery v5. -type DiscPort uint16 +// TCP is the "tcp" key, which holds the TCP port of the node. +type TCP uint16 -func (v DiscPort) ENRKey() string { return "discv5" } +func (v TCP) ENRKey() string { return "tcp" } + +// UDP is the "udp" key, which holds the UDP port of the node. +type UDP uint16 + +func (v UDP) ENRKey() string { return "udp" } // ID is the "id" key, which holds the name of the identity scheme. type ID string +const IDv4 = ID("v4") // the default identity scheme + func (v ID) ENRKey() string { return "id" } -// IP4 is the "ip4" key, which holds a 4-byte IPv4 address. -type IP4 net.IP +// IP is the "ip" key, which holds the IP address of the node. +type IP net.IP -func (v IP4) ENRKey() string { return "ip4" } +func (v IP) ENRKey() string { return "ip" } // EncodeRLP implements rlp.Encoder. -func (v IP4) EncodeRLP(w io.Writer) error { - ip4 := net.IP(v).To4() - if ip4 == nil { - return fmt.Errorf("invalid IPv4 address: %v", v) +func (v IP) EncodeRLP(w io.Writer) error { + if ip4 := net.IP(v).To4(); ip4 != nil { + return rlp.Encode(w, ip4) } - return rlp.Encode(w, ip4) + return rlp.Encode(w, net.IP(v)) } // DecodeRLP implements rlp.Decoder. -func (v *IP4) DecodeRLP(s *rlp.Stream) error { +func (v *IP) DecodeRLP(s *rlp.Stream) error { if err := s.Decode((*net.IP)(v)); err != nil { return err } - if len(*v) != 4 { - return fmt.Errorf("invalid IPv4 address, want 4 bytes: %v", *v) - } - return nil -} - -// IP6 is the "ip6" key, which holds a 16-byte IPv6 address. -type IP6 net.IP - -func (v IP6) ENRKey() string { return "ip6" } - -// EncodeRLP implements rlp.Encoder. -func (v IP6) EncodeRLP(w io.Writer) error { - ip6 := net.IP(v) - return rlp.Encode(w, ip6) -} - -// DecodeRLP implements rlp.Decoder. -func (v *IP6) DecodeRLP(s *rlp.Stream) error { - if err := s.Decode((*net.IP)(v)); err != nil { - return err - } - if len(*v) != 16 { - return fmt.Errorf("invalid IPv6 address, want 16 bytes: %v", *v) + if len(*v) != 4 && len(*v) != 16 { + return fmt.Errorf("invalid IP address, want 4 or 16 bytes: %v", *v) } return nil } diff --git a/p2p/enr/idscheme.go b/p2p/enr/idscheme.go new file mode 100644 index 0000000000..efaf68041d --- /dev/null +++ b/p2p/enr/idscheme.go @@ -0,0 +1,114 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package enr + +import ( + "crypto/ecdsa" + "fmt" + "sync" + + "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/crypto/sha3" + "github.com/ethereum/go-ethereum/rlp" +) + +// Registry of known identity schemes. +var schemes sync.Map + +// An IdentityScheme is capable of verifying record signatures and +// deriving node addresses. +type IdentityScheme interface { + Verify(r *Record, sig []byte) error + NodeAddr(r *Record) []byte +} + +// RegisterIdentityScheme adds an identity scheme to the global registry. +func RegisterIdentityScheme(name string, scheme IdentityScheme) { + if _, loaded := schemes.LoadOrStore(name, scheme); loaded { + panic("identity scheme " + name + " already registered") + } +} + +// FindIdentityScheme resolves name to an identity scheme in the global registry. +func FindIdentityScheme(name string) IdentityScheme { + s, ok := schemes.Load(name) + if !ok { + return nil + } + return s.(IdentityScheme) +} + +// v4ID is the "v4" identity scheme. +type v4ID struct{} + +func init() { + RegisterIdentityScheme("v4", v4ID{}) +} + +// SignV4 signs a record using the v4 scheme. +func SignV4(r *Record, privkey *ecdsa.PrivateKey) error { + // Copy r to avoid modifying it if signing fails. + cpy := *r + cpy.Set(ID("v4")) + cpy.Set(Secp256k1(privkey.PublicKey)) + + h := sha3.NewKeccak256() + rlp.Encode(h, cpy.AppendElements(nil)) + sig, err := crypto.Sign(h.Sum(nil), privkey) + if err != nil { + return err + } + sig = sig[:len(sig)-1] // remove v + if err = cpy.SetSig("v4", sig); err == nil { + *r = cpy + } + return err +} + +// s256raw is an unparsed secp256k1 public key entry. +type s256raw []byte + +func (s256raw) ENRKey() string { return "secp256k1" } + +func (v4ID) Verify(r *Record, sig []byte) error { + var entry s256raw + if err := r.Load(&entry); err != nil { + return err + } else if len(entry) != 33 { + return fmt.Errorf("invalid public key") + } + + h := sha3.NewKeccak256() + rlp.Encode(h, r.AppendElements(nil)) + if !crypto.VerifySignature(entry, h.Sum(nil), sig) { + return errInvalidSig + } + return nil +} + +func (v4ID) NodeAddr(r *Record) []byte { + var pubkey Secp256k1 + err := r.Load(&pubkey) + if err != nil { + return nil + } + buf := make([]byte, 64) + math.ReadBits(pubkey.X, buf[:32]) + math.ReadBits(pubkey.Y, buf[32:]) + return crypto.Keccak256(buf) +} diff --git a/p2p/enr/idscheme_test.go b/p2p/enr/idscheme_test.go new file mode 100644 index 0000000000..d790e12f14 --- /dev/null +++ b/p2p/enr/idscheme_test.go @@ -0,0 +1,36 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package enr + +import ( + "crypto/ecdsa" + "math/big" + "testing" +) + +// Checks that failure to sign leaves the record unmodified. +func TestSignError(t *testing.T) { + invalidKey := &ecdsa.PrivateKey{D: new(big.Int), PublicKey: *pubkey} + + var r Record + if err := SignV4(&r, invalidKey); err == nil { + t.Fatal("expected error from SignV4") + } + if len(r.pairs) > 0 { + t.Fatal("expected empty record, have", r.pairs) + } +} From a2e43d28d01ef9642c7f6992b78b86bd0696c847 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Thu, 10 May 2018 15:04:45 +0800 Subject: [PATCH 22/46] all: collate new transaction events together --- accounts/abi/bind/backends/simulated.go | 2 +- core/events.go | 4 +-- core/tx_journal.go | 32 ++++++++++++++---- core/tx_pool.go | 32 ++++++++++++------ core/tx_pool_test.go | 31 ++++++++++------- eth/api_backend.go | 2 +- eth/filters/api.go | 14 +++++--- eth/filters/filter.go | 2 +- eth/filters/filter_system.go | 38 +++++++++++---------- eth/filters/filter_system_test.go | 7 ++-- eth/handler.go | 41 ++++++++++++---------- eth/helper_test.go | 2 +- eth/protocol.go | 4 +-- eth/protocol_test.go | 2 +- ethstats/ethstats.go | 8 ++--- internal/ethapi/backend.go | 2 +- les/api_backend.go | 2 +- light/txpool.go | 6 ++-- miner/worker.go | 45 +++++++++++++++---------- 19 files changed, 165 insertions(+), 111 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 7b605e1c1e..6262f3688c 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -454,7 +454,7 @@ func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*ty return logs, nil } -func (fb *filterBackend) SubscribeTxPreEvent(ch chan<- core.TxPreEvent) event.Subscription { +func (fb *filterBackend) SubscribeTxPreEvent(ch chan<- core.TxsPreEvent) event.Subscription { return event.NewSubscription(func(quit <-chan struct{}) error { <-quit return nil diff --git a/core/events.go b/core/events.go index 6f404f612b..fee34b39d5 100644 --- a/core/events.go +++ b/core/events.go @@ -21,8 +21,8 @@ import ( "github.com/ethereum/go-ethereum/core/types" ) -// TxPreEvent is posted when a transaction enters the transaction pool. -type TxPreEvent struct{ Tx *types.Transaction } +// TxsPreEvent is posted when a batch of transactions enter the transaction pool. +type TxsPreEvent struct{ Txs types.Transactions } // PendingLogsEvent is posted pre mining and notifies of pending logs. type PendingLogsEvent struct { diff --git a/core/tx_journal.go b/core/tx_journal.go index e872d7b530..b344690b69 100644 --- a/core/tx_journal.go +++ b/core/tx_journal.go @@ -56,7 +56,7 @@ func newTxJournal(path string) *txJournal { // load parses a transaction journal dump from disk, loading its contents into // the specified pool. -func (journal *txJournal) load(add func(*types.Transaction) error) error { +func (journal *txJournal) load(add func([]*types.Transaction) []error) error { // Skip the parsing if the journal file doens't exist at all if _, err := os.Stat(journal.path); os.IsNotExist(err) { return nil @@ -76,7 +76,22 @@ func (journal *txJournal) load(add func(*types.Transaction) error) error { stream := rlp.NewStream(input, 0) total, dropped := 0, 0 - var failure error + // flush imports a batch of transactions and bump the appropriate progress counters + flush := func(txs types.Transactions) { + errs := add(txs) + for _, err := range errs { + if err != nil { + log.Debug("Failed to add journaled transaction", "err", err) + dropped++ + } + } + } + + var ( + failure error + txs types.Transactions + ) + for { // Parse the next transaction and terminate on error tx := new(types.Transaction) @@ -86,14 +101,17 @@ func (journal *txJournal) load(add func(*types.Transaction) error) error { } break } - // Import the transaction and bump the appropriate progress counters + txs = append(txs, tx) total++ - if err = add(tx); err != nil { - log.Debug("Failed to add journaled transaction", "err", err) - dropped++ - continue + if txs.Len() > 1024 { + flush(txs) + txs = types.Transactions{} } } + if txs.Len() > 0 { + flush(txs) + txs = types.Transactions{} + } log.Info("Loaded local transaction journal", "transactions", total, "dropped", dropped) return failure diff --git a/core/tx_pool.go b/core/tx_pool.go index 388b40058f..f23828cb4e 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -238,7 +238,7 @@ func NewTxPool(config TxPoolConfig, chainconfig *params.ChainConfig, chain block if !config.NoLocals && config.Journal != "" { pool.journal = newTxJournal(config.Journal) - if err := pool.journal.load(pool.AddLocal); err != nil { + if err := pool.journal.load(pool.AddLocals); err != nil { log.Warn("Failed to load transaction journal", "err", err) } if err := pool.journal.rotate(pool.local()); err != nil { @@ -444,9 +444,9 @@ func (pool *TxPool) Stop() { log.Info("Transaction pool stopped") } -// SubscribeTxPreEvent registers a subscription of TxPreEvent and +// SubscribeTxPreEvent registers a subscription of TxsPreEvent and // starts sending event to the given channel. -func (pool *TxPool) SubscribeTxPreEvent(ch chan<- TxPreEvent) event.Subscription { +func (pool *TxPool) SubscribeTxPreEvent(ch chan<- TxsPreEvent) event.Subscription { return pool.scope.Track(pool.txFeed.Subscribe(ch)) } @@ -653,7 +653,7 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (bool, error) { log.Trace("Pooled new executable transaction", "hash", hash, "from", from, "to", tx.To()) // We've directly injected a replacement transaction, notify subsystems - go pool.txFeed.Send(TxPreEvent{tx}) + go pool.txFeed.Send(TxsPreEvent{types.Transactions{tx}}) return old != nil, nil } @@ -715,7 +715,7 @@ func (pool *TxPool) journalTx(from common.Address, tx *types.Transaction) { // promoteTx adds a transaction to the pending (processable) list of transactions. // // Note, this method assumes the pool lock is held! -func (pool *TxPool) promoteTx(addr common.Address, hash common.Hash, tx *types.Transaction) { +func (pool *TxPool) promoteTx(addr common.Address, hash common.Hash, tx *types.Transaction) bool { // Try to insert the transaction into the pending queue if pool.pending[addr] == nil { pool.pending[addr] = newTxList(true) @@ -729,7 +729,7 @@ func (pool *TxPool) promoteTx(addr common.Address, hash common.Hash, tx *types.T pool.priced.Removed() pendingDiscardCounter.Inc(1) - return + return false } // Otherwise discard any previous transaction and mark this if old != nil { @@ -746,8 +746,7 @@ func (pool *TxPool) promoteTx(addr common.Address, hash common.Hash, tx *types.T // Set the potentially new pending nonce and notify any subsystems of the new tx pool.beats[addr] = time.Now() pool.pendingState.SetNonce(addr, tx.Nonce()+1) - - go pool.txFeed.Send(TxPreEvent{tx}) + return true } // AddLocal enqueues a single transaction into the pool if it is valid, marking @@ -907,6 +906,7 @@ func (pool *TxPool) removeTx(hash common.Hash, outofbound bool) { // future queue to the set of pending transactions. During this process, all // invalidated transactions (low nonce, low balance) are deleted. func (pool *TxPool) promoteExecutables(accounts []common.Address) { + var promotedTxs types.Transactions // Gather all the accounts potentially needing updates if accounts == nil { accounts = make([]common.Address, 0, len(pool.queue)) @@ -937,11 +937,16 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) { queuedNofundsCounter.Inc(1) } // Gather all executable transactions and promote them - for _, tx := range list.Ready(pool.pendingState.GetNonce(addr)) { + txs := list.Ready(pool.pendingState.GetNonce(addr)) + for _, tx := range txs { hash := tx.Hash() - log.Trace("Promoting queued transaction", "hash", hash) - pool.promoteTx(addr, hash, tx) + inserted := pool.promoteTx(addr, hash, tx) + if inserted { + log.Trace("Promoting queued transaction", "hash", hash) + promotedTxs = append(promotedTxs, tx) + } } + // Drop all transactions over the allowed limit if !pool.locals.contains(addr) { for _, tx := range list.Cap(int(pool.config.AccountQueue)) { @@ -957,6 +962,11 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) { delete(pool.queue, addr) } } + // Notify subsystem for new promoted transactions. + if promotedTxs.Len() > 0 { + pool.txFeed.Send(TxsPreEvent{promotedTxs}) + } + // If the pending limit is overflown, start equalizing allowances pending := uint64(0) for _, list := range pool.pending { diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index e7f52075e5..c19581e3f3 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -118,21 +118,26 @@ func validateTxPoolInternals(pool *TxPool) error { // validateEvents checks that the correct number of transaction addition events // were fired on the pool's event feed. -func validateEvents(events chan TxPreEvent, count int) error { - for i := 0; i < count; i++ { +func validateEvents(events chan TxsPreEvent, count int) error { + received := 0 + for { + if received == count { + break + } select { - case <-events: + case ev := <-events: + received += ev.Txs.Len() case <-time.After(time.Second): - return fmt.Errorf("event #%d not fired", i) + return fmt.Errorf("event #%d not fired", received) } } select { - case tx := <-events: - return fmt.Errorf("more than %d events fired: %v", count, tx.Tx) + case ev := <-events: + return fmt.Errorf("more than %d events fired: %v", count, ev.Txs) case <-time.After(50 * time.Millisecond): // This branch should be "default", but it's a data race between goroutines, - // reading the event channel and pushng into it, so better wait a bit ensuring + // reading the event channel and pushing into it, so better wait a bit ensuring // really nothing gets injected. } return nil @@ -669,7 +674,7 @@ func TestTransactionGapFilling(t *testing.T) { pool.currentState.AddBalance(account, big.NewInt(1000000)) // Keep track of transaction events to ensure all executables get announced - events := make(chan TxPreEvent, testTxPoolConfig.AccountQueue+5) + events := make(chan TxsPreEvent, testTxPoolConfig.AccountQueue+5) sub := pool.txFeed.Subscribe(events) defer sub.Unsubscribe() @@ -920,7 +925,7 @@ func TestTransactionPendingLimiting(t *testing.T) { pool.currentState.AddBalance(account, big.NewInt(1000000)) // Keep track of transaction events to ensure all executables get announced - events := make(chan TxPreEvent, testTxPoolConfig.AccountQueue+5) + events := make(chan TxsPreEvent, testTxPoolConfig.AccountQueue+5) sub := pool.txFeed.Subscribe(events) defer sub.Unsubscribe() @@ -1140,7 +1145,7 @@ func TestTransactionPoolRepricing(t *testing.T) { defer pool.Stop() // Keep track of transaction events to ensure all executables get announced - events := make(chan TxPreEvent, 32) + events := make(chan TxsPreEvent, 32) sub := pool.txFeed.Subscribe(events) defer sub.Unsubscribe() @@ -1327,7 +1332,7 @@ func TestTransactionPoolUnderpricing(t *testing.T) { defer pool.Stop() // Keep track of transaction events to ensure all executables get announced - events := make(chan TxPreEvent, 32) + events := make(chan TxsPreEvent, 32) sub := pool.txFeed.Subscribe(events) defer sub.Unsubscribe() @@ -1433,7 +1438,7 @@ func TestTransactionPoolStableUnderpricing(t *testing.T) { defer pool.Stop() // Keep track of transaction events to ensure all executables get announced - events := make(chan TxPreEvent, 32) + events := make(chan TxsPreEvent, 32) sub := pool.txFeed.Subscribe(events) defer sub.Unsubscribe() @@ -1495,7 +1500,7 @@ func TestTransactionReplacement(t *testing.T) { defer pool.Stop() // Keep track of transaction events to ensure all executables get announced - events := make(chan TxPreEvent, 32) + events := make(chan TxsPreEvent, 32) sub := pool.txFeed.Subscribe(events) defer sub.Unsubscribe() diff --git a/eth/api_backend.go b/eth/api_backend.go index 4ace9b5945..ef70b12b75 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -188,7 +188,7 @@ func (b *EthAPIBackend) TxPoolContent() (map[common.Address]types.Transactions, return b.eth.TxPool().Content() } -func (b *EthAPIBackend) SubscribeTxPreEvent(ch chan<- core.TxPreEvent) event.Subscription { +func (b *EthAPIBackend) SubscribeTxPreEvent(ch chan<- core.TxsPreEvent) event.Subscription { return b.eth.TxPool().SubscribeTxPreEvent(ch) } diff --git a/eth/filters/api.go b/eth/filters/api.go index 1297b74788..d2c9258f91 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -104,7 +104,7 @@ func (api *PublicFilterAPI) timeoutLoop() { // https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_newpendingtransactionfilter func (api *PublicFilterAPI) NewPendingTransactionFilter() rpc.ID { var ( - pendingTxs = make(chan common.Hash) + pendingTxs = make(chan []common.Hash) pendingTxSub = api.events.SubscribePendingTxEvents(pendingTxs) ) @@ -118,7 +118,7 @@ func (api *PublicFilterAPI) NewPendingTransactionFilter() rpc.ID { case ph := <-pendingTxs: api.filtersMu.Lock() if f, found := api.filters[pendingTxSub.ID]; found { - f.hashes = append(f.hashes, ph) + f.hashes = append(f.hashes, ph...) } api.filtersMu.Unlock() case <-pendingTxSub.Err(): @@ -144,13 +144,17 @@ func (api *PublicFilterAPI) NewPendingTransactions(ctx context.Context) (*rpc.Su rpcSub := notifier.CreateSubscription() go func() { - txHashes := make(chan common.Hash) + txHashes := make(chan []common.Hash, 128) pendingTxSub := api.events.SubscribePendingTxEvents(txHashes) for { select { - case h := <-txHashes: - notifier.Notify(rpcSub.ID, h) + case hashes := <-txHashes: + // To keep the original behaviour, send a single tx hash in one notification. + // TODO(rjl493456442) Send a batch of tx hashes in one notification + for _, h := range hashes { + notifier.Notify(rpcSub.ID, h) + } case <-rpcSub.Err(): pendingTxSub.Unsubscribe() return diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 5dfe60e772..45d91bea00 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -36,7 +36,7 @@ type Backend interface { GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) - SubscribeTxPreEvent(chan<- core.TxPreEvent) event.Subscription + SubscribeTxPreEvent(chan<- core.TxsPreEvent) event.Subscription SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index bb11734a76..5f1c12c6a8 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -59,7 +59,7 @@ const ( const ( - // txChanSize is the size of channel listening to TxPreEvent. + // txChanSize is the size of channel listening to TxsPreEvent. // The number is referenced from the size of tx pool. txChanSize = 4096 // rmLogsChanSize is the size of channel listening to RemovedLogsEvent. @@ -80,7 +80,7 @@ type subscription struct { created time.Time logsCrit ethereum.FilterQuery logs chan []*types.Log - hashes chan common.Hash + hashes chan []common.Hash headers chan *types.Header installed chan struct{} // closed when the filter is installed err chan error // closed when the filter is uninstalled @@ -95,7 +95,7 @@ type EventSystem struct { lastHead *types.Header // Subscriptions - txSub event.Subscription // Subscription for new transaction event + txsSub event.Subscription // Subscription for new transaction event logsSub event.Subscription // Subscription for new log event rmLogsSub event.Subscription // Subscription for removed log event chainSub event.Subscription // Subscription for new chain event @@ -104,7 +104,7 @@ type EventSystem struct { // Channels install chan *subscription // install filter for event notification uninstall chan *subscription // remove filter for event notification - txCh chan core.TxPreEvent // Channel to receive new transaction event + txsCh chan core.TxsPreEvent // Channel to receive new transactions event logsCh chan []*types.Log // Channel to receive new log event rmLogsCh chan core.RemovedLogsEvent // Channel to receive removed log event chainCh chan core.ChainEvent // Channel to receive new chain event @@ -123,14 +123,14 @@ func NewEventSystem(mux *event.TypeMux, backend Backend, lightMode bool) *EventS lightMode: lightMode, install: make(chan *subscription), uninstall: make(chan *subscription), - txCh: make(chan core.TxPreEvent, txChanSize), + txsCh: make(chan core.TxsPreEvent, txChanSize), logsCh: make(chan []*types.Log, logsChanSize), rmLogsCh: make(chan core.RemovedLogsEvent, rmLogsChanSize), chainCh: make(chan core.ChainEvent, chainEvChanSize), } // Subscribe events - m.txSub = m.backend.SubscribeTxPreEvent(m.txCh) + m.txsSub = m.backend.SubscribeTxPreEvent(m.txsCh) m.logsSub = m.backend.SubscribeLogsEvent(m.logsCh) m.rmLogsSub = m.backend.SubscribeRemovedLogsEvent(m.rmLogsCh) m.chainSub = m.backend.SubscribeChainEvent(m.chainCh) @@ -138,7 +138,7 @@ func NewEventSystem(mux *event.TypeMux, backend Backend, lightMode bool) *EventS m.pendingLogSub = m.mux.Subscribe(core.PendingLogsEvent{}) // Make sure none of the subscriptions are empty - if m.txSub == nil || m.logsSub == nil || m.rmLogsSub == nil || m.chainSub == nil || + if m.txsSub == nil || m.logsSub == nil || m.rmLogsSub == nil || m.chainSub == nil || m.pendingLogSub.Closed() { log.Crit("Subscribe for event system failed") } @@ -240,7 +240,7 @@ func (es *EventSystem) subscribeMinedPendingLogs(crit ethereum.FilterQuery, logs logsCrit: crit, created: time.Now(), logs: logs, - hashes: make(chan common.Hash), + hashes: make(chan []common.Hash), headers: make(chan *types.Header), installed: make(chan struct{}), err: make(chan error), @@ -257,7 +257,7 @@ func (es *EventSystem) subscribeLogs(crit ethereum.FilterQuery, logs chan []*typ logsCrit: crit, created: time.Now(), logs: logs, - hashes: make(chan common.Hash), + hashes: make(chan []common.Hash), headers: make(chan *types.Header), installed: make(chan struct{}), err: make(chan error), @@ -274,7 +274,7 @@ func (es *EventSystem) subscribePendingLogs(crit ethereum.FilterQuery, logs chan logsCrit: crit, created: time.Now(), logs: logs, - hashes: make(chan common.Hash), + hashes: make(chan []common.Hash), headers: make(chan *types.Header), installed: make(chan struct{}), err: make(chan error), @@ -290,7 +290,7 @@ func (es *EventSystem) SubscribeNewHeads(headers chan *types.Header) *Subscripti typ: BlocksSubscription, created: time.Now(), logs: make(chan []*types.Log), - hashes: make(chan common.Hash), + hashes: make(chan []common.Hash), headers: headers, installed: make(chan struct{}), err: make(chan error), @@ -300,7 +300,7 @@ func (es *EventSystem) SubscribeNewHeads(headers chan *types.Header) *Subscripti // SubscribePendingTxEvents creates a subscription that writes transaction hashes for // transactions that enter the transaction pool. -func (es *EventSystem) SubscribePendingTxEvents(hashes chan common.Hash) *Subscription { +func (es *EventSystem) SubscribePendingTxEvents(hashes chan []common.Hash) *Subscription { sub := &subscription{ id: rpc.NewID(), typ: PendingTransactionsSubscription, @@ -348,9 +348,13 @@ func (es *EventSystem) broadcast(filters filterIndex, ev interface{}) { } } } - case core.TxPreEvent: + case core.TxsPreEvent: + hashes := make([]common.Hash, 0, e.Txs.Len()) + for _, tx := range e.Txs { + hashes = append(hashes, tx.Hash()) + } for _, f := range filters[PendingTransactionsSubscription] { - f.hashes <- e.Tx.Hash() + f.hashes <- hashes } case core.ChainEvent: for _, f := range filters[BlocksSubscription] { @@ -446,7 +450,7 @@ func (es *EventSystem) eventLoop() { // Ensure all subscriptions get cleaned up defer func() { es.pendingLogSub.Unsubscribe() - es.txSub.Unsubscribe() + es.txsSub.Unsubscribe() es.logsSub.Unsubscribe() es.rmLogsSub.Unsubscribe() es.chainSub.Unsubscribe() @@ -460,7 +464,7 @@ func (es *EventSystem) eventLoop() { for { select { // Handle subscribed events - case ev := <-es.txCh: + case ev := <-es.txsCh: es.broadcast(index, ev) case ev := <-es.logsCh: es.broadcast(index, ev) @@ -495,7 +499,7 @@ func (es *EventSystem) eventLoop() { close(f.err) // System stopped - case <-es.txSub.Err(): + case <-es.txsSub.Err(): return case <-es.logsSub.Err(): return diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index b4df24b471..c43d282ae6 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -96,7 +96,7 @@ func (b *testBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types return logs, nil } -func (b *testBackend) SubscribeTxPreEvent(ch chan<- core.TxPreEvent) event.Subscription { +func (b *testBackend) SubscribeTxPreEvent(ch chan<- core.TxsPreEvent) event.Subscription { return b.txFeed.Subscribe(ch) } @@ -232,10 +232,7 @@ func TestPendingTxFilter(t *testing.T) { fid0 := api.NewPendingTransactionFilter() time.Sleep(1 * time.Second) - for _, tx := range transactions { - ev := core.TxPreEvent{Tx: tx} - txFeed.Send(ev) - } + txFeed.Send(core.TxsPreEvent{transactions}) timeout := time.Now().Add(1 * time.Second) for { diff --git a/eth/handler.go b/eth/handler.go index c8f7e13f16..7ef8e957a4 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -46,7 +46,7 @@ const ( softResponseLimit = 2 * 1024 * 1024 // Target maximum size of returned blocks, headers or node data. estHeaderRlpSize = 500 // Approximate size of an RLP encoded block header - // txChanSize is the size of channel listening to TxPreEvent. + // txChanSize is the size of channel listening to TxsPreEvent. // The number is referenced from the size of tx pool. txChanSize = 4096 ) @@ -81,8 +81,8 @@ type ProtocolManager struct { SubProtocols []p2p.Protocol eventMux *event.TypeMux - txCh chan core.TxPreEvent - txSub event.Subscription + txsCh chan core.TxsPreEvent + txsSub event.Subscription minedBlockSub *event.TypeMuxSubscription // channels for fetcher, syncer, txsyncLoop @@ -204,8 +204,8 @@ func (pm *ProtocolManager) Start(maxPeers int) { pm.maxPeers = maxPeers // broadcast transactions - pm.txCh = make(chan core.TxPreEvent, txChanSize) - pm.txSub = pm.txpool.SubscribeTxPreEvent(pm.txCh) + pm.txsCh = make(chan core.TxsPreEvent, txChanSize) + pm.txsSub = pm.txpool.SubscribeTxPreEvent(pm.txsCh) go pm.txBroadcastLoop() // broadcast mined blocks @@ -220,7 +220,7 @@ func (pm *ProtocolManager) Start(maxPeers int) { func (pm *ProtocolManager) Stop() { log.Info("Stopping Ethereum protocol") - pm.txSub.Unsubscribe() // quits txBroadcastLoop + pm.txsSub.Unsubscribe() // quits txBroadcastLoop pm.minedBlockSub.Unsubscribe() // quits blockBroadcastLoop // Quit the sync loop. @@ -712,16 +712,23 @@ func (pm *ProtocolManager) BroadcastBlock(block *types.Block, propagate bool) { } } -// BroadcastTx will propagate a transaction to all peers which are not known to +// BroadcastTxs will propagate a batch of transactions to all peers which are not known to // already have the given transaction. -func (pm *ProtocolManager) BroadcastTx(hash common.Hash, tx *types.Transaction) { - // Broadcast transaction to a batch of peers not knowing about it - peers := pm.peers.PeersWithoutTx(hash) - //FIXME include this again: peers = peers[:int(math.Sqrt(float64(len(peers))))] - for _, peer := range peers { - peer.SendTransactions(types.Transactions{tx}) +func (pm *ProtocolManager) BroadcastTxs(txs types.Transactions) { + var txset = make(map[*peer]types.Transactions) + + // Broadcast transactions to a batch of peers not knowing about it + for _, tx := range txs { + peers := pm.peers.PeersWithoutTx(tx.Hash()) + for _, peer := range peers { + txset[peer] = append(txset[peer], tx) + } + log.Trace("Broadcast transaction", "hash", tx.Hash(), "recipients", len(peers)) + } + // FIXME include this again: peers = peers[:int(math.Sqrt(float64(len(peers))))] + for peer, txs := range txset { + peer.SendTransactions(txs) } - log.Trace("Broadcast transaction", "hash", hash, "recipients", len(peers)) } // Mined broadcast loop @@ -739,11 +746,11 @@ func (pm *ProtocolManager) minedBroadcastLoop() { func (pm *ProtocolManager) txBroadcastLoop() { for { select { - case event := <-pm.txCh: - pm.BroadcastTx(event.Tx.Hash(), event.Tx) + case event := <-pm.txsCh: + pm.BroadcastTxs(event.Txs) // Err() channel will be closed when unsubscribing. - case <-pm.txSub.Err(): + case <-pm.txsSub.Err(): return } } diff --git a/eth/helper_test.go b/eth/helper_test.go index 8a0260fc95..40a172ef83 100644 --- a/eth/helper_test.go +++ b/eth/helper_test.go @@ -124,7 +124,7 @@ func (p *testTxPool) Pending() (map[common.Address]types.Transactions, error) { return batches, nil } -func (p *testTxPool) SubscribeTxPreEvent(ch chan<- core.TxPreEvent) event.Subscription { +func (p *testTxPool) SubscribeTxPreEvent(ch chan<- core.TxsPreEvent) event.Subscription { return p.txFeed.Subscribe(ch) } diff --git a/eth/protocol.go b/eth/protocol.go index 328d5b9939..bee3d83651 100644 --- a/eth/protocol.go +++ b/eth/protocol.go @@ -104,8 +104,8 @@ type txPool interface { Pending() (map[common.Address]types.Transactions, error) // SubscribeTxPreEvent should return an event subscription of - // TxPreEvent and send events to the given channel. - SubscribeTxPreEvent(chan<- core.TxPreEvent) event.Subscription + // TxsPreEvent and send events to the given channel. + SubscribeTxPreEvent(chan<- core.TxsPreEvent) event.Subscription } // statusData is the network packet for the status message. diff --git a/eth/protocol_test.go b/eth/protocol_test.go index b2f93d8dd1..1c32ea7d83 100644 --- a/eth/protocol_test.go +++ b/eth/protocol_test.go @@ -116,7 +116,7 @@ func testRecvTransactions(t *testing.T, protocol int) { t.Errorf("added wrong tx hash: got %v, want %v", added[0].Hash(), tx.Hash()) } case <-time.After(2 * time.Second): - t.Errorf("no TxPreEvent received within 2 seconds") + t.Errorf("no TxsPreEvent received within 2 seconds") } } diff --git a/ethstats/ethstats.go b/ethstats/ethstats.go index a15d846150..17f9f3c7a5 100644 --- a/ethstats/ethstats.go +++ b/ethstats/ethstats.go @@ -49,7 +49,7 @@ const ( // history request. historyUpdateRange = 50 - // txChanSize is the size of channel listening to TxPreEvent. + // txChanSize is the size of channel listening to TxsPreEvent. // The number is referenced from the size of tx pool. txChanSize = 4096 // chainHeadChanSize is the size of channel listening to ChainHeadEvent. @@ -58,8 +58,8 @@ const ( type txPool interface { // SubscribeTxPreEvent should return an event subscription of - // TxPreEvent and send events to the given channel. - SubscribeTxPreEvent(chan<- core.TxPreEvent) event.Subscription + // TxsPreEvent and send events to the given channel. + SubscribeTxPreEvent(chan<- core.TxsPreEvent) event.Subscription } type blockChain interface { @@ -150,7 +150,7 @@ func (s *Service) loop() { headSub := blockchain.SubscribeChainHeadEvent(chainHeadCh) defer headSub.Unsubscribe() - txEventCh := make(chan core.TxPreEvent, txChanSize) + txEventCh := make(chan core.TxsPreEvent, txChanSize) txSub := txpool.SubscribeTxPreEvent(txEventCh) defer txSub.Unsubscribe() diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index af95d7906f..e351152b9c 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -65,7 +65,7 @@ type Backend interface { GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) Stats() (pending int, queued int) TxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions) - SubscribeTxPreEvent(chan<- core.TxPreEvent) event.Subscription + SubscribeTxPreEvent(chan<- core.TxsPreEvent) event.Subscription ChainConfig() *params.ChainConfig CurrentBlock() *types.Block diff --git a/les/api_backend.go b/les/api_backend.go index 1d3c995134..957c8ae649 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -136,7 +136,7 @@ func (b *LesApiBackend) TxPoolContent() (map[common.Address]types.Transactions, return b.eth.txPool.Content() } -func (b *LesApiBackend) SubscribeTxPreEvent(ch chan<- core.TxPreEvent) event.Subscription { +func (b *LesApiBackend) SubscribeTxPreEvent(ch chan<- core.TxsPreEvent) event.Subscription { return b.eth.txPool.SubscribeTxPreEvent(ch) } diff --git a/light/txpool.go b/light/txpool.go index 94c8139cbd..bedfd48a25 100644 --- a/light/txpool.go +++ b/light/txpool.go @@ -321,9 +321,9 @@ func (pool *TxPool) Stop() { log.Info("Transaction pool stopped") } -// SubscribeTxPreEvent registers a subscription of core.TxPreEvent and +// SubscribeTxPreEvent registers a subscription of core.TxsPreEvent and // starts sending event to the given channel. -func (pool *TxPool) SubscribeTxPreEvent(ch chan<- core.TxPreEvent) event.Subscription { +func (pool *TxPool) SubscribeTxPreEvent(ch chan<- core.TxsPreEvent) event.Subscription { return pool.scope.Track(pool.txFeed.Subscribe(ch)) } @@ -412,7 +412,7 @@ func (self *TxPool) add(ctx context.Context, tx *types.Transaction) error { // Notify the subscribers. This event is posted in a goroutine // because it's possible that somewhere during the post "Remove transaction" // gets called which will then wait for the global tx pool lock and deadlock. - go self.txFeed.Send(core.TxPreEvent{Tx: tx}) + go self.txFeed.Send(core.TxsPreEvent{types.Transactions{tx}}) } // Print a log message if low enough level is set diff --git a/miner/worker.go b/miner/worker.go index 48b0b27652..3d086d6aee 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -42,7 +42,7 @@ const ( resultQueueSize = 10 miningLogAtDepth = 5 - // txChanSize is the size of channel listening to TxPreEvent. + // txChanSize is the size of channel listening to TxsPreEvent. // The number is referenced from the size of tx pool. txChanSize = 4096 // chainHeadChanSize is the size of channel listening to ChainHeadEvent. @@ -71,6 +71,7 @@ type Work struct { family *set.Set // family set (used for checking uncle invalidity) uncles *set.Set // uncle set tcount int // tx count in cycle + gasPool *core.GasPool // available gas used to pack transaction. Block *types.Block // the new block @@ -95,8 +96,8 @@ type worker struct { // update loop mux *event.TypeMux - txCh chan core.TxPreEvent - txSub event.Subscription + txsCh chan core.TxsPreEvent + txsSub event.Subscription chainHeadCh chan core.ChainHeadEvent chainHeadSub event.Subscription chainSideCh chan core.ChainSideEvent @@ -137,7 +138,7 @@ func newWorker(config *params.ChainConfig, engine consensus.Engine, coinbase com engine: engine, eth: eth, mux: mux, - txCh: make(chan core.TxPreEvent, txChanSize), + txsCh: make(chan core.TxsPreEvent, txChanSize), chainHeadCh: make(chan core.ChainHeadEvent, chainHeadChanSize), chainSideCh: make(chan core.ChainSideEvent, chainSideChanSize), chainDb: eth.ChainDb(), @@ -149,8 +150,8 @@ func newWorker(config *params.ChainConfig, engine consensus.Engine, coinbase com agents: make(map[Agent]struct{}), unconfirmed: newUnconfirmedBlocks(eth.BlockChain(), miningLogAtDepth), } - // Subscribe TxPreEvent for tx pool - worker.txSub = eth.TxPool().SubscribeTxPreEvent(worker.txCh) + // Subscribe TxsPreEvent for tx pool + worker.txsSub = eth.TxPool().SubscribeTxPreEvent(worker.txsCh) // Subscribe events for blockchain worker.chainHeadSub = eth.BlockChain().SubscribeChainHeadEvent(worker.chainHeadCh) worker.chainSideSub = eth.BlockChain().SubscribeChainSideEvent(worker.chainSideCh) @@ -241,7 +242,7 @@ func (self *worker) unregister(agent Agent) { } func (self *worker) update() { - defer self.txSub.Unsubscribe() + defer self.txsSub.Unsubscribe() defer self.chainHeadSub.Unsubscribe() defer self.chainSideSub.Unsubscribe() @@ -258,15 +259,21 @@ func (self *worker) update() { self.possibleUncles[ev.Block.Hash()] = ev.Block self.uncleMu.Unlock() - // Handle TxPreEvent - case ev := <-self.txCh: - // Apply transaction to the pending state if we're not mining + // Handle TxsPreEvent + case ev := <-self.txsCh: + // Apply transactions to the pending state if we're not mining. + // + // Note all transactions received may not be continuous with transactions + // already included in the current mining block. These transactions will + // be automatically eliminated. if atomic.LoadInt32(&self.mining) == 0 { self.currentMu.Lock() - acc, _ := types.Sender(self.current.signer, ev.Tx) - txs := map[common.Address]types.Transactions{acc: {ev.Tx}} + txs := make(map[common.Address]types.Transactions) + for _, tx := range ev.Txs { + acc, _ := types.Sender(self.current.signer, tx) + txs[acc] = append(txs[acc], tx) + } txset := types.NewTransactionsByPriceAndNonce(self.current.signer, txs) - self.current.commitTransactions(self.mux, txset, self.chain, self.coinbase) self.updateSnapshot() self.currentMu.Unlock() @@ -278,7 +285,7 @@ func (self *worker) update() { } // System stopped - case <-self.txSub.Err(): + case <-self.txsSub.Err(): return case <-self.chainHeadSub.Err(): return @@ -522,14 +529,16 @@ func (self *worker) updateSnapshot() { } func (env *Work) commitTransactions(mux *event.TypeMux, txs *types.TransactionsByPriceAndNonce, bc *core.BlockChain, coinbase common.Address) { - gp := new(core.GasPool).AddGas(env.header.GasLimit) + if env.gasPool == nil { + env.gasPool = new(core.GasPool).AddGas(env.header.GasLimit) + } var coalescedLogs []*types.Log for { // If we don't have enough gas for any further transactions then we're done - if gp.Gas() < params.TxGas { - log.Trace("Not enough gas for further transactions", "gp", gp) + if env.gasPool.Gas() < params.TxGas { + log.Trace("Not enough gas for further transactions", "gp", env.gasPool) break } // Retrieve the next transaction and abort if all done @@ -553,7 +562,7 @@ func (env *Work) commitTransactions(mux *event.TypeMux, txs *types.TransactionsB // Start executing the transaction env.state.Prepare(tx.Hash(), common.Hash{}, env.tcount) - err, logs := env.commitTransaction(tx, bc, coinbase, gp) + err, logs := env.commitTransaction(tx, bc, coinbase, env.gasPool) switch err { case core.ErrGasLimitReached: // Pop the current out-of-gas transaction without shifting in the next from the account From 49719e21bcd740c5890334f8c0ec8ac3777fb4c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 18 May 2018 11:45:52 +0300 Subject: [PATCH 23/46] core, eth: minor txpool event cleanups --- accounts/abi/bind/backends/simulated.go | 2 +- core/events.go | 7 ++---- core/tx_journal.go | 29 ++++++++++++------------- core/tx_pool.go | 28 ++++++++++++------------ core/tx_pool_test.go | 27 ++++++++++++----------- eth/api_backend.go | 4 ++-- eth/filters/api.go | 4 ++-- eth/filters/filter.go | 2 +- eth/filters/filter_system.go | 16 +++++++------- eth/filters/filter_system_test.go | 4 ++-- eth/handler.go | 8 +++---- eth/helper_test.go | 2 +- eth/protocol.go | 6 ++--- eth/protocol_test.go | 2 +- ethstats/ethstats.go | 12 +++++----- internal/ethapi/backend.go | 2 +- les/api_backend.go | 4 ++-- light/txpool.go | 6 ++--- miner/worker.go | 16 +++++++------- 19 files changed, 89 insertions(+), 92 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 6262f3688c..fd69538d53 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -454,7 +454,7 @@ func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*ty return logs, nil } -func (fb *filterBackend) SubscribeTxPreEvent(ch chan<- core.TxsPreEvent) event.Subscription { +func (fb *filterBackend) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subscription { return event.NewSubscription(func(quit <-chan struct{}) error { <-quit return nil diff --git a/core/events.go b/core/events.go index fee34b39d5..8d200f2a29 100644 --- a/core/events.go +++ b/core/events.go @@ -21,8 +21,8 @@ import ( "github.com/ethereum/go-ethereum/core/types" ) -// TxsPreEvent is posted when a batch of transactions enter the transaction pool. -type TxsPreEvent struct{ Txs types.Transactions } +// NewTxsEvent is posted when a batch of transactions enter the transaction pool. +type NewTxsEvent struct{ Txs []*types.Transaction } // PendingLogsEvent is posted pre mining and notifies of pending logs. type PendingLogsEvent struct { @@ -35,9 +35,6 @@ type PendingStateEvent struct{} // NewMinedBlockEvent is posted when a block has been imported. type NewMinedBlockEvent struct{ Block *types.Block } -// RemovedTransactionEvent is posted when a reorg happens -type RemovedTransactionEvent struct{ Txs types.Transactions } - // RemovedLogsEvent is posted when a reorg happens type RemovedLogsEvent struct{ Logs []*types.Log } diff --git a/core/tx_journal.go b/core/tx_journal.go index b344690b69..1397e9fd34 100644 --- a/core/tx_journal.go +++ b/core/tx_journal.go @@ -76,22 +76,21 @@ func (journal *txJournal) load(add func([]*types.Transaction) []error) error { stream := rlp.NewStream(input, 0) total, dropped := 0, 0 - // flush imports a batch of transactions and bump the appropriate progress counters - flush := func(txs types.Transactions) { - errs := add(txs) - for _, err := range errs { + // Create a method to load a limited batch of transactions and bump the + // appropriate progress counters. Then use this method to load all the + // journalled transactions in small-ish batches. + loadBatch := func(txs types.Transactions) { + for _, err := range add(txs) { if err != nil { log.Debug("Failed to add journaled transaction", "err", err) dropped++ } } } - var ( failure error - txs types.Transactions + batch types.Transactions ) - for { // Parse the next transaction and terminate on error tx := new(types.Transaction) @@ -99,19 +98,19 @@ func (journal *txJournal) load(add func([]*types.Transaction) []error) error { if err != io.EOF { failure = err } + if batch.Len() > 0 { + loadBatch(batch) + } break } - txs = append(txs, tx) + // New transaction parsed, queue up for later, import if threnshold is reached total++ - if txs.Len() > 1024 { - flush(txs) - txs = types.Transactions{} + + if batch = append(batch, tx); batch.Len() > 1024 { + loadBatch(batch) + batch = batch[:0] } } - if txs.Len() > 0 { - flush(txs) - txs = types.Transactions{} - } log.Info("Loaded local transaction journal", "transactions", total, "dropped", dropped) return failure diff --git a/core/tx_pool.go b/core/tx_pool.go index f23828cb4e..f89e11441e 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -444,9 +444,9 @@ func (pool *TxPool) Stop() { log.Info("Transaction pool stopped") } -// SubscribeTxPreEvent registers a subscription of TxsPreEvent and +// SubscribeNewTxsEvent registers a subscription of NewTxsEvent and // starts sending event to the given channel. -func (pool *TxPool) SubscribeTxPreEvent(ch chan<- TxsPreEvent) event.Subscription { +func (pool *TxPool) SubscribeNewTxsEvent(ch chan<- NewTxsEvent) event.Subscription { return pool.scope.Track(pool.txFeed.Subscribe(ch)) } @@ -653,7 +653,7 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (bool, error) { log.Trace("Pooled new executable transaction", "hash", hash, "from", from, "to", tx.To()) // We've directly injected a replacement transaction, notify subsystems - go pool.txFeed.Send(TxsPreEvent{types.Transactions{tx}}) + go pool.txFeed.Send(NewTxsEvent{types.Transactions{tx}}) return old != nil, nil } @@ -712,7 +712,8 @@ func (pool *TxPool) journalTx(from common.Address, tx *types.Transaction) { } } -// promoteTx adds a transaction to the pending (processable) list of transactions. +// promoteTx adds a transaction to the pending (processable) list of transactions +// and returns whether it was inserted or an older was better. // // Note, this method assumes the pool lock is held! func (pool *TxPool) promoteTx(addr common.Address, hash common.Hash, tx *types.Transaction) bool { @@ -746,6 +747,7 @@ func (pool *TxPool) promoteTx(addr common.Address, hash common.Hash, tx *types.T // Set the potentially new pending nonce and notify any subsystems of the new tx pool.beats[addr] = time.Now() pool.pendingState.SetNonce(addr, tx.Nonce()+1) + return true } @@ -906,7 +908,9 @@ func (pool *TxPool) removeTx(hash common.Hash, outofbound bool) { // future queue to the set of pending transactions. During this process, all // invalidated transactions (low nonce, low balance) are deleted. func (pool *TxPool) promoteExecutables(accounts []common.Address) { - var promotedTxs types.Transactions + // Track the promoted transactions to broadcast them at once + var promoted []*types.Transaction + // Gather all the accounts potentially needing updates if accounts == nil { accounts = make([]common.Address, 0, len(pool.queue)) @@ -937,16 +941,13 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) { queuedNofundsCounter.Inc(1) } // Gather all executable transactions and promote them - txs := list.Ready(pool.pendingState.GetNonce(addr)) - for _, tx := range txs { + for _, tx := range list.Ready(pool.pendingState.GetNonce(addr)) { hash := tx.Hash() - inserted := pool.promoteTx(addr, hash, tx) - if inserted { + if pool.promoteTx(addr, hash, tx) { log.Trace("Promoting queued transaction", "hash", hash) - promotedTxs = append(promotedTxs, tx) + promoted = append(promoted, tx) } } - // Drop all transactions over the allowed limit if !pool.locals.contains(addr) { for _, tx := range list.Cap(int(pool.config.AccountQueue)) { @@ -963,10 +964,9 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) { } } // Notify subsystem for new promoted transactions. - if promotedTxs.Len() > 0 { - pool.txFeed.Send(TxsPreEvent{promotedTxs}) + if len(promoted) > 0 { + pool.txFeed.Send(NewTxsEvent{promoted}) } - // If the pending limit is overflown, start equalizing allowances pending := uint64(0) for _, list := range pool.pending { diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index c19581e3f3..25993c258b 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -118,19 +118,20 @@ func validateTxPoolInternals(pool *TxPool) error { // validateEvents checks that the correct number of transaction addition events // were fired on the pool's event feed. -func validateEvents(events chan TxsPreEvent, count int) error { - received := 0 - for { - if received == count { - break - } +func validateEvents(events chan NewTxsEvent, count int) error { + var received []*types.Transaction + + for len(received) < count { select { case ev := <-events: - received += ev.Txs.Len() + received = append(received, ev.Txs...) case <-time.After(time.Second): return fmt.Errorf("event #%d not fired", received) } } + if len(received) > count { + return fmt.Errorf("more than %d events fired: %v", count, received[count:]) + } select { case ev := <-events: return fmt.Errorf("more than %d events fired: %v", count, ev.Txs) @@ -674,7 +675,7 @@ func TestTransactionGapFilling(t *testing.T) { pool.currentState.AddBalance(account, big.NewInt(1000000)) // Keep track of transaction events to ensure all executables get announced - events := make(chan TxsPreEvent, testTxPoolConfig.AccountQueue+5) + events := make(chan NewTxsEvent, testTxPoolConfig.AccountQueue+5) sub := pool.txFeed.Subscribe(events) defer sub.Unsubscribe() @@ -925,7 +926,7 @@ func TestTransactionPendingLimiting(t *testing.T) { pool.currentState.AddBalance(account, big.NewInt(1000000)) // Keep track of transaction events to ensure all executables get announced - events := make(chan TxsPreEvent, testTxPoolConfig.AccountQueue+5) + events := make(chan NewTxsEvent, testTxPoolConfig.AccountQueue+5) sub := pool.txFeed.Subscribe(events) defer sub.Unsubscribe() @@ -1145,7 +1146,7 @@ func TestTransactionPoolRepricing(t *testing.T) { defer pool.Stop() // Keep track of transaction events to ensure all executables get announced - events := make(chan TxsPreEvent, 32) + events := make(chan NewTxsEvent, 32) sub := pool.txFeed.Subscribe(events) defer sub.Unsubscribe() @@ -1332,7 +1333,7 @@ func TestTransactionPoolUnderpricing(t *testing.T) { defer pool.Stop() // Keep track of transaction events to ensure all executables get announced - events := make(chan TxsPreEvent, 32) + events := make(chan NewTxsEvent, 32) sub := pool.txFeed.Subscribe(events) defer sub.Unsubscribe() @@ -1438,7 +1439,7 @@ func TestTransactionPoolStableUnderpricing(t *testing.T) { defer pool.Stop() // Keep track of transaction events to ensure all executables get announced - events := make(chan TxsPreEvent, 32) + events := make(chan NewTxsEvent, 32) sub := pool.txFeed.Subscribe(events) defer sub.Unsubscribe() @@ -1500,7 +1501,7 @@ func TestTransactionReplacement(t *testing.T) { defer pool.Stop() // Keep track of transaction events to ensure all executables get announced - events := make(chan TxsPreEvent, 32) + events := make(chan NewTxsEvent, 32) sub := pool.txFeed.Subscribe(events) defer sub.Unsubscribe() diff --git a/eth/api_backend.go b/eth/api_backend.go index ef70b12b75..91a7dc7e0e 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -188,8 +188,8 @@ func (b *EthAPIBackend) TxPoolContent() (map[common.Address]types.Transactions, return b.eth.TxPool().Content() } -func (b *EthAPIBackend) SubscribeTxPreEvent(ch chan<- core.TxsPreEvent) event.Subscription { - return b.eth.TxPool().SubscribeTxPreEvent(ch) +func (b *EthAPIBackend) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subscription { + return b.eth.TxPool().SubscribeNewTxsEvent(ch) } func (b *EthAPIBackend) Downloader() *downloader.Downloader { diff --git a/eth/filters/api.go b/eth/filters/api.go index d2c9258f91..592ad3b82f 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -105,7 +105,7 @@ func (api *PublicFilterAPI) timeoutLoop() { func (api *PublicFilterAPI) NewPendingTransactionFilter() rpc.ID { var ( pendingTxs = make(chan []common.Hash) - pendingTxSub = api.events.SubscribePendingTxEvents(pendingTxs) + pendingTxSub = api.events.SubscribePendingTxs(pendingTxs) ) api.filtersMu.Lock() @@ -145,7 +145,7 @@ func (api *PublicFilterAPI) NewPendingTransactions(ctx context.Context) (*rpc.Su go func() { txHashes := make(chan []common.Hash, 128) - pendingTxSub := api.events.SubscribePendingTxEvents(txHashes) + pendingTxSub := api.events.SubscribePendingTxs(txHashes) for { select { diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 45d91bea00..67b4612ae4 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -36,7 +36,7 @@ type Backend interface { GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) - SubscribeTxPreEvent(chan<- core.TxsPreEvent) event.Subscription + SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index 5f1c12c6a8..4e999cda85 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -59,7 +59,7 @@ const ( const ( - // txChanSize is the size of channel listening to TxsPreEvent. + // txChanSize is the size of channel listening to NewTxsEvent. // The number is referenced from the size of tx pool. txChanSize = 4096 // rmLogsChanSize is the size of channel listening to RemovedLogsEvent. @@ -104,7 +104,7 @@ type EventSystem struct { // Channels install chan *subscription // install filter for event notification uninstall chan *subscription // remove filter for event notification - txsCh chan core.TxsPreEvent // Channel to receive new transactions event + txsCh chan core.NewTxsEvent // Channel to receive new transactions event logsCh chan []*types.Log // Channel to receive new log event rmLogsCh chan core.RemovedLogsEvent // Channel to receive removed log event chainCh chan core.ChainEvent // Channel to receive new chain event @@ -123,14 +123,14 @@ func NewEventSystem(mux *event.TypeMux, backend Backend, lightMode bool) *EventS lightMode: lightMode, install: make(chan *subscription), uninstall: make(chan *subscription), - txsCh: make(chan core.TxsPreEvent, txChanSize), + txsCh: make(chan core.NewTxsEvent, txChanSize), logsCh: make(chan []*types.Log, logsChanSize), rmLogsCh: make(chan core.RemovedLogsEvent, rmLogsChanSize), chainCh: make(chan core.ChainEvent, chainEvChanSize), } // Subscribe events - m.txsSub = m.backend.SubscribeTxPreEvent(m.txsCh) + m.txsSub = m.backend.SubscribeNewTxsEvent(m.txsCh) m.logsSub = m.backend.SubscribeLogsEvent(m.logsCh) m.rmLogsSub = m.backend.SubscribeRemovedLogsEvent(m.rmLogsCh) m.chainSub = m.backend.SubscribeChainEvent(m.chainCh) @@ -298,9 +298,9 @@ func (es *EventSystem) SubscribeNewHeads(headers chan *types.Header) *Subscripti return es.subscribe(sub) } -// SubscribePendingTxEvents creates a subscription that writes transaction hashes for +// SubscribePendingTxs creates a subscription that writes transaction hashes for // transactions that enter the transaction pool. -func (es *EventSystem) SubscribePendingTxEvents(hashes chan []common.Hash) *Subscription { +func (es *EventSystem) SubscribePendingTxs(hashes chan []common.Hash) *Subscription { sub := &subscription{ id: rpc.NewID(), typ: PendingTransactionsSubscription, @@ -348,8 +348,8 @@ func (es *EventSystem) broadcast(filters filterIndex, ev interface{}) { } } } - case core.TxsPreEvent: - hashes := make([]common.Hash, 0, e.Txs.Len()) + case core.NewTxsEvent: + hashes := make([]common.Hash, 0, len(e.Txs)) for _, tx := range e.Txs { hashes = append(hashes, tx.Hash()) } diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index c43d282ae6..ff1af85a80 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -96,7 +96,7 @@ func (b *testBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types return logs, nil } -func (b *testBackend) SubscribeTxPreEvent(ch chan<- core.TxsPreEvent) event.Subscription { +func (b *testBackend) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subscription { return b.txFeed.Subscribe(ch) } @@ -232,7 +232,7 @@ func TestPendingTxFilter(t *testing.T) { fid0 := api.NewPendingTransactionFilter() time.Sleep(1 * time.Second) - txFeed.Send(core.TxsPreEvent{transactions}) + txFeed.Send(core.NewTxsEvent{Txs: transactions}) timeout := time.Now().Add(1 * time.Second) for { diff --git a/eth/handler.go b/eth/handler.go index 7ef8e957a4..8993afe150 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -46,7 +46,7 @@ const ( softResponseLimit = 2 * 1024 * 1024 // Target maximum size of returned blocks, headers or node data. estHeaderRlpSize = 500 // Approximate size of an RLP encoded block header - // txChanSize is the size of channel listening to TxsPreEvent. + // txChanSize is the size of channel listening to NewTxsEvent. // The number is referenced from the size of tx pool. txChanSize = 4096 ) @@ -81,7 +81,7 @@ type ProtocolManager struct { SubProtocols []p2p.Protocol eventMux *event.TypeMux - txsCh chan core.TxsPreEvent + txsCh chan core.NewTxsEvent txsSub event.Subscription minedBlockSub *event.TypeMuxSubscription @@ -204,8 +204,8 @@ func (pm *ProtocolManager) Start(maxPeers int) { pm.maxPeers = maxPeers // broadcast transactions - pm.txsCh = make(chan core.TxsPreEvent, txChanSize) - pm.txsSub = pm.txpool.SubscribeTxPreEvent(pm.txsCh) + pm.txsCh = make(chan core.NewTxsEvent, txChanSize) + pm.txsSub = pm.txpool.SubscribeNewTxsEvent(pm.txsCh) go pm.txBroadcastLoop() // broadcast mined blocks diff --git a/eth/helper_test.go b/eth/helper_test.go index 40a172ef83..3d2ab0aba1 100644 --- a/eth/helper_test.go +++ b/eth/helper_test.go @@ -124,7 +124,7 @@ func (p *testTxPool) Pending() (map[common.Address]types.Transactions, error) { return batches, nil } -func (p *testTxPool) SubscribeTxPreEvent(ch chan<- core.TxsPreEvent) event.Subscription { +func (p *testTxPool) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subscription { return p.txFeed.Subscribe(ch) } diff --git a/eth/protocol.go b/eth/protocol.go index bee3d83651..0e90e6a2ef 100644 --- a/eth/protocol.go +++ b/eth/protocol.go @@ -103,9 +103,9 @@ type txPool interface { // The slice should be modifiable by the caller. Pending() (map[common.Address]types.Transactions, error) - // SubscribeTxPreEvent should return an event subscription of - // TxsPreEvent and send events to the given channel. - SubscribeTxPreEvent(chan<- core.TxsPreEvent) event.Subscription + // SubscribeNewTxsEvent should return an event subscription of + // NewTxsEvent and send events to the given channel. + SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription } // statusData is the network packet for the status message. diff --git a/eth/protocol_test.go b/eth/protocol_test.go index 1c32ea7d83..aa43dfa920 100644 --- a/eth/protocol_test.go +++ b/eth/protocol_test.go @@ -116,7 +116,7 @@ func testRecvTransactions(t *testing.T, protocol int) { t.Errorf("added wrong tx hash: got %v, want %v", added[0].Hash(), tx.Hash()) } case <-time.After(2 * time.Second): - t.Errorf("no TxsPreEvent received within 2 seconds") + t.Errorf("no NewTxsEvent received within 2 seconds") } } diff --git a/ethstats/ethstats.go b/ethstats/ethstats.go index 17f9f3c7a5..c116014351 100644 --- a/ethstats/ethstats.go +++ b/ethstats/ethstats.go @@ -49,7 +49,7 @@ const ( // history request. historyUpdateRange = 50 - // txChanSize is the size of channel listening to TxsPreEvent. + // txChanSize is the size of channel listening to NewTxsEvent. // The number is referenced from the size of tx pool. txChanSize = 4096 // chainHeadChanSize is the size of channel listening to ChainHeadEvent. @@ -57,9 +57,9 @@ const ( ) type txPool interface { - // SubscribeTxPreEvent should return an event subscription of - // TxsPreEvent and send events to the given channel. - SubscribeTxPreEvent(chan<- core.TxsPreEvent) event.Subscription + // SubscribeNewTxsEvent should return an event subscription of + // NewTxsEvent and send events to the given channel. + SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription } type blockChain interface { @@ -150,8 +150,8 @@ func (s *Service) loop() { headSub := blockchain.SubscribeChainHeadEvent(chainHeadCh) defer headSub.Unsubscribe() - txEventCh := make(chan core.TxsPreEvent, txChanSize) - txSub := txpool.SubscribeTxPreEvent(txEventCh) + txEventCh := make(chan core.NewTxsEvent, txChanSize) + txSub := txpool.SubscribeNewTxsEvent(txEventCh) defer txSub.Unsubscribe() // Start a goroutine that exhausts the subsciptions to avoid events piling up diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index e351152b9c..c9ffe230c6 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -65,7 +65,7 @@ type Backend interface { GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) Stats() (pending int, queued int) TxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions) - SubscribeTxPreEvent(chan<- core.TxsPreEvent) event.Subscription + SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription ChainConfig() *params.ChainConfig CurrentBlock() *types.Block diff --git a/les/api_backend.go b/les/api_backend.go index 957c8ae649..dea33c4702 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -136,8 +136,8 @@ func (b *LesApiBackend) TxPoolContent() (map[common.Address]types.Transactions, return b.eth.txPool.Content() } -func (b *LesApiBackend) SubscribeTxPreEvent(ch chan<- core.TxsPreEvent) event.Subscription { - return b.eth.txPool.SubscribeTxPreEvent(ch) +func (b *LesApiBackend) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subscription { + return b.eth.txPool.SubscribeNewTxsEvent(ch) } func (b *LesApiBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { diff --git a/light/txpool.go b/light/txpool.go index bedfd48a25..1fabc3dc5a 100644 --- a/light/txpool.go +++ b/light/txpool.go @@ -321,9 +321,9 @@ func (pool *TxPool) Stop() { log.Info("Transaction pool stopped") } -// SubscribeTxPreEvent registers a subscription of core.TxsPreEvent and +// SubscribeNewTxsEvent registers a subscription of core.NewTxsEvent and // starts sending event to the given channel. -func (pool *TxPool) SubscribeTxPreEvent(ch chan<- core.TxsPreEvent) event.Subscription { +func (pool *TxPool) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subscription { return pool.scope.Track(pool.txFeed.Subscribe(ch)) } @@ -412,7 +412,7 @@ func (self *TxPool) add(ctx context.Context, tx *types.Transaction) error { // Notify the subscribers. This event is posted in a goroutine // because it's possible that somewhere during the post "Remove transaction" // gets called which will then wait for the global tx pool lock and deadlock. - go self.txFeed.Send(core.TxsPreEvent{types.Transactions{tx}}) + go self.txFeed.Send(core.NewTxsEvent{Txs: types.Transactions{tx}}) } // Print a log message if low enough level is set diff --git a/miner/worker.go b/miner/worker.go index 3d086d6aee..640e9032e7 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -42,7 +42,7 @@ const ( resultQueueSize = 10 miningLogAtDepth = 5 - // txChanSize is the size of channel listening to TxsPreEvent. + // txChanSize is the size of channel listening to NewTxsEvent. // The number is referenced from the size of tx pool. txChanSize = 4096 // chainHeadChanSize is the size of channel listening to ChainHeadEvent. @@ -71,7 +71,7 @@ type Work struct { family *set.Set // family set (used for checking uncle invalidity) uncles *set.Set // uncle set tcount int // tx count in cycle - gasPool *core.GasPool // available gas used to pack transaction. + gasPool *core.GasPool // available gas used to pack transactions Block *types.Block // the new block @@ -96,7 +96,7 @@ type worker struct { // update loop mux *event.TypeMux - txsCh chan core.TxsPreEvent + txsCh chan core.NewTxsEvent txsSub event.Subscription chainHeadCh chan core.ChainHeadEvent chainHeadSub event.Subscription @@ -138,7 +138,7 @@ func newWorker(config *params.ChainConfig, engine consensus.Engine, coinbase com engine: engine, eth: eth, mux: mux, - txsCh: make(chan core.TxsPreEvent, txChanSize), + txsCh: make(chan core.NewTxsEvent, txChanSize), chainHeadCh: make(chan core.ChainHeadEvent, chainHeadChanSize), chainSideCh: make(chan core.ChainSideEvent, chainSideChanSize), chainDb: eth.ChainDb(), @@ -150,8 +150,8 @@ func newWorker(config *params.ChainConfig, engine consensus.Engine, coinbase com agents: make(map[Agent]struct{}), unconfirmed: newUnconfirmedBlocks(eth.BlockChain(), miningLogAtDepth), } - // Subscribe TxsPreEvent for tx pool - worker.txsSub = eth.TxPool().SubscribeTxPreEvent(worker.txsCh) + // Subscribe NewTxsEvent for tx pool + worker.txsSub = eth.TxPool().SubscribeNewTxsEvent(worker.txsCh) // Subscribe events for blockchain worker.chainHeadSub = eth.BlockChain().SubscribeChainHeadEvent(worker.chainHeadCh) worker.chainSideSub = eth.BlockChain().SubscribeChainSideEvent(worker.chainSideCh) @@ -259,7 +259,7 @@ func (self *worker) update() { self.possibleUncles[ev.Block.Hash()] = ev.Block self.uncleMu.Unlock() - // Handle TxsPreEvent + // Handle NewTxsEvent case ev := <-self.txsCh: // Apply transactions to the pending state if we're not mining. // @@ -538,7 +538,7 @@ func (env *Work) commitTransactions(mux *event.TypeMux, txs *types.TransactionsB for { // If we don't have enough gas for any further transactions then we're done if env.gasPool.Gas() < params.TxGas { - log.Trace("Not enough gas for further transactions", "gp", env.gasPool) + log.Trace("Not enough gas for further transactions", "have", env.gasPool, "want", params.TxGas) break } // Retrieve the next transaction and abort if all done From 579bd0f9fb368736288ca23e72c6b8f9256ac8c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 18 May 2018 12:24:04 +0300 Subject: [PATCH 24/46] travis, appveyor: bump Go release to 1.10.2 --- .travis.yml | 2 +- appveyor.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 09b77d09d7..6fb70f27df 100644 --- a/.travis.yml +++ b/.travis.yml @@ -146,7 +146,7 @@ matrix: git: submodules: false # avoid cloning ethereum/tests before_install: - - curl https://storage.googleapis.com/golang/go1.10.1.linux-amd64.tar.gz | tar -xz + - curl https://storage.googleapis.com/golang/go1.10.2.linux-amd64.tar.gz | tar -xz - export PATH=`pwd`/go/bin:$PATH - export GOROOT=`pwd`/go - export GOPATH=$HOME/go diff --git a/appveyor.yml b/appveyor.yml index 141ef16ff8..f10000af86 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -23,8 +23,8 @@ environment: install: - git submodule update --init - rmdir C:\go /s /q - - appveyor DownloadFile https://storage.googleapis.com/golang/go1.10.1.windows-%GETH_ARCH%.zip - - 7z x go1.10.1.windows-%GETH_ARCH%.zip -y -oC:\ > NUL + - appveyor DownloadFile https://storage.googleapis.com/golang/go1.10.2.windows-%GETH_ARCH%.zip + - 7z x go1.10.2.windows-%GETH_ARCH%.zip -y -oC:\ > NUL - go version - gcc --version From f2fdb75dd98769146d3284bfbd74b25e6e18e5ef Mon Sep 17 00:00:00 2001 From: hadv Date: Sat, 19 May 2018 15:39:41 +0700 Subject: [PATCH 25/46] core, consensus: fix some typos in comment code and output log --- consensus/clique/clique.go | 2 +- core/state/statedb.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index 2bdad9092a..01df4d5c78 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -383,7 +383,7 @@ func (c *Clique) snapshot(chain consensus.ChainReader, number uint64, hash commo // If an on-disk checkpoint snapshot can be found, use that if number%checkpointInterval == 0 { if s, err := loadSnapshot(c.config, c.signatures, c.db, hash); err == nil { - log.Trace("Loaded voting snapshot form disk", "number", number, "hash", hash) + log.Trace("Loaded voting snapshot from disk", "number", number, "hash", hash) snap = s break } diff --git a/core/state/statedb.go b/core/state/statedb.go index a952027d6d..ffea761d9f 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -358,7 +358,7 @@ func (self *StateDB) deleteStateObject(stateObject *stateObject) { self.setError(self.trie.TryDelete(addr[:])) } -// Retrieve a state object given my the address. Returns nil if not found. +// Retrieve a state object given by the address. Returns nil if not found. func (self *StateDB) getStateObject(addr common.Address) (stateObject *stateObject) { // Prefer 'live' objects. if obj := self.stateObjects[addr]; obj != nil { From d9cee2c172144c1d7efe68c454afac8132fe0911 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 21 May 2018 11:32:42 +0300 Subject: [PATCH 26/46] eth: propagate blocks and transactions async --- eth/handler.go | 6 +-- eth/peer.go | 123 +++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 117 insertions(+), 12 deletions(-) diff --git a/eth/handler.go b/eth/handler.go index 8993afe150..918d71088d 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -698,7 +698,7 @@ func (pm *ProtocolManager) BroadcastBlock(block *types.Block, propagate bool) { // Send the block to a subset of our peers transfer := peers[:int(math.Sqrt(float64(len(peers))))] for _, peer := range transfer { - peer.SendNewBlock(block, td) + peer.AsyncSendNewBlock(block, td) } log.Trace("Propagated block", "hash", hash, "recipients", len(transfer), "duration", common.PrettyDuration(time.Since(block.ReceivedAt))) return @@ -706,7 +706,7 @@ func (pm *ProtocolManager) BroadcastBlock(block *types.Block, propagate bool) { // Otherwise if the block is indeed in out own chain, announce it if pm.blockchain.HasBlock(hash, block.NumberU64()) { for _, peer := range peers { - peer.SendNewBlockHashes([]common.Hash{hash}, []uint64{block.NumberU64()}) + peer.AsyncSendNewBlockHash(block) } log.Trace("Announced block", "hash", hash, "recipients", len(peers), "duration", common.PrettyDuration(time.Since(block.ReceivedAt))) } @@ -727,7 +727,7 @@ func (pm *ProtocolManager) BroadcastTxs(txs types.Transactions) { } // FIXME include this again: peers = peers[:int(math.Sqrt(float64(len(peers))))] for peer, txs := range txset { - peer.SendTransactions(txs) + peer.AsyncSendTransactions(txs) } } diff --git a/eth/peer.go b/eth/peer.go index 42ead53965..953aca17b9 100644 --- a/eth/peer.go +++ b/eth/peer.go @@ -37,8 +37,24 @@ var ( ) const ( - maxKnownTxs = 32768 // Maximum transactions hashes to keep in the known list (prevent DOS) - maxKnownBlocks = 1024 // Maximum block hashes to keep in the known list (prevent DOS) + maxKnownTxs = 32768 // Maximum transactions hashes to keep in the known list (prevent DOS) + maxKnownBlocks = 1024 // Maximum block hashes to keep in the known list (prevent DOS) + + // maxQueuedTxs is the maximum number of transaction lists to queue up before + // dropping broadcasts. This is a sensitive number as a transaction list might + // contain a single transaction, or thousands. + maxQueuedTxs = 128 + + // maxQueuedProps is the maximum number of block propagations to queue up before + // dropping broadcasts. There's not much point in queueing stale blocks, so a few + // that might cover uncles should be enough. + maxQueuedProps = 4 + + // maxQueuedAnns is the maximum number of block announcements to queue up before + // dropping broadcasts. Similarly to block propagations, there's no point to queue + // above some healthy uncle limit, so use that. + maxQueuedAnns = 4 + handshakeTimeout = 5 * time.Second ) @@ -50,6 +66,12 @@ type PeerInfo struct { Head string `json:"head"` // SHA3 hash of the peer's best owned block } +// propEvent is a block propagation, waiting for its turn in the broadcast queue. +type propEvent struct { + block *types.Block + td *big.Int +} + type peer struct { id string @@ -63,23 +85,64 @@ type peer struct { td *big.Int lock sync.RWMutex - knownTxs *set.Set // Set of transaction hashes known to be known by this peer - knownBlocks *set.Set // Set of block hashes known to be known by this peer + knownTxs *set.Set // Set of transaction hashes known to be known by this peer + knownBlocks *set.Set // Set of block hashes known to be known by this peer + queuedTxs chan []*types.Transaction // Queue of transactions to broadcast to the peer + queuedProps chan *propEvent // Queue of blocks to broadcast to the peer + queuedAnns chan *types.Block // Queue of blocks to announce to the peer + term chan struct{} // Termination channel to stop the broadcaster } func newPeer(version int, p *p2p.Peer, rw p2p.MsgReadWriter) *peer { - id := p.ID() - return &peer{ Peer: p, rw: rw, version: version, - id: fmt.Sprintf("%x", id[:8]), + id: fmt.Sprintf("%x", p.ID().Bytes()[:8]), knownTxs: set.New(), knownBlocks: set.New(), + queuedTxs: make(chan []*types.Transaction, maxQueuedTxs), + queuedProps: make(chan *propEvent, maxQueuedProps), + queuedAnns: make(chan *types.Block, maxQueuedAnns), + term: make(chan struct{}), } } +// broadcast is a write loop that multiplexes block propagations, announcements +// and transaction broadcasts into the remote peer. The goal is to have an async +// writer that does not lock up node internals. +func (p *peer) broadcast() { + for { + select { + case txs := <-p.queuedTxs: + if err := p.SendTransactions(txs); err != nil { + return + } + p.Log().Trace("Broadcast transactions", "count", len(txs)) + + case prop := <-p.queuedProps: + if err := p.SendNewBlock(prop.block, prop.td); err != nil { + return + } + p.Log().Trace("Propagated block", "number", prop.block.Number(), "hash", prop.block.Hash(), "td", prop.td) + + case block := <-p.queuedAnns: + if err := p.SendNewBlockHashes([]common.Hash{block.Hash()}, []uint64{block.NumberU64()}); err != nil { + return + } + p.Log().Trace("Announced block", "number", block.Number(), "hash", block.Hash()) + + case <-p.term: + return + } + } +} + +// close signals the broadcast goroutine to terminate. +func (p *peer) close() { + close(p.term) +} + // Info gathers and returns a collection of metadata known about a peer. func (p *peer) Info() *PeerInfo { hash, td := p.Head() @@ -139,6 +202,19 @@ func (p *peer) SendTransactions(txs types.Transactions) error { return p2p.Send(p.rw, TxMsg, txs) } +// AsyncSendTransactions queues list of transactions propagation to a remote +// peer. If the peer's broadcast queue is full, the event is silently dropped. +func (p *peer) AsyncSendTransactions(txs []*types.Transaction) { + select { + case p.queuedTxs <- txs: + for _, tx := range txs { + p.knownTxs.Add(tx.Hash()) + } + default: + p.Log().Debug("Dropping transaction propagation", "count", len(txs)) + } +} + // SendNewBlockHashes announces the availability of a number of blocks through // a hash notification. func (p *peer) SendNewBlockHashes(hashes []common.Hash, numbers []uint64) error { @@ -153,12 +229,35 @@ func (p *peer) SendNewBlockHashes(hashes []common.Hash, numbers []uint64) error return p2p.Send(p.rw, NewBlockHashesMsg, request) } +// AsyncSendNewBlockHash queues the availability of a block for propagation to a +// remote peer. If the peer's broadcast queue is full, the event is silently +// dropped. +func (p *peer) AsyncSendNewBlockHash(block *types.Block) { + select { + case p.queuedAnns <- block: + p.knownBlocks.Add(block.Hash()) + default: + p.Log().Debug("Dropping block announcement", "number", block.NumberU64(), "hash", block.Hash()) + } +} + // SendNewBlock propagates an entire block to a remote peer. func (p *peer) SendNewBlock(block *types.Block, td *big.Int) error { p.knownBlocks.Add(block.Hash()) return p2p.Send(p.rw, NewBlockMsg, []interface{}{block, td}) } +// AsyncSendNewBlock queues an entire block for propagation to a remote peer. If +// the peer's broadcast queue is full, the event is silently dropped. +func (p *peer) AsyncSendNewBlock(block *types.Block, td *big.Int) { + select { + case p.queuedProps <- &propEvent{block: block, td: td}: + p.knownBlocks.Add(block.Hash()) + default: + p.Log().Debug("Dropping block propagation", "number", block.NumberU64(), "hash", block.Hash()) + } +} + // SendBlockHeaders sends a batch of block headers to the remote peer. func (p *peer) SendBlockHeaders(headers []*types.Header) error { return p2p.Send(p.rw, BlockHeadersMsg, headers) @@ -313,7 +412,8 @@ func newPeerSet() *peerSet { } // Register injects a new peer into the working set, or returns an error if the -// peer is already known. +// peer is already known. If a new peer it registered, its broadcast loop is also +// started. func (ps *peerSet) Register(p *peer) error { ps.lock.Lock() defer ps.lock.Unlock() @@ -325,6 +425,8 @@ func (ps *peerSet) Register(p *peer) error { return errAlreadyRegistered } ps.peers[p.id] = p + go p.broadcast() + return nil } @@ -334,10 +436,13 @@ func (ps *peerSet) Unregister(id string) error { ps.lock.Lock() defer ps.lock.Unlock() - if _, ok := ps.peers[id]; !ok { + p, ok := ps.peers[id] + if !ok { return errNotRegistered } delete(ps.peers, id) + p.close() + return nil } From 0fe47e98c4d93f952dc7415f401fd03b2664a21d Mon Sep 17 00:00:00 2001 From: kiel barry Date: Mon, 21 May 2018 13:41:31 -0700 Subject: [PATCH 27/46] trie: fixes to comply with golint (#16771) --- les/odr_requests.go | 10 +++++----- trie/iterator.go | 14 +++++++------- trie/proof.go | 10 +++++----- trie/proof_test.go | 8 ++++---- trie/secure_trie.go | 5 +++++ 5 files changed, 26 insertions(+), 21 deletions(-) diff --git a/les/odr_requests.go b/les/odr_requests.go index c7f06df94d..075fcd92ca 100644 --- a/les/odr_requests.go +++ b/les/odr_requests.go @@ -230,7 +230,7 @@ func (r *TrieRequest) Validate(db ethdb.Database, msg *Msg) error { } nodeSet := proofs[0].NodeSet() // Verify the proof and store if checks out - if _, err, _ := trie.VerifyProof(r.Id.Root, r.Key, nodeSet); err != nil { + if _, _, err := trie.VerifyProof(r.Id.Root, r.Key, nodeSet); err != nil { return fmt.Errorf("merkle proof verification failed: %v", err) } r.Proof = nodeSet @@ -241,7 +241,7 @@ func (r *TrieRequest) Validate(db ethdb.Database, msg *Msg) error { // Verify the proof and store if checks out nodeSet := proofs.NodeSet() reads := &readTraceDB{db: nodeSet} - if _, err, _ := trie.VerifyProof(r.Id.Root, r.Key, reads); err != nil { + if _, _, err := trie.VerifyProof(r.Id.Root, r.Key, reads); err != nil { return fmt.Errorf("merkle proof verification failed: %v", err) } // check if all nodes have been read by VerifyProof @@ -400,7 +400,7 @@ func (r *ChtRequest) Validate(db ethdb.Database, msg *Msg) error { var encNumber [8]byte binary.BigEndian.PutUint64(encNumber[:], r.BlockNum) - value, err, _ := trie.VerifyProof(r.ChtRoot, encNumber[:], light.NodeList(proof.Proof).NodeSet()) + value, _, err := trie.VerifyProof(r.ChtRoot, encNumber[:], light.NodeList(proof.Proof).NodeSet()) if err != nil { return err } @@ -435,7 +435,7 @@ func (r *ChtRequest) Validate(db ethdb.Database, msg *Msg) error { binary.BigEndian.PutUint64(encNumber[:], r.BlockNum) reads := &readTraceDB{db: nodeSet} - value, err, _ := trie.VerifyProof(r.ChtRoot, encNumber[:], reads) + value, _, err := trie.VerifyProof(r.ChtRoot, encNumber[:], reads) if err != nil { return fmt.Errorf("merkle proof verification failed: %v", err) } @@ -529,7 +529,7 @@ func (r *BloomRequest) Validate(db ethdb.Database, msg *Msg) error { for i, idx := range r.SectionIdxList { binary.BigEndian.PutUint64(encNumber[2:], idx) - value, err, _ := trie.VerifyProof(r.BloomTrieRoot, encNumber[:], reads) + value, _, err := trie.VerifyProof(r.BloomTrieRoot, encNumber[:], reads) if err != nil { return err } diff --git a/trie/iterator.go b/trie/iterator.go index 3bae8e186b..64110c6d96 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -99,8 +99,8 @@ type nodeIterator struct { err error // Failure set in case of an internal error in the iterator } -// iteratorEnd is stored in nodeIterator.err when iteration is done. -var iteratorEnd = errors.New("end of iteration") +// errIteratorEnd is stored in nodeIterator.err when iteration is done. +var errIteratorEnd = errors.New("end of iteration") // seekError is stored in nodeIterator.err if the initial seek has failed. type seekError struct { @@ -162,7 +162,7 @@ func (it *nodeIterator) Path() []byte { } func (it *nodeIterator) Error() error { - if it.err == iteratorEnd { + if it.err == errIteratorEnd { return nil } if seek, ok := it.err.(seekError); ok { @@ -176,7 +176,7 @@ func (it *nodeIterator) Error() error { // sets the Error field to the encountered failure. If `descend` is false, // skips iterating over any subnodes of the current node. func (it *nodeIterator) Next(descend bool) bool { - if it.err == iteratorEnd { + if it.err == errIteratorEnd { return false } if seek, ok := it.err.(seekError); ok { @@ -201,8 +201,8 @@ func (it *nodeIterator) seek(prefix []byte) error { // Move forward until we're just before the closest match to key. for { state, parentIndex, path, err := it.peek(bytes.HasPrefix(key, it.path)) - if err == iteratorEnd { - return iteratorEnd + if err == errIteratorEnd { + return errIteratorEnd } else if err != nil { return seekError{prefix, err} } else if bytes.Compare(path, key) >= 0 { @@ -246,7 +246,7 @@ func (it *nodeIterator) peek(descend bool) (*nodeIteratorState, *int, []byte, er // No more child nodes, move back up. it.pop() } - return nil, nil, nil, iteratorEnd + return nil, nil, nil, errIteratorEnd } func (st *nodeIteratorState) resolve(tr *Trie, path []byte) error { diff --git a/trie/proof.go b/trie/proof.go index 508e4a6cf4..6cb8f4d5f7 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -102,28 +102,28 @@ func (t *SecureTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) err // VerifyProof checks merkle proofs. The given proof must contain the value for // key in a trie with the given root hash. VerifyProof returns an error if the // proof contains invalid trie nodes or the wrong value. -func VerifyProof(rootHash common.Hash, key []byte, proofDb DatabaseReader) (value []byte, err error, nodes int) { +func VerifyProof(rootHash common.Hash, key []byte, proofDb DatabaseReader) (value []byte, nodes int, err error) { key = keybytesToHex(key) wantHash := rootHash for i := 0; ; i++ { buf, _ := proofDb.Get(wantHash[:]) if buf == nil { - return nil, fmt.Errorf("proof node %d (hash %064x) missing", i, wantHash), i + return nil, i, fmt.Errorf("proof node %d (hash %064x) missing", i, wantHash) } n, err := decodeNode(wantHash[:], buf, 0) if err != nil { - return nil, fmt.Errorf("bad proof node %d: %v", i, err), i + return nil, i, fmt.Errorf("bad proof node %d: %v", i, err) } keyrest, cld := get(n, key) switch cld := cld.(type) { case nil: // The trie doesn't contain the key. - return nil, nil, i + return nil, i, nil case hashNode: key = keyrest copy(wantHash[:], cld) case valueNode: - return cld, nil, i + 1 + return cld, i + 1, nil } } } diff --git a/trie/proof_test.go b/trie/proof_test.go index a3537787cc..dee6f7d859 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -40,7 +40,7 @@ func TestProof(t *testing.T) { if trie.Prove(kv.k, 0, proofs) != nil { t.Fatalf("missing key %x while constructing proof", kv.k) } - val, err, _ := VerifyProof(root, kv.k, proofs) + val, _, err := VerifyProof(root, kv.k, proofs) if err != nil { t.Fatalf("VerifyProof error for key %x: %v\nraw proof: %v", kv.k, err, proofs) } @@ -58,7 +58,7 @@ func TestOneElementProof(t *testing.T) { if len(proofs.Keys()) != 1 { t.Error("proof should have one element") } - val, err, _ := VerifyProof(trie.Hash(), []byte("k"), proofs) + val, _, err := VerifyProof(trie.Hash(), []byte("k"), proofs) if err != nil { t.Fatalf("VerifyProof error: %v\nproof hashes: %v", err, proofs.Keys()) } @@ -82,7 +82,7 @@ func TestVerifyBadProof(t *testing.T) { proofs.Delete(key) mutateByte(node) proofs.Put(crypto.Keccak256(node), node) - if _, err, _ := VerifyProof(root, kv.k, proofs); err == nil { + if _, _, err := VerifyProof(root, kv.k, proofs); err == nil { t.Fatalf("expected proof to fail for key %x", kv.k) } } @@ -131,7 +131,7 @@ func BenchmarkVerifyProof(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { im := i % len(keys) - if _, err, _ := VerifyProof(root, []byte(keys[im]), proofs[im]); err != nil { + if _, _, err := VerifyProof(root, []byte(keys[im]), proofs[im]); err != nil { b.Fatalf("key %x: %v", keys[im], err) } } diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 3881ee18a0..6a50cfd5a6 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -155,14 +155,19 @@ func (t *SecureTrie) Commit(onleaf LeafCallback) (root common.Hash, err error) { return t.trie.Commit(onleaf) } +// Hash returns the root hash of SecureTrie. It does not write to the +// database and can be used even if the trie doesn't have one. func (t *SecureTrie) Hash() common.Hash { return t.trie.Hash() } +// Root returns the root hash of SecureTrie. +// Deprecated: use Hash instead. func (t *SecureTrie) Root() []byte { return t.trie.Root() } +// Copy returns a copy of SecureTrie. func (t *SecureTrie) Copy() *SecureTrie { cpy := *t return &cpy From 09d44247f746b177e46754ba56ad7e9bc0bd8ef6 Mon Sep 17 00:00:00 2001 From: kiel barry Date: Tue, 22 May 2018 00:28:43 -0700 Subject: [PATCH 28/46] log: fixes for golint warnings (#16775) --- log/README.md | 2 +- log/doc.go | 6 +++--- log/format.go | 16 ++++++++-------- log/handler.go | 10 +++++----- log/logger.go | 5 +++-- 5 files changed, 20 insertions(+), 19 deletions(-) diff --git a/log/README.md b/log/README.md index 0951b21cb5..b4476577b6 100644 --- a/log/README.md +++ b/log/README.md @@ -45,7 +45,7 @@ srvlog.SetHandler(log.MultiHandler( log.StreamHandler(os.Stderr, log.LogfmtFormat()), log.LvlFilterHandler( log.LvlError, - log.Must.FileHandler("errors.json", log.JsonFormat())))) + log.Must.FileHandler("errors.json", log.JSONFormat())))) ``` Will result in output that looks like this: diff --git a/log/doc.go b/log/doc.go index 83ad8c54f6..bff2f4966a 100644 --- a/log/doc.go +++ b/log/doc.go @@ -86,7 +86,7 @@ from the rpc package in logfmt to standard out. The other prints records at Erro or above in JSON formatted output to the file /var/log/service.json handler := log.MultiHandler( - log.LvlFilterHandler(log.LvlError, log.Must.FileHandler("/var/log/service.json", log.JsonFormat())), + log.LvlFilterHandler(log.LvlError, log.Must.FileHandler("/var/log/service.json", log.JSONFormat())), log.MatchFilterHandler("pkg", "app/rpc" log.StdoutHandler()) ) @@ -304,8 +304,8 @@ For all Handler functions which can return an error, there is a version of that function which will return no error but panics on failure. They are all available on the Must object. For example: - log.Must.FileHandler("/path", log.JsonFormat) - log.Must.NetHandler("tcp", ":1234", log.JsonFormat) + log.Must.FileHandler("/path", log.JSONFormat) + log.Must.NetHandler("tcp", ":1234", log.JSONFormat) Inspiration and Credit diff --git a/log/format.go b/log/format.go index 0b07abb2ac..fb1ea1a7b4 100644 --- a/log/format.go +++ b/log/format.go @@ -196,16 +196,16 @@ func logfmt(buf *bytes.Buffer, ctx []interface{}, color int, term bool) { buf.WriteByte('\n') } -// JsonFormat formats log records as JSON objects separated by newlines. -// It is the equivalent of JsonFormatEx(false, true). -func JsonFormat() Format { - return JsonFormatEx(false, true) +// JSONFormat formats log records as JSON objects separated by newlines. +// It is the equivalent of JSONFormatEx(false, true). +func JSONFormat() Format { + return JSONFormatEx(false, true) } -// JsonFormatEx formats log records as JSON objects. If pretty is true, +// JSONFormatEx formats log records as JSON objects. If pretty is true, // records will be pretty-printed. If lineSeparated is true, records // will be logged with a new line between each record. -func JsonFormatEx(pretty, lineSeparated bool) Format { +func JSONFormatEx(pretty, lineSeparated bool) Format { jsonMarshal := json.Marshal if pretty { jsonMarshal = func(v interface{}) ([]byte, error) { @@ -225,7 +225,7 @@ func JsonFormatEx(pretty, lineSeparated bool) Format { if !ok { props[errorKey] = fmt.Sprintf("%+v is not a string key", r.Ctx[i]) } - props[k] = formatJsonValue(r.Ctx[i+1]) + props[k] = formatJSONValue(r.Ctx[i+1]) } b, err := jsonMarshal(props) @@ -270,7 +270,7 @@ func formatShared(value interface{}) (result interface{}) { } } -func formatJsonValue(value interface{}) interface{} { +func formatJSONValue(value interface{}) interface{} { value = formatShared(value) switch value.(type) { case int, int8, int16, int32, int64, float32, float64, uint, uint8, uint16, uint32, uint64, string: diff --git a/log/handler.go b/log/handler.go index 41d5718ddb..3c99114dcb 100644 --- a/log/handler.go +++ b/log/handler.go @@ -11,8 +11,8 @@ import ( "github.com/go-stack/stack" ) +// Handler defines where and how log records are written. // A Logger prints its log records by writing to a Handler. -// The Handler interface defines where and how log records are written. // Handlers are composable, providing you great flexibility in combining // them to achieve the logging structure that suits your applications. type Handler interface { @@ -193,7 +193,7 @@ func LvlFilterHandler(maxLvl Lvl, h Handler) Handler { }, h) } -// A MultiHandler dispatches any write to each of its handlers. +// MultiHandler dispatches any write to each of its handlers. // This is useful for writing different types of log information // to different locations. For example, to log to a file and // standard error: @@ -212,7 +212,7 @@ func MultiHandler(hs ...Handler) Handler { }) } -// A FailoverHandler writes all log records to the first handler +// FailoverHandler writes all log records to the first handler // specified, but will failover and write to the second handler if // the first handler has failed, and so on for all handlers specified. // For example you might want to log to a network socket, but failover @@ -220,7 +220,7 @@ func MultiHandler(hs ...Handler) Handler { // standard out if the file write fails: // // log.FailoverHandler( -// log.Must.NetHandler("tcp", ":9090", log.JsonFormat()), +// log.Must.NetHandler("tcp", ":9090", log.JSONFormat()), // log.Must.FileHandler("/var/log/app.log", log.LogfmtFormat()), // log.StdoutHandler) // @@ -336,7 +336,7 @@ func DiscardHandler() Handler { }) } -// The Must object provides the following Handler creation functions +// Must provides the following Handler creation functions // which instead of returning an error parameter only return a Handler // and panic on failure: FileHandler, NetHandler, SyslogHandler, SyslogNetHandler var Must muster diff --git a/log/logger.go b/log/logger.go index 15c83a9b25..a2fe6dc580 100644 --- a/log/logger.go +++ b/log/logger.go @@ -24,7 +24,7 @@ const ( LvlTrace ) -// Aligned returns a 5-character string containing the name of a Lvl. +// AlignedString returns a 5-character string containing the name of a Lvl. func (l Lvl) AlignedString() string { switch l { case LvlTrace: @@ -64,7 +64,7 @@ func (l Lvl) String() string { } } -// Returns the appropriate Lvl from a string name. +// LvlFromString returns the appropriate Lvl from a string name. // Useful for parsing command line args and configuration files. func LvlFromString(lvlString string) (Lvl, error) { switch lvlString { @@ -95,6 +95,7 @@ type Record struct { KeyNames RecordKeyNames } +// RecordKeyNames gets stored in a Record when the write function is executed. type RecordKeyNames struct { Time string Msg string From 9af364e42b2fbbacf2febf9c43068ddb8a058b79 Mon Sep 17 00:00:00 2001 From: kiel barry Date: Tue, 22 May 2018 00:29:41 -0700 Subject: [PATCH 29/46] node: all golint warnings fixed (#16773) * node: all golint warnings fixed * node: rm per peter * node: rm per peter --- node/api.go | 2 +- node/node_test.go | 4 ++-- node/utils_test.go | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/node/api.go b/node/api.go index a3b8bc0bb5..da9da5bd72 100644 --- a/node/api.go +++ b/node/api.go @@ -217,7 +217,7 @@ func (api *PrivateAdminAPI) StartWS(host *string, port *int, allowedOrigins *str return true, nil } -// StopRPC terminates an already running websocket RPC API endpoint. +// StopWS terminates an already running websocket RPC API endpoint. func (api *PrivateAdminAPI) StopWS() (bool, error) { api.node.lock.Lock() defer api.node.lock.Unlock() diff --git a/node/node_test.go b/node/node_test.go index 2880efa619..e51900bd1e 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -507,8 +507,8 @@ func TestAPIGather(t *testing.T) { } // Register a batch of services with some configured APIs calls := make(chan string, 1) - makeAPI := func(result string) *OneMethodApi { - return &OneMethodApi{fun: func() { calls <- result }} + makeAPI := func(result string) *OneMethodAPI { + return &OneMethodAPI{fun: func() { calls <- result }} } services := map[string]struct { APIs []rpc.API diff --git a/node/utils_test.go b/node/utils_test.go index 8eddce3ed7..9801b1ed45 100644 --- a/node/utils_test.go +++ b/node/utils_test.go @@ -121,12 +121,12 @@ func InstrumentedServiceMakerC(base ServiceConstructor) ServiceConstructor { return InstrumentingWrapperMaker(base, reflect.TypeOf(InstrumentedServiceC{})) } -// OneMethodApi is a single-method API handler to be returned by test services. -type OneMethodApi struct { +// OneMethodAPI is a single-method API handler to be returned by test services. +type OneMethodAPI struct { fun func() } -func (api *OneMethodApi) TheOneMethod() { +func (api *OneMethodAPI) TheOneMethod() { if api.fun != nil { api.fun() } From 6ce21a4744461fc35c05b1c5fcc92136761be747 Mon Sep 17 00:00:00 2001 From: gary rong Date: Tue, 22 May 2018 16:12:52 +0800 Subject: [PATCH 30/46] vendor, ethdb: print warning log if leveldb is performing compaction (#16766) * vendor: update leveldb package * ethdb: print warning log if db is performing compaction * ethdb: update annotation and log --- ethdb/database.go | 12 ++- .../github.com/syndtr/goleveldb/leveldb/db.go | 73 ++++++++++++++++++- .../syndtr/goleveldb/leveldb/db_write.go | 4 + vendor/vendor.json | 6 +- 4 files changed, 90 insertions(+), 5 deletions(-) diff --git a/ethdb/database.go b/ethdb/database.go index 001d8f0bb9..86dd210762 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -207,6 +207,7 @@ func (db *LDBDatabase) meter(refresh time.Duration) { delaystats [2]int64 lastWriteDelay time.Time lastWriteDelayN time.Time + lastWritePaused time.Time ) // Iterate ad infinitum and collect the stats @@ -267,8 +268,9 @@ func (db *LDBDatabase) meter(refresh time.Duration) { delayN int64 delayDuration string duration time.Duration + paused bool ) - if n, err := fmt.Sscanf(writedelay, "DelayN:%d Delay:%s", &delayN, &delayDuration); n != 2 || err != nil { + if n, err := fmt.Sscanf(writedelay, "DelayN:%d Delay:%s Paused:%t", &delayN, &delayDuration, &paused); n != 3 || err != nil { db.log.Error("Write delay statistic not found") return } @@ -301,6 +303,14 @@ func (db *LDBDatabase) meter(refresh time.Duration) { lastWriteDelay = time.Now() } } + // If a warning that db is performing compaction has been displayed, any subsequent + // warnings will be withheld for one minute not to overwhelm the user. + if paused && delayN-delaystats[0] == 0 && duration.Nanoseconds()-delaystats[1] == 0 && + time.Now().After(lastWritePaused.Add(writeDelayWarningThrottler)) { + db.log.Warn("Database compacting, degraded performance") + lastWritePaused = time.Now() + } + delaystats[0], delaystats[1] = delayN, duration.Nanoseconds() // Retrieve the database iostats. diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db.go b/vendor/github.com/syndtr/goleveldb/leveldb/db.go index 3655418ad3..e7ac065418 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/db.go @@ -35,6 +35,7 @@ type DB struct { // Stats. Need 64-bit alignment. cWriteDelay int64 // The cumulative duration of write delays cWriteDelayN int32 // The cumulative number of write delays + inWritePaused int32 // The indicator whether write operation is paused by compaction aliveSnaps, aliveIters int32 // Session. @@ -967,7 +968,8 @@ func (db *DB) GetProperty(name string) (value string, err error) { float64(db.s.stor.writes())/1048576.0) case p == "writedelay": writeDelayN, writeDelay := atomic.LoadInt32(&db.cWriteDelayN), time.Duration(atomic.LoadInt64(&db.cWriteDelay)) - value = fmt.Sprintf("DelayN:%d Delay:%s", writeDelayN, writeDelay) + paused := atomic.LoadInt32(&db.inWritePaused) == 1 + value = fmt.Sprintf("DelayN:%d Delay:%s Paused:%t", writeDelayN, writeDelay, paused) case p == "sstables": for level, tables := range v.levels { value += fmt.Sprintf("--- level %d ---\n", level) @@ -996,6 +998,75 @@ func (db *DB) GetProperty(name string) (value string, err error) { return } +// DBStats is database statistics. +type DBStats struct { + WriteDelayCount int32 + WriteDelayDuration time.Duration + WritePaused bool + + AliveSnapshots int32 + AliveIterators int32 + + IOWrite uint64 + IORead uint64 + + BlockCacheSize int + OpenedTablesCount int + + LevelSizes []int64 + LevelTablesCounts []int + LevelRead []int64 + LevelWrite []int64 + LevelDurations []time.Duration +} + +// Stats populates s with database statistics. +func (db *DB) Stats(s *DBStats) error { + err := db.ok() + if err != nil { + return err + } + + s.IORead = db.s.stor.reads() + s.IOWrite = db.s.stor.writes() + s.WriteDelayCount = atomic.LoadInt32(&db.cWriteDelayN) + s.WriteDelayDuration = time.Duration(atomic.LoadInt64(&db.cWriteDelay)) + s.WritePaused = atomic.LoadInt32(&db.inWritePaused) == 1 + + s.OpenedTablesCount = db.s.tops.cache.Size() + if db.s.tops.bcache != nil { + s.BlockCacheSize = db.s.tops.bcache.Size() + } else { + s.BlockCacheSize = 0 + } + + s.AliveIterators = atomic.LoadInt32(&db.aliveIters) + s.AliveSnapshots = atomic.LoadInt32(&db.aliveSnaps) + + s.LevelDurations = s.LevelDurations[:0] + s.LevelRead = s.LevelRead[:0] + s.LevelWrite = s.LevelWrite[:0] + s.LevelSizes = s.LevelSizes[:0] + s.LevelTablesCounts = s.LevelTablesCounts[:0] + + v := db.s.version() + defer v.release() + + for level, tables := range v.levels { + duration, read, write := db.compStats.getStat(level) + if len(tables) == 0 && duration == 0 { + continue + } + s.LevelDurations = append(s.LevelDurations, duration) + s.LevelRead = append(s.LevelRead, read) + s.LevelWrite = append(s.LevelWrite, write) + s.LevelSizes = append(s.LevelSizes, tables.size()) + s.LevelTablesCounts = append(s.LevelTablesCounts, len(tables)) + } + + return nil +} + // SizeOf calculates approximate sizes of the given key ranges. // The length of the returned sizes are equal with the length of the given // ranges. The returned sizes measure storage space usage, so if the user diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db_write.go b/vendor/github.com/syndtr/goleveldb/leveldb/db_write.go index 31f4bc5efc..db0c1bece1 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db_write.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/db_write.go @@ -89,7 +89,11 @@ func (db *DB) flush(n int) (mdb *memDB, mdbFree int, err error) { return false case tLen >= pauseTrigger: delayed = true + // Set the write paused flag explicitly. + atomic.StoreInt32(&db.inWritePaused, 1) err = db.compTriggerWait(db.tcompCmdC) + // Unset the write paused flag. + atomic.StoreInt32(&db.inWritePaused, 0) if err != nil { return false } diff --git a/vendor/vendor.json b/vendor/vendor.json index fdc7789364..dc46b182a9 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -418,10 +418,10 @@ "revisionTime": "2017-07-05T02:17:15Z" }, { - "checksumSHA1": "k13cCuMJO7+KhR8ZXx5oUqDKGQA=", + "checksumSHA1": "TJV50D0q8E3vtc90ibC+qOYdjrw=", "path": "github.com/syndtr/goleveldb/leveldb", - "revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a", - "revisionTime": "2018-05-02T07:23:49Z" + "revision": "59047f74db0d042c8d8dd8e30bb030bc774a7d7a", + "revisionTime": "2018-05-21T04:45:49Z" }, { "checksumSHA1": "EKIow7XkgNdWvR/982ffIZxKG8Y=", From fbf57d53e272c2d79d4d899bb94db824678de2d5 Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 23 May 2018 16:10:24 +0800 Subject: [PATCH 31/46] core/types: convert status type from uint to uint64 (#16784) --- core/types/gen_receipt_json.go | 10 ++++++---- core/types/receipt.go | 8 ++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/core/types/gen_receipt_json.go b/core/types/gen_receipt_json.go index c297adebb6..5c807a4ccb 100644 --- a/core/types/gen_receipt_json.go +++ b/core/types/gen_receipt_json.go @@ -12,10 +12,11 @@ import ( var _ = (*receiptMarshaling)(nil) +// MarshalJSON marshals as JSON. func (r Receipt) MarshalJSON() ([]byte, error) { type Receipt struct { PostState hexutil.Bytes `json:"root"` - Status hexutil.Uint `json:"status"` + Status hexutil.Uint64 `json:"status"` CumulativeGasUsed hexutil.Uint64 `json:"cumulativeGasUsed" gencodec:"required"` Bloom Bloom `json:"logsBloom" gencodec:"required"` Logs []*Log `json:"logs" gencodec:"required"` @@ -25,7 +26,7 @@ func (r Receipt) MarshalJSON() ([]byte, error) { } var enc Receipt enc.PostState = r.PostState - enc.Status = hexutil.Uint(r.Status) + enc.Status = hexutil.Uint64(r.Status) enc.CumulativeGasUsed = hexutil.Uint64(r.CumulativeGasUsed) enc.Bloom = r.Bloom enc.Logs = r.Logs @@ -35,10 +36,11 @@ func (r Receipt) MarshalJSON() ([]byte, error) { return json.Marshal(&enc) } +// UnmarshalJSON unmarshals from JSON. func (r *Receipt) UnmarshalJSON(input []byte) error { type Receipt struct { PostState *hexutil.Bytes `json:"root"` - Status *hexutil.Uint `json:"status"` + Status *hexutil.Uint64 `json:"status"` CumulativeGasUsed *hexutil.Uint64 `json:"cumulativeGasUsed" gencodec:"required"` Bloom *Bloom `json:"logsBloom" gencodec:"required"` Logs []*Log `json:"logs" gencodec:"required"` @@ -54,7 +56,7 @@ func (r *Receipt) UnmarshalJSON(input []byte) error { r.PostState = *dec.PostState } if dec.Status != nil { - r.Status = uint(*dec.Status) + r.Status = uint64(*dec.Status) } if dec.CumulativeGasUsed == nil { return errors.New("missing required field 'cumulativeGasUsed' for Receipt") diff --git a/core/types/receipt.go b/core/types/receipt.go index 613f03d50f..3d1fc95aab 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -36,17 +36,17 @@ var ( const ( // ReceiptStatusFailed is the status code of a transaction if execution failed. - ReceiptStatusFailed = uint(0) + ReceiptStatusFailed = uint64(0) // ReceiptStatusSuccessful is the status code of a transaction if execution succeeded. - ReceiptStatusSuccessful = uint(1) + ReceiptStatusSuccessful = uint64(1) ) // Receipt represents the results of a transaction. type Receipt struct { // Consensus fields PostState []byte `json:"root"` - Status uint `json:"status"` + Status uint64 `json:"status"` CumulativeGasUsed uint64 `json:"cumulativeGasUsed" gencodec:"required"` Bloom Bloom `json:"logsBloom" gencodec:"required"` Logs []*Log `json:"logs" gencodec:"required"` @@ -59,7 +59,7 @@ type Receipt struct { type receiptMarshaling struct { PostState hexutil.Bytes - Status hexutil.Uint + Status hexutil.Uint64 CumulativeGasUsed hexutil.Uint64 GasUsed hexutil.Uint64 } From c934c06cc1cf7d14928dce91f7053ef7ed746f92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 10 May 2018 12:49:27 +0300 Subject: [PATCH 32/46] trie: support proof generation from the iterator --- trie/iterator.go | 71 ++++++++++++++++++++----- trie/proof_test.go | 127 ++++++++++++++++++++++++++++++++------------- 2 files changed, 150 insertions(+), 48 deletions(-) diff --git a/trie/iterator.go b/trie/iterator.go index 64110c6d96..00b890eb89 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -22,6 +22,7 @@ import ( "errors" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rlp" ) // Iterator is a key-value trie iterator that traverses a Trie. @@ -55,31 +56,50 @@ func (it *Iterator) Next() bool { return false } +// Prove generates the Merkle proof for the leaf node the iterator is currently +// positioned on. +func (it *Iterator) Prove() [][]byte { + return it.nodeIt.LeafProof() +} + // NodeIterator is an iterator to traverse the trie pre-order. type NodeIterator interface { // Next moves the iterator to the next node. If the parameter is false, any child // nodes will be skipped. Next(bool) bool + // Error returns the error status of the iterator. Error() error // Hash returns the hash of the current node. Hash() common.Hash + // Parent returns the hash of the parent of the current node. The hash may be the one // grandparent if the immediate parent is an internal node with no hash. Parent() common.Hash + // Path returns the hex-encoded path to the current node. // Callers must not retain references to the return value after calling Next. // For leaf nodes, the last element of the path is the 'terminator symbol' 0x10. Path() []byte // Leaf returns true iff the current node is a leaf node. - // LeafBlob, LeafKey return the contents and key of the leaf node. These - // method panic if the iterator is not positioned at a leaf. - // Callers must not retain references to their return value after calling Next Leaf() bool - LeafBlob() []byte + + // LeafKey returns the key of the leaf. The method panics if the iterator is not + // positioned at a leaf. Callers must not retain references to the value after + // calling Next. LeafKey() []byte + + // LeafBlob returns the content of the leaf. The method panics if the iterator + // is not positioned at a leaf. Callers must not retain references to the value + // after calling Next. + LeafBlob() []byte + + // LeafProof returns the Merkle proof of the leaf. The method panics if the + // iterator is not positioned at a leaf. Callers must not retain references + // to the value after calling Next. + LeafProof() [][]byte } // nodeIteratorState represents the iteration state at one particular node of the @@ -139,6 +159,15 @@ func (it *nodeIterator) Leaf() bool { return hasTerm(it.path) } +func (it *nodeIterator) LeafKey() []byte { + if len(it.stack) > 0 { + if _, ok := it.stack[len(it.stack)-1].node.(valueNode); ok { + return hexToKeybytes(it.path) + } + } + panic("not at leaf") +} + func (it *nodeIterator) LeafBlob() []byte { if len(it.stack) > 0 { if node, ok := it.stack[len(it.stack)-1].node.(valueNode); ok { @@ -148,10 +177,22 @@ func (it *nodeIterator) LeafBlob() []byte { panic("not at leaf") } -func (it *nodeIterator) LeafKey() []byte { +func (it *nodeIterator) LeafProof() [][]byte { if len(it.stack) > 0 { if _, ok := it.stack[len(it.stack)-1].node.(valueNode); ok { - return hexToKeybytes(it.path) + hasher := newHasher(0, 0, nil) + proofs := make([][]byte, 0, len(it.stack)) + + for i, item := range it.stack[:len(it.stack)-1] { + // Gather nodes that end up as hash nodes (or the root) + node, _, _ := hasher.hashChildren(item.node, nil) + hashed, _ := hasher.store(node, nil, false) + if _, ok := hashed.(hashNode); ok || i == 0 { + enc, _ := rlp.EncodeToBytes(node) + proofs = append(proofs, enc) + } + } + return proofs } } panic("not at leaf") @@ -361,12 +402,16 @@ func (it *differenceIterator) Leaf() bool { return it.b.Leaf() } +func (it *differenceIterator) LeafKey() []byte { + return it.b.LeafKey() +} + func (it *differenceIterator) LeafBlob() []byte { return it.b.LeafBlob() } -func (it *differenceIterator) LeafKey() []byte { - return it.b.LeafKey() +func (it *differenceIterator) LeafProof() [][]byte { + return it.b.LeafProof() } func (it *differenceIterator) Path() []byte { @@ -464,12 +509,16 @@ func (it *unionIterator) Leaf() bool { return (*it.items)[0].Leaf() } +func (it *unionIterator) LeafKey() []byte { + return (*it.items)[0].LeafKey() +} + func (it *unionIterator) LeafBlob() []byte { return (*it.items)[0].LeafBlob() } -func (it *unionIterator) LeafKey() []byte { - return (*it.items)[0].LeafKey() +func (it *unionIterator) LeafProof() [][]byte { + return (*it.items)[0].LeafProof() } func (it *unionIterator) Path() []byte { @@ -509,12 +558,10 @@ func (it *unionIterator) Next(descend bool) bool { heap.Push(it.items, skipped) } } - if least.Next(descend) { it.count++ heap.Push(it.items, least) } - return len(*it.items) > 0 } diff --git a/trie/proof_test.go b/trie/proof_test.go index dee6f7d859..996f874781 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -32,20 +32,46 @@ func init() { mrand.Seed(time.Now().Unix()) } +// makeProvers creates Merkle trie provers based on different implementations to +// test all variations. +func makeProvers(trie *Trie) []func(key []byte) *ethdb.MemDatabase { + var provers []func(key []byte) *ethdb.MemDatabase + + // Create a direct trie based Merkle prover + provers = append(provers, func(key []byte) *ethdb.MemDatabase { + proof := ethdb.NewMemDatabase() + trie.Prove(key, 0, proof) + return proof + }) + // Create a leaf iterator based Merkle prover + provers = append(provers, func(key []byte) *ethdb.MemDatabase { + proof := ethdb.NewMemDatabase() + if it := NewIterator(trie.NodeIterator(key)); it.Next() && bytes.Equal(key, it.Key) { + for _, p := range it.Prove() { + proof.Put(crypto.Keccak256(p), p) + } + } + return proof + }) + return provers +} + func TestProof(t *testing.T) { trie, vals := randomTrie(500) root := trie.Hash() - for _, kv := range vals { - proofs := ethdb.NewMemDatabase() - if trie.Prove(kv.k, 0, proofs) != nil { - t.Fatalf("missing key %x while constructing proof", kv.k) - } - val, _, err := VerifyProof(root, kv.k, proofs) - if err != nil { - t.Fatalf("VerifyProof error for key %x: %v\nraw proof: %v", kv.k, err, proofs) - } - if !bytes.Equal(val, kv.v) { - t.Fatalf("VerifyProof returned wrong value for key %x: got %x, want %x", kv.k, val, kv.v) + for i, prover := range makeProvers(trie) { + for _, kv := range vals { + proof := prover(kv.k) + if proof == nil { + t.Fatalf("prover %d: missing key %x while constructing proof", i, kv.k) + } + val, _, err := VerifyProof(root, kv.k, proof) + if err != nil { + t.Fatalf("prover %d: failed to verify proof for key %x: %v\nraw proof: %x", i, kv.k, err, proof) + } + if !bytes.Equal(val, kv.v) { + t.Fatalf("prover %d: verified value mismatch for key %x: have %x, want %x", i, kv.k, val, kv.v) + } } } } @@ -53,37 +79,66 @@ func TestProof(t *testing.T) { func TestOneElementProof(t *testing.T) { trie := new(Trie) updateString(trie, "k", "v") - proofs := ethdb.NewMemDatabase() - trie.Prove([]byte("k"), 0, proofs) - if len(proofs.Keys()) != 1 { - t.Error("proof should have one element") - } - val, _, err := VerifyProof(trie.Hash(), []byte("k"), proofs) - if err != nil { - t.Fatalf("VerifyProof error: %v\nproof hashes: %v", err, proofs.Keys()) - } - if !bytes.Equal(val, []byte("v")) { - t.Fatalf("VerifyProof returned wrong value: got %x, want 'k'", val) + for i, prover := range makeProvers(trie) { + proof := prover([]byte("k")) + if proof == nil { + t.Fatalf("prover %d: nil proof", i) + } + if proof.Len() != 1 { + t.Errorf("prover %d: proof should have one element", i) + } + val, _, err := VerifyProof(trie.Hash(), []byte("k"), proof) + if err != nil { + t.Fatalf("prover %d: failed to verify proof: %v\nraw proof: %x", i, err, proof) + } + if !bytes.Equal(val, []byte("v")) { + t.Fatalf("prover %d: verified value mismatch: have %x, want 'k'", i, val) + } } } -func TestVerifyBadProof(t *testing.T) { +func TestBadProof(t *testing.T) { trie, vals := randomTrie(800) root := trie.Hash() - for _, kv := range vals { - proofs := ethdb.NewMemDatabase() - trie.Prove(kv.k, 0, proofs) - if len(proofs.Keys()) == 0 { - t.Fatal("zero length proof") + for i, prover := range makeProvers(trie) { + for _, kv := range vals { + proof := prover(kv.k) + if proof == nil { + t.Fatalf("prover %d: nil proof", i) + } + key := proof.Keys()[mrand.Intn(proof.Len())] + val, _ := proof.Get(key) + proof.Delete(key) + + mutateByte(val) + proof.Put(crypto.Keccak256(val), val) + + if _, _, err := VerifyProof(root, kv.k, proof); err == nil { + t.Fatalf("prover %d: expected proof to fail for key %x", i, kv.k) + } } - keys := proofs.Keys() - key := keys[mrand.Intn(len(keys))] - node, _ := proofs.Get(key) - proofs.Delete(key) - mutateByte(node) - proofs.Put(crypto.Keccak256(node), node) - if _, _, err := VerifyProof(root, kv.k, proofs); err == nil { - t.Fatalf("expected proof to fail for key %x", kv.k) + } +} + +// Tests that missing keys can also be proven. The test explicitly uses a single +// entry trie and checks for missing keys both before and after the single entry. +func TestMissingKeyProof(t *testing.T) { + trie := new(Trie) + updateString(trie, "k", "v") + + for i, key := range []string{"a", "j", "l", "z"} { + proof := ethdb.NewMemDatabase() + trie.Prove([]byte(key), 0, proof) + + if proof.Len() != 1 { + t.Errorf("test %d: proof should have one element", i) + } + val, _, err := VerifyProof(trie.Hash(), []byte(key), proof) + if err != nil { + t.Fatalf("test %d: failed to verify proof: %v\nraw proof: %x", i, err, proof) + } + if val != nil { + t.Fatalf("test %d: verified value mismatch: have %x, want nil", i, val) } } } From be22ee8ddac890044ca66500f4f8b32c635e3d1f Mon Sep 17 00:00:00 2001 From: Abel Nieto Date: Wed, 23 May 2018 14:02:10 +0200 Subject: [PATCH 33/46] core/vm: fix typo in instructions.go (#16788) --- core/vm/instructions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 0689ee39cf..3a67e1865f 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -850,7 +850,7 @@ func makePush(size uint64, pushByteSize int) executionFunc { } } -// make push instruction function +// make dup instruction function func makeDup(size int64) executionFunc { return func(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { stack.dup(evm.interpreter.intPool, int(size)) From 55b579e02ce6f374bf81061269eabde0d82ae567 Mon Sep 17 00:00:00 2001 From: Ryan Schneider Date: Wed, 23 May 2018 08:55:42 -0400 Subject: [PATCH 34/46] core: use a wrapped map to remove contention in `TxPool.Get`. (#16670) * core: use a wrapped `map` and `sync.RWMutex` for `TxPool.all` to remove contention in `TxPool.Get`. * core: Remove redundant `txLookup.Find` and improve comments on txLookup methods. --- core/tx_list.go | 21 +++---- core/tx_pool.go | 130 ++++++++++++++++++++++++++++++++----------- core/tx_pool_test.go | 50 ++++++++--------- 3 files changed, 132 insertions(+), 69 deletions(-) diff --git a/core/tx_list.go b/core/tx_list.go index ea6ee7019f..287dda4c33 100644 --- a/core/tx_list.go +++ b/core/tx_list.go @@ -397,13 +397,13 @@ func (h *priceHeap) Pop() interface{} { // txPricedList is a price-sorted heap to allow operating on transactions pool // contents in a price-incrementing way. type txPricedList struct { - all *map[common.Hash]*types.Transaction // Pointer to the map of all transactions - items *priceHeap // Heap of prices of all the stored transactions - stales int // Number of stale price points to (re-heap trigger) + all *txLookup // Pointer to the map of all transactions + items *priceHeap // Heap of prices of all the stored transactions + stales int // Number of stale price points to (re-heap trigger) } // newTxPricedList creates a new price-sorted transaction heap. -func newTxPricedList(all *map[common.Hash]*types.Transaction) *txPricedList { +func newTxPricedList(all *txLookup) *txPricedList { return &txPricedList{ all: all, items: new(priceHeap), @@ -425,12 +425,13 @@ func (l *txPricedList) Removed() { return } // Seems we've reached a critical number of stale transactions, reheap - reheap := make(priceHeap, 0, len(*l.all)) + reheap := make(priceHeap, 0, l.all.Count()) l.stales, l.items = 0, &reheap - for _, tx := range *l.all { + l.all.Range(func(hash common.Hash, tx *types.Transaction) bool { *l.items = append(*l.items, tx) - } + return true + }) heap.Init(l.items) } @@ -443,7 +444,7 @@ func (l *txPricedList) Cap(threshold *big.Int, local *accountSet) types.Transact for len(*l.items) > 0 { // Discard stale transactions if found during cleanup tx := heap.Pop(l.items).(*types.Transaction) - if _, ok := (*l.all)[tx.Hash()]; !ok { + if l.all.Get(tx.Hash()) == nil { l.stales-- continue } @@ -475,7 +476,7 @@ func (l *txPricedList) Underpriced(tx *types.Transaction, local *accountSet) boo // Discard stale price points if found at the heap start for len(*l.items) > 0 { head := []*types.Transaction(*l.items)[0] - if _, ok := (*l.all)[head.Hash()]; !ok { + if l.all.Get(head.Hash()) == nil { l.stales-- heap.Pop(l.items) continue @@ -500,7 +501,7 @@ func (l *txPricedList) Discard(count int, local *accountSet) types.Transactions for len(*l.items) > 0 && count > 0 { // Discard stale transactions if found during cleanup tx := heap.Pop(l.items).(*types.Transaction) - if _, ok := (*l.all)[tx.Hash()]; !ok { + if l.all.Get(tx.Hash()) == nil { l.stales-- continue } diff --git a/core/tx_pool.go b/core/tx_pool.go index f89e11441e..1c9516b1b7 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -200,11 +200,11 @@ type TxPool struct { locals *accountSet // Set of local transaction to exempt from eviction rules journal *txJournal // Journal of local transaction to back up to disk - pending map[common.Address]*txList // All currently processable transactions - queue map[common.Address]*txList // Queued but non-processable transactions - beats map[common.Address]time.Time // Last heartbeat from each known account - all map[common.Hash]*types.Transaction // All transactions to allow lookups - priced *txPricedList // All transactions sorted by price + pending map[common.Address]*txList // All currently processable transactions + queue map[common.Address]*txList // Queued but non-processable transactions + beats map[common.Address]time.Time // Last heartbeat from each known account + all *txLookup // All transactions to allow lookups + priced *txPricedList // All transactions sorted by price wg sync.WaitGroup // for shutdown sync @@ -226,12 +226,12 @@ func NewTxPool(config TxPoolConfig, chainconfig *params.ChainConfig, chain block pending: make(map[common.Address]*txList), queue: make(map[common.Address]*txList), beats: make(map[common.Address]time.Time), - all: make(map[common.Hash]*types.Transaction), + all: newTxLookup(), chainHeadCh: make(chan ChainHeadEvent, chainHeadChanSize), gasPrice: new(big.Int).SetUint64(config.PriceLimit), } pool.locals = newAccountSet(pool.signer) - pool.priced = newTxPricedList(&pool.all) + pool.priced = newTxPricedList(pool.all) pool.reset(nil, chain.CurrentBlock().Header()) // If local transactions and journaling is enabled, load from disk @@ -605,7 +605,7 @@ func (pool *TxPool) validateTx(tx *types.Transaction, local bool) error { func (pool *TxPool) add(tx *types.Transaction, local bool) (bool, error) { // If the transaction is already known, discard it hash := tx.Hash() - if pool.all[hash] != nil { + if pool.all.Get(hash) != nil { log.Trace("Discarding already known transaction", "hash", hash) return false, fmt.Errorf("known transaction: %x", hash) } @@ -616,7 +616,7 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (bool, error) { return false, err } // If the transaction pool is full, discard underpriced transactions - if uint64(len(pool.all)) >= pool.config.GlobalSlots+pool.config.GlobalQueue { + if uint64(pool.all.Count()) >= pool.config.GlobalSlots+pool.config.GlobalQueue { // If the new transaction is underpriced, don't accept it if !local && pool.priced.Underpriced(tx, pool.locals) { log.Trace("Discarding underpriced transaction", "hash", hash, "price", tx.GasPrice()) @@ -624,7 +624,7 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (bool, error) { return false, ErrUnderpriced } // New transaction is better than our worse ones, make room for it - drop := pool.priced.Discard(len(pool.all)-int(pool.config.GlobalSlots+pool.config.GlobalQueue-1), pool.locals) + drop := pool.priced.Discard(pool.all.Count()-int(pool.config.GlobalSlots+pool.config.GlobalQueue-1), pool.locals) for _, tx := range drop { log.Trace("Discarding freshly underpriced transaction", "hash", tx.Hash(), "price", tx.GasPrice()) underpricedTxCounter.Inc(1) @@ -642,11 +642,11 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (bool, error) { } // New transaction is better, replace old one if old != nil { - delete(pool.all, old.Hash()) + pool.all.Remove(old.Hash()) pool.priced.Removed() pendingReplaceCounter.Inc(1) } - pool.all[tx.Hash()] = tx + pool.all.Add(tx) pool.priced.Put(tx) pool.journalTx(from, tx) @@ -689,12 +689,12 @@ func (pool *TxPool) enqueueTx(hash common.Hash, tx *types.Transaction) (bool, er } // Discard any previous transaction and mark this if old != nil { - delete(pool.all, old.Hash()) + pool.all.Remove(old.Hash()) pool.priced.Removed() queuedReplaceCounter.Inc(1) } - if pool.all[hash] == nil { - pool.all[hash] = tx + if pool.all.Get(hash) == nil { + pool.all.Add(tx) pool.priced.Put(tx) } return old != nil, nil @@ -726,7 +726,7 @@ func (pool *TxPool) promoteTx(addr common.Address, hash common.Hash, tx *types.T inserted, old := list.Add(tx, pool.config.PriceBump) if !inserted { // An older transaction was better, discard this - delete(pool.all, hash) + pool.all.Remove(hash) pool.priced.Removed() pendingDiscardCounter.Inc(1) @@ -734,14 +734,14 @@ func (pool *TxPool) promoteTx(addr common.Address, hash common.Hash, tx *types.T } // Otherwise discard any previous transaction and mark this if old != nil { - delete(pool.all, old.Hash()) + pool.all.Remove(old.Hash()) pool.priced.Removed() pendingReplaceCounter.Inc(1) } // Failsafe to work around direct pending inserts (tests) - if pool.all[hash] == nil { - pool.all[hash] = tx + if pool.all.Get(hash) == nil { + pool.all.Add(tx) pool.priced.Put(tx) } // Set the potentially new pending nonce and notify any subsystems of the new tx @@ -840,7 +840,7 @@ func (pool *TxPool) Status(hashes []common.Hash) []TxStatus { status := make([]TxStatus, len(hashes)) for i, hash := range hashes { - if tx := pool.all[hash]; tx != nil { + if tx := pool.all.Get(hash); tx != nil { from, _ := types.Sender(pool.signer, tx) // already validated if pool.pending[from] != nil && pool.pending[from].txs.items[tx.Nonce()] != nil { status[i] = TxStatusPending @@ -855,24 +855,21 @@ func (pool *TxPool) Status(hashes []common.Hash) []TxStatus { // Get returns a transaction if it is contained in the pool // and nil otherwise. func (pool *TxPool) Get(hash common.Hash) *types.Transaction { - pool.mu.RLock() - defer pool.mu.RUnlock() - - return pool.all[hash] + return pool.all.Get(hash) } // removeTx removes a single transaction from the queue, moving all subsequent // transactions back to the future queue. func (pool *TxPool) removeTx(hash common.Hash, outofbound bool) { // Fetch the transaction we wish to delete - tx, ok := pool.all[hash] - if !ok { + tx := pool.all.Get(hash) + if tx == nil { return } addr, _ := types.Sender(pool.signer, tx) // already validated during insertion // Remove it from the list of known transactions - delete(pool.all, hash) + pool.all.Remove(hash) if outofbound { pool.priced.Removed() } @@ -928,7 +925,7 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) { for _, tx := range list.Forward(pool.currentState.GetNonce(addr)) { hash := tx.Hash() log.Trace("Removed old queued transaction", "hash", hash) - delete(pool.all, hash) + pool.all.Remove(hash) pool.priced.Removed() } // Drop all transactions that are too costly (low balance or out of gas) @@ -936,7 +933,7 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) { for _, tx := range drops { hash := tx.Hash() log.Trace("Removed unpayable queued transaction", "hash", hash) - delete(pool.all, hash) + pool.all.Remove(hash) pool.priced.Removed() queuedNofundsCounter.Inc(1) } @@ -952,7 +949,7 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) { if !pool.locals.contains(addr) { for _, tx := range list.Cap(int(pool.config.AccountQueue)) { hash := tx.Hash() - delete(pool.all, hash) + pool.all.Remove(hash) pool.priced.Removed() queuedRateLimitCounter.Inc(1) log.Trace("Removed cap-exceeding queued transaction", "hash", hash) @@ -1001,7 +998,7 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) { for _, tx := range list.Cap(list.Len() - 1) { // Drop the transaction from the global pools too hash := tx.Hash() - delete(pool.all, hash) + pool.all.Remove(hash) pool.priced.Removed() // Update the account nonce to the dropped transaction @@ -1023,7 +1020,7 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) { for _, tx := range list.Cap(list.Len() - 1) { // Drop the transaction from the global pools too hash := tx.Hash() - delete(pool.all, hash) + pool.all.Remove(hash) pool.priced.Removed() // Update the account nonce to the dropped transaction @@ -1092,7 +1089,7 @@ func (pool *TxPool) demoteUnexecutables() { for _, tx := range list.Forward(nonce) { hash := tx.Hash() log.Trace("Removed old pending transaction", "hash", hash) - delete(pool.all, hash) + pool.all.Remove(hash) pool.priced.Removed() } // Drop all transactions that are too costly (low balance or out of gas), and queue any invalids back for later @@ -1100,7 +1097,7 @@ func (pool *TxPool) demoteUnexecutables() { for _, tx := range drops { hash := tx.Hash() log.Trace("Removed unpayable pending transaction", "hash", hash) - delete(pool.all, hash) + pool.all.Remove(hash) pool.priced.Removed() pendingNofundsCounter.Inc(1) } @@ -1172,3 +1169,68 @@ func (as *accountSet) containsTx(tx *types.Transaction) bool { func (as *accountSet) add(addr common.Address) { as.accounts[addr] = struct{}{} } + +// txLookup is used internally by TxPool to track transactions while allowing lookup without +// mutex contention. +// +// Note, although this type is properly protected against concurrent access, it +// is **not** a type that should ever be mutated or even exposed outside of the +// transaction pool, since its internal state is tightly coupled with the pools +// internal mechanisms. The sole purpose of the type is to permit out-of-bound +// peeking into the pool in TxPool.Get without having to acquire the widely scoped +// TxPool.mu mutex. +type txLookup struct { + all map[common.Hash]*types.Transaction + lock sync.RWMutex +} + +// newTxLookup returns a new txLookup structure. +func newTxLookup() *txLookup { + return &txLookup{ + all: make(map[common.Hash]*types.Transaction), + } +} + +// Range calls f on each key and value present in the map. +func (t *txLookup) Range(f func(hash common.Hash, tx *types.Transaction) bool) { + t.lock.RLock() + defer t.lock.RUnlock() + + for key, value := range t.all { + if !f(key, value) { + break + } + } +} + +// Get returns a transaction if it exists in the lookup, or nil if not found. +func (t *txLookup) Get(hash common.Hash) *types.Transaction { + t.lock.RLock() + defer t.lock.RUnlock() + + return t.all[hash] +} + +// Count returns the current number of items in the lookup. +func (t *txLookup) Count() int { + t.lock.RLock() + defer t.lock.RUnlock() + + return len(t.all) +} + +// Add adds a transaction to the lookup. +func (t *txLookup) Add(tx *types.Transaction) { + t.lock.Lock() + defer t.lock.Unlock() + + t.all[tx.Hash()] = tx +} + +// Remove removes a transaction from the lookup. +func (t *txLookup) Remove(hash common.Hash) { + t.lock.Lock() + defer t.lock.Unlock() + + delete(t.all, hash) +} diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index 25993c258b..5a59205442 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -94,7 +94,7 @@ func validateTxPoolInternals(pool *TxPool) error { // Ensure the total transaction set is consistent with pending + queued pending, queued := pool.stats() - if total := len(pool.all); total != pending+queued { + if total := pool.all.Count(); total != pending+queued { return fmt.Errorf("total transaction count %d != %d pending + %d queued", total, pending, queued) } if priced := pool.priced.items.Len() - pool.priced.stales; priced != pending+queued { @@ -401,8 +401,8 @@ func TestTransactionDoubleNonce(t *testing.T) { t.Errorf("transaction mismatch: have %x, want %x", tx.Hash(), tx2.Hash()) } // Ensure the total transaction count is correct - if len(pool.all) != 1 { - t.Error("expected 1 total transactions, got", len(pool.all)) + if pool.all.Count() != 1 { + t.Error("expected 1 total transactions, got", pool.all.Count()) } } @@ -424,8 +424,8 @@ func TestTransactionMissingNonce(t *testing.T) { if pool.queue[addr].Len() != 1 { t.Error("expected 1 queued transaction, got", pool.queue[addr].Len()) } - if len(pool.all) != 1 { - t.Error("expected 1 total transactions, got", len(pool.all)) + if pool.all.Count() != 1 { + t.Error("expected 1 total transactions, got", pool.all.Count()) } } @@ -488,8 +488,8 @@ func TestTransactionDropping(t *testing.T) { if pool.queue[account].Len() != 3 { t.Errorf("queued transaction mismatch: have %d, want %d", pool.queue[account].Len(), 3) } - if len(pool.all) != 6 { - t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), 6) + if pool.all.Count() != 6 { + t.Errorf("total transaction mismatch: have %d, want %d", pool.all.Count(), 6) } pool.lockedReset(nil, nil) if pool.pending[account].Len() != 3 { @@ -498,8 +498,8 @@ func TestTransactionDropping(t *testing.T) { if pool.queue[account].Len() != 3 { t.Errorf("queued transaction mismatch: have %d, want %d", pool.queue[account].Len(), 3) } - if len(pool.all) != 6 { - t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), 6) + if pool.all.Count() != 6 { + t.Errorf("total transaction mismatch: have %d, want %d", pool.all.Count(), 6) } // Reduce the balance of the account, and check that invalidated transactions are dropped pool.currentState.AddBalance(account, big.NewInt(-650)) @@ -523,8 +523,8 @@ func TestTransactionDropping(t *testing.T) { if _, ok := pool.queue[account].txs.items[tx12.Nonce()]; ok { t.Errorf("out-of-fund queued transaction present: %v", tx11) } - if len(pool.all) != 4 { - t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), 4) + if pool.all.Count() != 4 { + t.Errorf("total transaction mismatch: have %d, want %d", pool.all.Count(), 4) } // Reduce the block gas limit, check that invalidated transactions are dropped pool.chain.(*testBlockChain).gasLimit = 100 @@ -542,8 +542,8 @@ func TestTransactionDropping(t *testing.T) { if _, ok := pool.queue[account].txs.items[tx11.Nonce()]; ok { t.Errorf("over-gased queued transaction present: %v", tx11) } - if len(pool.all) != 2 { - t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), 2) + if pool.all.Count() != 2 { + t.Errorf("total transaction mismatch: have %d, want %d", pool.all.Count(), 2) } } @@ -596,8 +596,8 @@ func TestTransactionPostponing(t *testing.T) { if len(pool.queue) != 0 { t.Errorf("queued accounts mismatch: have %d, want %d", len(pool.queue), 0) } - if len(pool.all) != len(txs) { - t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), len(txs)) + if pool.all.Count() != len(txs) { + t.Errorf("total transaction mismatch: have %d, want %d", pool.all.Count(), len(txs)) } pool.lockedReset(nil, nil) if pending := pool.pending[accs[0]].Len() + pool.pending[accs[1]].Len(); pending != len(txs) { @@ -606,8 +606,8 @@ func TestTransactionPostponing(t *testing.T) { if len(pool.queue) != 0 { t.Errorf("queued accounts mismatch: have %d, want %d", len(pool.queue), 0) } - if len(pool.all) != len(txs) { - t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), len(txs)) + if pool.all.Count() != len(txs) { + t.Errorf("total transaction mismatch: have %d, want %d", pool.all.Count(), len(txs)) } // Reduce the balance of the account, and check that transactions are reorganised for _, addr := range accs { @@ -656,8 +656,8 @@ func TestTransactionPostponing(t *testing.T) { } } } - if len(pool.all) != len(txs)/2 { - t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), len(txs)/2) + if pool.all.Count() != len(txs)/2 { + t.Errorf("total transaction mismatch: have %d, want %d", pool.all.Count(), len(txs)/2) } } @@ -748,8 +748,8 @@ func TestTransactionQueueAccountLimiting(t *testing.T) { } } } - if len(pool.all) != int(testTxPoolConfig.AccountQueue) { - t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), testTxPoolConfig.AccountQueue) + if pool.all.Count() != int(testTxPoolConfig.AccountQueue) { + t.Errorf("total transaction mismatch: have %d, want %d", pool.all.Count(), testTxPoolConfig.AccountQueue) } } @@ -942,8 +942,8 @@ func TestTransactionPendingLimiting(t *testing.T) { t.Errorf("tx %d: queue size mismatch: have %d, want %d", i, pool.queue[account].Len(), 0) } } - if len(pool.all) != int(testTxPoolConfig.AccountQueue+5) { - t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), testTxPoolConfig.AccountQueue+5) + if pool.all.Count() != int(testTxPoolConfig.AccountQueue+5) { + t.Errorf("total transaction mismatch: have %d, want %d", pool.all.Count(), testTxPoolConfig.AccountQueue+5) } if err := validateEvents(events, int(testTxPoolConfig.AccountQueue+5)); err != nil { t.Fatalf("event firing failed: %v", err) @@ -993,8 +993,8 @@ func testTransactionLimitingEquivalency(t *testing.T, origin uint64) { if len(pool1.queue) != len(pool2.queue) { t.Errorf("queued transaction count mismatch: one-by-one algo: %d, batch algo: %d", len(pool1.queue), len(pool2.queue)) } - if len(pool1.all) != len(pool2.all) { - t.Errorf("total transaction count mismatch: one-by-one algo %d, batch algo %d", len(pool1.all), len(pool2.all)) + if pool1.all.Count() != pool2.all.Count() { + t.Errorf("total transaction count mismatch: one-by-one algo %d, batch algo %d", pool1.all.Count(), pool2.all.Count()) } if err := validateTxPoolInternals(pool1); err != nil { t.Errorf("pool 1 internal state corrupted: %v", err) From d31802312a7ae1ed816daf8fe674efd42493adb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 24 May 2018 13:46:45 +0300 Subject: [PATCH 35/46] trie: cleaner logic, one less func call --- trie/hasher.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trie/hasher.go b/trie/hasher.go index 2fc44787ac..ff61e70922 100644 --- a/trie/hasher.go +++ b/trie/hasher.go @@ -196,12 +196,12 @@ func (h *hasher) store(n node, db *Database, force bool) (node, error) { if h.onleaf != nil { switch n := n.(type) { case *shortNode: - if child, ok := n.Val.(valueNode); ok { + if child, ok := n.Val.(valueNode); ok && child != nil { h.onleaf(child, hash) } case *fullNode: for i := 0; i < 16; i++ { - if child, ok := n.Children[i].(valueNode); ok { + if child, ok := n.Children[i].(valueNode); ok && child != nil { h.onleaf(child, hash) } } From d6ed2f67a8cdb07ce7b4eaea93452afb0cdbaa23 Mon Sep 17 00:00:00 2001 From: Wenbiao Zheng Date: Thu, 24 May 2018 20:55:20 +0800 Subject: [PATCH 36/46] eth, node, trie: fix minor typos (#16802) --- eth/backend.go | 10 +++++----- node/doc.go | 4 ++-- trie/trie.go | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/eth/backend.go b/eth/backend.go index ea70e3826c..e07d5efc9a 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -215,14 +215,14 @@ func CreateConsensusEngine(ctx *node.ServiceContext, config *ethash.Config, chai return clique.New(chainConfig.Clique, db) } // Otherwise assume proof-of-work - switch { - case config.PowMode == ethash.ModeFake: + switch config.PowMode { + case ethash.ModeFake: log.Warn("Ethash used in fake mode") return ethash.NewFaker() - case config.PowMode == ethash.ModeTest: + case ethash.ModeTest: log.Warn("Ethash used in test mode") return ethash.NewTester() - case config.PowMode == ethash.ModeShared: + case ethash.ModeShared: log.Warn("Ethash used in shared mode") return ethash.NewShared() default: @@ -239,7 +239,7 @@ func CreateConsensusEngine(ctx *node.ServiceContext, config *ethash.Config, chai } } -// APIs returns the collection of RPC services the ethereum package offers. +// APIs return the collection of RPC services the ethereum package offers. // NOTE, some of these services probably need to be moved to somewhere else. func (s *Ethereum) APIs() []rpc.API { apis := ethapi.GetAPIs(s.APIBackend) diff --git a/node/doc.go b/node/doc.go index d9688e0a12..41a88c19a1 100644 --- a/node/doc.go +++ b/node/doc.go @@ -59,7 +59,7 @@ using the same data directory will store this information in different subdirect the data directory. LevelDB databases are also stored within the instance subdirectory. If multiple node -instances use the same data directory, openening the databases with identical names will +instances use the same data directory, opening the databases with identical names will create one database for each instance. The account key store is shared among all node instances using the same data directory @@ -84,7 +84,7 @@ directory. Mode instance A opens the database "db", node instance B opens the da static-nodes.json -- devp2p static node list of instance B db/ -- LevelDB content for "db" db-2/ -- LevelDB content for "db-2" - B.ipc -- JSON-RPC UNIX domain socket endpoint of instance A + B.ipc -- JSON-RPC UNIX domain socket endpoint of instance B keystore/ -- account key store, used by both instances */ package node diff --git a/trie/trie.go b/trie/trie.go index 31a404e3a0..30543c5496 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -101,7 +101,7 @@ func New(root common.Hash, db *Database) (*Trie, error) { db: db, originalRoot: root, } - if (root != common.Hash{}) && root != emptyRoot { + if root != (common.Hash{}) && root != emptyRoot { rootnode, err := trie.resolveHash(root[:], nil) if err != nil { return nil, err From ff9b14617e73c07dfa028bc477fe86c1e04d80c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 28 May 2018 12:58:22 +0300 Subject: [PATCH 37/46] params: release go-ethereum v1.8.9 --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 1e8c43bf81..50653fc741 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 8 // Minor version component of the current release - VersionPatch = 9 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 8 // Minor version component of the current release + VersionPatch = 9 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From ccc0debb63124ee99906c2cfff6125de30e8c62f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 28 May 2018 13:01:20 +0300 Subject: [PATCH 38/46] VERSION, params: begin 1.8.10 release cycle --- VERSION | 2 +- params/version.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/VERSION b/VERSION index 53ed4ba51e..d9a03a9e5a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.8.9 +1.8.10 diff --git a/params/version.go b/params/version.go index 50653fc741..aa1b7aea4c 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 8 // Minor version component of the current release - VersionPatch = 9 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 8 // Minor version component of the current release + VersionPatch = 10 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From a9c6ef6905261bd25c4a2d14bf13b093ad6444fd Mon Sep 17 00:00:00 2001 From: Mohanson Date: Tue, 29 May 2018 15:44:06 +0800 Subject: [PATCH 39/46] ethereum: fix a typo in FilterQuery{} (#16827) Fix a spelling mistake in comment --- interfaces.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interfaces.go b/interfaces.go index 1ae1eba48a..a8b48c93d7 100644 --- a/interfaces.go +++ b/interfaces.go @@ -144,7 +144,7 @@ type FilterQuery struct { // {} or nil matches any topic list // {{A}} matches topic A in first position // {{}, {B}} matches any topic in first position, B in second position - // {{A}}, {B}} matches topic A in first position, B in second position + // {{A}, {B}} matches topic A in first position, B in second position // {{A, B}}, {C, D}} matches topic (A OR B) in first position, (C OR D) in second position Topics [][]common.Hash } From 40a2c523975d8bf74a0f64c09e628c4f95319376 Mon Sep 17 00:00:00 2001 From: Smilenator Date: Tue, 29 May 2018 10:57:08 +0300 Subject: [PATCH 40/46] eth/fetcher: reuse variables for hash and number (#16819) --- eth/fetcher/fetcher.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eth/fetcher/fetcher.go b/eth/fetcher/fetcher.go index 0c679cec3a..a2e7cdecf9 100644 --- a/eth/fetcher/fetcher.go +++ b/eth/fetcher/fetcher.go @@ -292,20 +292,20 @@ func (f *Fetcher) loop() { height := f.chainHeight() for !f.queue.Empty() { op := f.queue.PopItem().(*inject) + hash := op.block.Hash() if f.queueChangeHook != nil { - f.queueChangeHook(op.block.Hash(), false) + f.queueChangeHook(hash, false) } // If too high up the chain or phase, continue later number := op.block.NumberU64() if number > height+1 { - f.queue.Push(op, -float32(op.block.NumberU64())) + f.queue.Push(op, -float32(number)) if f.queueChangeHook != nil { - f.queueChangeHook(op.block.Hash(), true) + f.queueChangeHook(hash, true) } break } // Otherwise if fresh and still unknown, try and import - hash := op.block.Hash() if number+maxUncleDist < height || f.getBlock(hash) != nil { f.forgetBlock(hash) continue From 998f6564b28ea9241d0052c2abee090d2b9a8b63 Mon Sep 17 00:00:00 2001 From: Andrea Franz Date: Tue, 29 May 2018 10:36:31 +0200 Subject: [PATCH 41/46] whisper/shhclient: update call to shh_post to expect string instead of bool (#16757) Fixes #16756 --- whisper/shhclient/client.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/whisper/shhclient/client.go b/whisper/shhclient/client.go index 8e7085a0a6..a814154e47 100644 --- a/whisper/shhclient/client.go +++ b/whisper/shhclient/client.go @@ -159,9 +159,9 @@ func (sc *Client) DeleteSymmetricKey(ctx context.Context, id string) error { } // Post a message onto the network. -func (sc *Client) Post(ctx context.Context, message whisper.NewMessage) error { - var ignored bool - return sc.c.CallContext(ctx, &ignored, "shh_post", message) +func (sc *Client) Post(ctx context.Context, message whisper.NewMessage) (string, error) { + var hash string + return hash, sc.c.CallContext(ctx, &hash, "shh_post", message) } // SubscribeMessages subscribes to messages that match the given criteria. This method From 84f8c0cc1fbe1ab9c128555392a82ba609820fef Mon Sep 17 00:00:00 2001 From: kiel barry Date: Tue, 29 May 2018 03:42:21 -0700 Subject: [PATCH 42/46] common: improve documentation comments (#16701) This commit adds many comments and removes unused code. It also removes the EmptyHash function, which had some uses but was silly. --- common/bytes.go | 20 +++++++--- common/math/big.go | 5 +-- common/number/int.go | 43 ++++++++++----------- common/path.go | 6 ++- common/types.go | 74 ++++++++++++++++++------------------ common/types_template.go | 64 ------------------------------- core/state/state_test.go | 2 +- core/vm/gas_table.go | 6 +-- eth/downloader/downloader.go | 2 +- 9 files changed, 84 insertions(+), 138 deletions(-) delete mode 100644 common/types_template.go diff --git a/common/bytes.go b/common/bytes.go index e2adc80059..cbab2c3fa9 100644 --- a/common/bytes.go +++ b/common/bytes.go @@ -19,15 +19,20 @@ package common import "encoding/hex" +// ToHex returns the hex representation of b, prefixed with '0x'. +// For empty slices, the return value is "0x0". +// +// Deprecated: use hexutil.Encode instead. func ToHex(b []byte) string { hex := Bytes2Hex(b) - // Prefer output of "0x0" instead of "0x" if len(hex) == 0 { hex = "0" } return "0x" + hex } +// FromHex returns the bytes represented by the hexadecimal string s. +// s may be prefixed with "0x". func FromHex(s string) []byte { if len(s) > 1 { if s[0:2] == "0x" || s[0:2] == "0X" { @@ -40,9 +45,7 @@ func FromHex(s string) []byte { return Hex2Bytes(s) } -// Copy bytes -// -// Returns an exact copy of the provided bytes +// CopyBytes returns an exact copy of the provided bytes. func CopyBytes(b []byte) (copiedBytes []byte) { if b == nil { return nil @@ -53,14 +56,17 @@ func CopyBytes(b []byte) (copiedBytes []byte) { return } +// hasHexPrefix validates str begins with '0x' or '0X'. func hasHexPrefix(str string) bool { return len(str) >= 2 && str[0] == '0' && (str[1] == 'x' || str[1] == 'X') } +// isHexCharacter returns bool of c being a valid hexadecimal. func isHexCharacter(c byte) bool { return ('0' <= c && c <= '9') || ('a' <= c && c <= 'f') || ('A' <= c && c <= 'F') } +// isHex validates whether each byte is valid hexadecimal string. func isHex(str string) bool { if len(str)%2 != 0 { return false @@ -73,16 +79,18 @@ func isHex(str string) bool { return true } +// Bytes2Hex returns the hexadecimal encoding of d. func Bytes2Hex(d []byte) string { return hex.EncodeToString(d) } +// Hex2Bytes returns the bytes represented by the hexadecimal string str. func Hex2Bytes(str string) []byte { h, _ := hex.DecodeString(str) - return h } +// Hex2BytesFixed returns bytes of a specified fixed length flen. func Hex2BytesFixed(str string, flen int) []byte { h, _ := hex.DecodeString(str) if len(h) == flen { @@ -96,6 +104,7 @@ func Hex2BytesFixed(str string, flen int) []byte { return hh } +// RightPadBytes zero-pads slice to the right up to length l. func RightPadBytes(slice []byte, l int) []byte { if l <= len(slice) { return slice @@ -107,6 +116,7 @@ func RightPadBytes(slice []byte, l int) []byte { return padded } +// LeftPadBytes zero-pads slice to the left up to length l. func LeftPadBytes(slice []byte, l int) []byte { if l <= len(slice) { return slice diff --git a/common/math/big.go b/common/math/big.go index 7872786503..dbf2770a94 100644 --- a/common/math/big.go +++ b/common/math/big.go @@ -78,7 +78,7 @@ func ParseBig256(s string) (*big.Int, bool) { return bigint, ok } -// MustParseBig parses s as a 256 bit big integer and panics if the string is invalid. +// MustParseBig256 parses s as a 256 bit big integer and panics if the string is invalid. func MustParseBig256(s string) *big.Int { v, ok := ParseBig256(s) if !ok { @@ -186,9 +186,8 @@ func U256(x *big.Int) *big.Int { func S256(x *big.Int) *big.Int { if x.Cmp(tt255) < 0 { return x - } else { - return new(big.Int).Sub(x, tt256) } + return new(big.Int).Sub(x, tt256) } // Exp implements exponentiation by squaring. diff --git a/common/number/int.go b/common/number/int.go index 6dab2436de..5b50669703 100644 --- a/common/number/int.go +++ b/common/number/int.go @@ -34,13 +34,12 @@ func limitUnsigned256(x *Number) *Number { func limitSigned256(x *Number) *Number { if x.num.Cmp(tt255) < 0 { return x - } else { - x.num.Sub(x.num, tt256) - return x } + x.num.Sub(x.num, tt256) + return x } -// Number function +// Initialiser is a Number function type Initialiser func(n int64) *Number // A Number represents a generic integer with a bounding function limiter. Limit is called after each operations @@ -51,65 +50,65 @@ type Number struct { limit func(n *Number) *Number } -// Returns a new initialiser for a new *Number without having to expose certain fields +// NewInitialiser returns a new initialiser for a new *Number without having to expose certain fields func NewInitialiser(limiter func(*Number) *Number) Initialiser { return func(n int64) *Number { return &Number{big.NewInt(n), limiter} } } -// Return a Number with a UNSIGNED limiter up to 256 bits +// Uint256 returns a Number with a UNSIGNED limiter up to 256 bits func Uint256(n int64) *Number { return &Number{big.NewInt(n), limitUnsigned256} } -// Return a Number with a SIGNED limiter up to 256 bits +// Int256 returns Number with a SIGNED limiter up to 256 bits func Int256(n int64) *Number { return &Number{big.NewInt(n), limitSigned256} } -// Returns a Number with a SIGNED unlimited size +// Big returns a Number with a SIGNED unlimited size func Big(n int64) *Number { return &Number{big.NewInt(n), func(x *Number) *Number { return x }} } -// Sets i to sum of x+y +// Add sets i to sum of x+y func (i *Number) Add(x, y *Number) *Number { i.num.Add(x.num, y.num) return i.limit(i) } -// Sets i to difference of x-y +// Sub sets i to difference of x-y func (i *Number) Sub(x, y *Number) *Number { i.num.Sub(x.num, y.num) return i.limit(i) } -// Sets i to product of x*y +// Mul sets i to product of x*y func (i *Number) Mul(x, y *Number) *Number { i.num.Mul(x.num, y.num) return i.limit(i) } -// Sets i to the quotient prodject of x/y +// Div sets i to the quotient prodject of x/y func (i *Number) Div(x, y *Number) *Number { i.num.Div(x.num, y.num) return i.limit(i) } -// Sets i to x % y +// Mod sets i to x % y func (i *Number) Mod(x, y *Number) *Number { i.num.Mod(x.num, y.num) return i.limit(i) } -// Sets i to x << s +// Lsh sets i to x << s func (i *Number) Lsh(x *Number, s uint) *Number { i.num.Lsh(x.num, s) return i.limit(i) } -// Sets i to x^y +// Pow sets i to x^y func (i *Number) Pow(x, y *Number) *Number { i.num.Exp(x.num, y.num, big.NewInt(0)) return i.limit(i) @@ -117,13 +116,13 @@ func (i *Number) Pow(x, y *Number) *Number { // Setters -// Set x to i +// Set sets x to i func (i *Number) Set(x *Number) *Number { i.num.Set(x.num) return i.limit(i) } -// Set x bytes to i +// SetBytes sets x bytes to i func (i *Number) SetBytes(x []byte) *Number { i.num.SetBytes(x) return i.limit(i) @@ -140,12 +139,12 @@ func (i *Number) Cmp(x *Number) int { // Getters -// Returns the string representation of i +// String returns the string representation of i func (i *Number) String() string { return i.num.String() } -// Returns the byte representation of i +// Bytes returns the byte representation of i func (i *Number) Bytes() []byte { return i.num.Bytes() } @@ -160,17 +159,17 @@ func (i *Number) Int64() int64 { return i.num.Int64() } -// Returns the signed version of i +// Int256 returns the signed version of i func (i *Number) Int256() *Number { return Int(0).Set(i) } -// Returns the unsigned version of i +// Uint256 returns the unsigned version of i func (i *Number) Uint256() *Number { return Uint(0).Set(i) } -// Returns the index of the first bit that's set to 1 +// FirstBitSet returns the index of the first bit that's set to 1 func (i *Number) FirstBitSet() int { for j := 0; j < i.num.BitLen(); j++ { if i.num.Bit(j) > 0 { diff --git a/common/path.go b/common/path.go index bd8da86e74..69820cfe5d 100644 --- a/common/path.go +++ b/common/path.go @@ -30,6 +30,7 @@ func MakeName(name, version string) string { return fmt.Sprintf("%s/v%s/%s/%s", name, version, runtime.GOOS, runtime.Version()) } +// FileExist checks if a file exists at filePath. func FileExist(filePath string) bool { _, err := os.Stat(filePath) if err != nil && os.IsNotExist(err) { @@ -39,9 +40,10 @@ func FileExist(filePath string) bool { return true } -func AbsolutePath(Datadir string, filename string) string { +// AbsolutePath returns datadir + filename, or filename if it is absolute. +func AbsolutePath(datadir string, filename string) string { if filepath.IsAbs(filename) { return filename } - return filepath.Join(Datadir, filename) + return filepath.Join(datadir, filename) } diff --git a/common/types.go b/common/types.go index 0b94fb2c25..12c26d94bc 100644 --- a/common/types.go +++ b/common/types.go @@ -42,19 +42,30 @@ var ( // Hash represents the 32 byte Keccak256 hash of arbitrary data. type Hash [HashLength]byte +// BytesToHash sets b to hash. +// If b is larger than len(h), b will be cropped from the left. func BytesToHash(b []byte) Hash { var h Hash h.SetBytes(b) return h } -func BigToHash(b *big.Int) Hash { return BytesToHash(b.Bytes()) } -func HexToHash(s string) Hash { return BytesToHash(FromHex(s)) } -// Get the string representation of the underlying hash -func (h Hash) Str() string { return string(h[:]) } +// BigToHash sets byte representation of b to hash. +// If b is larger than len(h), b will be cropped from the left. +func BigToHash(b *big.Int) Hash { return BytesToHash(b.Bytes()) } + +// HexToHash sets byte representation of s to hash. +// If b is larger than len(h), b will be cropped from the left. +func HexToHash(s string) Hash { return BytesToHash(FromHex(s)) } + +// Bytes gets the byte representation of the underlying hash. func (h Hash) Bytes() []byte { return h[:] } + +// Big converts a hash to a big integer. func (h Hash) Big() *big.Int { return new(big.Int).SetBytes(h[:]) } -func (h Hash) Hex() string { return hexutil.Encode(h[:]) } + +// Hex converts a hash to a hex string. +func (h Hash) Hex() string { return hexutil.Encode(h[:]) } // TerminalString implements log.TerminalStringer, formatting a string for console // output during logging. @@ -89,7 +100,8 @@ func (h Hash) MarshalText() ([]byte, error) { return hexutil.Bytes(h[:]).MarshalText() } -// Sets the hash to the value of b. If b is larger than len(h), 'b' will be cropped (from the left). +// SetBytes sets the hash to the value of b. +// If b is larger than len(h), b will be cropped from the left. func (h *Hash) SetBytes(b []byte) { if len(b) > len(h) { b = b[len(b)-HashLength:] @@ -98,16 +110,6 @@ func (h *Hash) SetBytes(b []byte) { copy(h[HashLength-len(b):], b) } -// Set string `s` to h. If s is larger than len(h) s will be cropped (from left) to fit. -func (h *Hash) SetString(s string) { h.SetBytes([]byte(s)) } - -// Sets h to other -func (h *Hash) Set(other Hash) { - for i, v := range other { - h[i] = v - } -} - // Generate implements testing/quick.Generator. func (h Hash) Generate(rand *rand.Rand, size int) reflect.Value { m := rand.Intn(len(h)) @@ -117,10 +119,6 @@ func (h Hash) Generate(rand *rand.Rand, size int) reflect.Value { return reflect.ValueOf(h) } -func EmptyHash(h Hash) bool { - return h == Hash{} -} - // UnprefixedHash allows marshaling a Hash without 0x prefix. type UnprefixedHash Hash @@ -139,13 +137,21 @@ func (h UnprefixedHash) MarshalText() ([]byte, error) { // Address represents the 20 byte address of an Ethereum account. type Address [AddressLength]byte +// BytesToAddress returns Address with value b. +// If b is larger than len(h), b will be cropped from the left. func BytesToAddress(b []byte) Address { var a Address a.SetBytes(b) return a } + +// BigToAddress returns Address with byte values of b. +// If b is larger than len(h), b will be cropped from the left. func BigToAddress(b *big.Int) Address { return BytesToAddress(b.Bytes()) } -func HexToAddress(s string) Address { return BytesToAddress(FromHex(s)) } + +// HexToAddress returns Address with byte values of s. +// If s is larger than len(h), s will be cropped from the left. +func HexToAddress(s string) Address { return BytesToAddress(FromHex(s)) } // IsHexAddress verifies whether a string can represent a valid hex-encoded // Ethereum address or not. @@ -156,11 +162,14 @@ func IsHexAddress(s string) bool { return len(s) == 2*AddressLength && isHex(s) } -// Get the string representation of the underlying address -func (a Address) Str() string { return string(a[:]) } +// Bytes gets the string representation of the underlying address. func (a Address) Bytes() []byte { return a[:] } + +// Big converts an address to a big integer. func (a Address) Big() *big.Int { return new(big.Int).SetBytes(a[:]) } -func (a Address) Hash() Hash { return BytesToHash(a[:]) } + +// Hash converts an address to a hash by left-padding it with zeros. +func (a Address) Hash() Hash { return BytesToHash(a[:]) } // Hex returns an EIP55-compliant hex string representation of the address. func (a Address) Hex() string { @@ -184,7 +193,7 @@ func (a Address) Hex() string { return "0x" + string(result) } -// String implements the stringer interface and is used also by the logger. +// String implements fmt.Stringer. func (a Address) String() string { return a.Hex() } @@ -195,7 +204,8 @@ func (a Address) Format(s fmt.State, c rune) { fmt.Fprintf(s, "%"+string(c), a[:]) } -// Sets the address to the value of b. If b is larger than len(a) it will panic +// SetBytes sets the address to the value of b. +// If b is larger than len(a) it will panic. func (a *Address) SetBytes(b []byte) { if len(b) > len(a) { b = b[len(b)-AddressLength:] @@ -203,16 +213,6 @@ func (a *Address) SetBytes(b []byte) { copy(a[AddressLength-len(b):], b) } -// Set string `s` to a. If s is larger than len(a) it will panic -func (a *Address) SetString(s string) { a.SetBytes([]byte(s)) } - -// Sets a to other -func (a *Address) Set(other Address) { - for i, v := range other { - a[i] = v - } -} - // MarshalText returns the hex representation of a. func (a Address) MarshalText() ([]byte, error) { return hexutil.Bytes(a[:]).MarshalText() @@ -228,7 +228,7 @@ func (a *Address) UnmarshalJSON(input []byte) error { return hexutil.UnmarshalFixedJSON(addressT, input, a[:]) } -// UnprefixedHash allows marshaling an Address without 0x prefix. +// UnprefixedAddress allows marshaling an Address without 0x prefix. type UnprefixedAddress Address // UnmarshalText decodes the address from hex. The 0x prefix is optional. diff --git a/common/types_template.go b/common/types_template.go deleted file mode 100644 index 9a8f29977b..0000000000 --- a/common/types_template.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// +build none -//sed -e 's/_N_/Hash/g' -e 's/_S_/32/g' -e '1d' types_template.go | gofmt -w hash.go - -package common - -import "math/big" - -type _N_ [_S_]byte - -func BytesTo_N_(b []byte) _N_ { - var h _N_ - h.SetBytes(b) - return h -} -func StringTo_N_(s string) _N_ { return BytesTo_N_([]byte(s)) } -func BigTo_N_(b *big.Int) _N_ { return BytesTo_N_(b.Bytes()) } -func HexTo_N_(s string) _N_ { return BytesTo_N_(FromHex(s)) } - -// Don't use the default 'String' method in case we want to overwrite - -// Get the string representation of the underlying hash -func (h _N_) Str() string { return string(h[:]) } -func (h _N_) Bytes() []byte { return h[:] } -func (h _N_) Big() *big.Int { return new(big.Int).SetBytes(h[:]) } -func (h _N_) Hex() string { return "0x" + Bytes2Hex(h[:]) } - -// Sets the hash to the value of b. If b is larger than len(h) it will panic -func (h *_N_) SetBytes(b []byte) { - // Use the right most bytes - if len(b) > len(h) { - b = b[len(b)-_S_:] - } - - // Reverse the loop - for i := len(b) - 1; i >= 0; i-- { - h[_S_-len(b)+i] = b[i] - } -} - -// Set string `s` to h. If s is larger than len(h) it will panic -func (h *_N_) SetString(s string) { h.SetBytes([]byte(s)) } - -// Sets h to other -func (h *_N_) Set(other _N_) { - for i, v := range other { - h[i] = v - } -} diff --git a/core/state/state_test.go b/core/state/state_test.go index 12778f6f12..123559ea9b 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -99,7 +99,7 @@ func (s *StateSuite) TestNull(c *checker.C) { s.state.SetState(address, common.Hash{}, value) s.state.Commit(false) value = s.state.GetState(address, common.Hash{}) - if !common.EmptyHash(value) { + if value != (common.Hash{}) { c.Errorf("expected empty hash. got %x", value) } } diff --git a/core/vm/gas_table.go b/core/vm/gas_table.go index 83adba428e..0764c67a4d 100644 --- a/core/vm/gas_table.go +++ b/core/vm/gas_table.go @@ -124,12 +124,12 @@ func gasSStore(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, m // 1. From a zero-value address to a non-zero value (NEW VALUE) // 2. From a non-zero value address to a zero-value address (DELETE) // 3. From a non-zero to a non-zero (CHANGE) - if common.EmptyHash(val) && !common.EmptyHash(common.BigToHash(y)) { + if val == (common.Hash{}) && y.Sign() != 0 { // 0 => non 0 return params.SstoreSetGas, nil - } else if !common.EmptyHash(val) && common.EmptyHash(common.BigToHash(y)) { + } else if val != (common.Hash{}) && y.Sign() == 0 { + // non 0 => 0 evm.StateDB.AddRefund(params.SstoreRefundGas) - return params.SstoreClearGas, nil } else { // non 0 => non 0 (or 0 => 0) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index dc23354929..51c5936015 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -680,7 +680,7 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err } } // If the head fetch already found an ancestor, return - if !common.EmptyHash(hash) { + if hash != (common.Hash{}) { if int64(number) <= floor { p.log.Warn("Ancestor below allowance", "number", number, "hash", hash, "allowance", floor) return 0, errInvalidAncestor From d258eee21177326c136893bb100285376a3f297c Mon Sep 17 00:00:00 2001 From: Abel Nieto Date: Tue, 29 May 2018 13:22:00 +0200 Subject: [PATCH 43/46] core/vm: fix typo in comment --- core/vm/jump_table.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go index 3389941353..49a94d9646 100644 --- a/core/vm/jump_table.go +++ b/core/vm/jump_table.go @@ -33,7 +33,7 @@ type ( var errGasUintOverflow = errors.New("gas uint64 overflow") type operation struct { - // op is the operation function + // execute is the operation function execute executionFunc // gasCost is the gas function and returns the gas required for execution gasCost gasFunc From 7677ec1f3414c1e3712bf33598f8780c3b76742b Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Tue, 29 May 2018 14:46:09 +0300 Subject: [PATCH 44/46] p2p/discv5: add egress/ingress traffic metrics to discv5 udp transport (#16369) --- p2p/discv5/metrics.go | 8 ++++++++ p2p/discv5/udp.go | 5 ++++- 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 p2p/discv5/metrics.go diff --git a/p2p/discv5/metrics.go b/p2p/discv5/metrics.go new file mode 100644 index 0000000000..cb11d7eacf --- /dev/null +++ b/p2p/discv5/metrics.go @@ -0,0 +1,8 @@ +package discv5 + +import "github.com/ethereum/go-ethereum/metrics" + +var ( + ingressTrafficMeter = metrics.NewRegisteredMeter("discv5/InboundTraffic", nil) + egressTrafficMeter = metrics.NewRegisteredMeter("discv5/OutboundTraffic", nil) +) diff --git a/p2p/discv5/udp.go b/p2p/discv5/udp.go index 09e5f8b374..49e1cb811a 100644 --- a/p2p/discv5/udp.go +++ b/p2p/discv5/udp.go @@ -334,8 +334,10 @@ func (t *udp) sendPacket(toid NodeID, toaddr *net.UDPAddr, ptype byte, req inter return hash, err } log.Trace(fmt.Sprintf(">>> %v to %x@%v", nodeEvent(ptype), toid[:8], toaddr)) - if _, err = t.conn.WriteToUDP(packet, toaddr); err != nil { + if nbytes, err := t.conn.WriteToUDP(packet, toaddr); err != nil { log.Trace(fmt.Sprint("UDP send failed:", err)) + } else { + egressTrafficMeter.Mark(int64(nbytes)) } //fmt.Println(err) return hash, err @@ -374,6 +376,7 @@ func (t *udp) readLoop() { buf := make([]byte, 1280) for { nbytes, from, err := t.conn.ReadFromUDP(buf) + ingressTrafficMeter.Mark(int64(nbytes)) if netutil.IsTemporaryError(err) { // Ignore temporary read errors. log.Debug(fmt.Sprintf("Temporary read error: %v", err)) From 426f62f1a8f25c10dbf17efe99242b4e1a44b83c Mon Sep 17 00:00:00 2001 From: kimmylin <30611210+kimmylin@users.noreply.github.com> Date: Tue, 29 May 2018 20:21:04 +0800 Subject: [PATCH 45/46] core: improve test for TransactionPriceNonceSort (#16413) --- core/types/transaction_test.go | 29 +++++++---------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/core/types/transaction_test.go b/core/types/transaction_test.go index d1861b14cd..b390f45c67 100644 --- a/core/types/transaction_test.go +++ b/core/types/transaction_test.go @@ -165,28 +165,13 @@ func TestTransactionPriceNonceSort(t *testing.T) { t.Errorf("invalid nonce ordering: tx #%d (A=%x N=%v) < tx #%d (A=%x N=%v)", i, fromi[:4], txi.Nonce(), i+j, fromj[:4], txj.Nonce()) } } - // Find the previous and next nonce of this account - prev, next := i-1, i+1 - for j := i - 1; j >= 0; j-- { - if fromj, _ := Sender(signer, txs[j]); fromi == fromj { - prev = j - break - } - } - for j := i + 1; j < len(txs); j++ { - if fromj, _ := Sender(signer, txs[j]); fromi == fromj { - next = j - break - } - } - // Make sure that in between the neighbor nonces, the transaction is correctly positioned price wise - for j := prev + 1; j < next; j++ { - fromj, _ := Sender(signer, txs[j]) - if j < i && txs[j].GasPrice().Cmp(txi.GasPrice()) < 0 { - t.Errorf("invalid gasprice ordering: tx #%d (A=%x P=%v) < tx #%d (A=%x P=%v)", j, fromj[:4], txs[j].GasPrice(), i, fromi[:4], txi.GasPrice()) - } - if j > i && txs[j].GasPrice().Cmp(txi.GasPrice()) > 0 { - t.Errorf("invalid gasprice ordering: tx #%d (A=%x P=%v) > tx #%d (A=%x P=%v)", j, fromj[:4], txs[j].GasPrice(), i, fromi[:4], txi.GasPrice()) + + // If the next tx has different from account, the price must be lower than the current one + if i+1 < len(txs) { + next := txs[i+1] + fromNext, _ := Sender(signer, next) + if fromi != fromNext && txi.GasPrice().Cmp(next.GasPrice()) < 0 { + t.Errorf("invalid gasprice ordering: tx #%d (A=%x P=%v) < tx #%d (A=%x P=%v)", i, fromi[:4], txi.GasPrice(), i+1, fromNext[:4], next.GasPrice()) } } } From 38c7eb0f26eaa8df229d27e92f12e253313a6c8d Mon Sep 17 00:00:00 2001 From: Wenbiao Zheng Date: Tue, 29 May 2018 23:48:43 +0800 Subject: [PATCH 46/46] trie: rename TrieSync to Sync and improve hexToKeybytes (#16804) This removes a golint warning: type name will be used as trie.TrieSync by other packages, and that stutters; consider calling this Sync. In hexToKeybytes len(hex) is even and (even+1)/2 == even/2, remove the +1. --- core/state/sync.go | 6 +++--- eth/downloader/statesync.go | 2 +- trie/encoding.go | 2 +- trie/sync.go | 28 ++++++++++++++-------------- trie/sync_test.go | 36 ++++++++++++++++++------------------ 5 files changed, 37 insertions(+), 37 deletions(-) diff --git a/core/state/sync.go b/core/state/sync.go index 28fcf6ae05..c566e79073 100644 --- a/core/state/sync.go +++ b/core/state/sync.go @@ -25,8 +25,8 @@ import ( ) // NewStateSync create a new state trie download scheduler. -func NewStateSync(root common.Hash, database trie.DatabaseReader) *trie.TrieSync { - var syncer *trie.TrieSync +func NewStateSync(root common.Hash, database trie.DatabaseReader) *trie.Sync { + var syncer *trie.Sync callback := func(leaf []byte, parent common.Hash) error { var obj Account if err := rlp.Decode(bytes.NewReader(leaf), &obj); err != nil { @@ -36,6 +36,6 @@ func NewStateSync(root common.Hash, database trie.DatabaseReader) *trie.TrieSync syncer.AddRawEntry(common.BytesToHash(obj.CodeHash), 64, parent) return nil } - syncer = trie.NewTrieSync(root, database, callback) + syncer = trie.NewSync(root, database, callback) return syncer } diff --git a/eth/downloader/statesync.go b/eth/downloader/statesync.go index 5b4b9ba1b7..8d33dfec74 100644 --- a/eth/downloader/statesync.go +++ b/eth/downloader/statesync.go @@ -214,7 +214,7 @@ func (d *Downloader) runStateSync(s *stateSync) *stateSync { type stateSync struct { d *Downloader // Downloader instance to access and manage current peerset - sched *trie.TrieSync // State trie sync scheduler defining the tasks + sched *trie.Sync // State trie sync scheduler defining the tasks keccak hash.Hash // Keccak256 hasher to verify deliveries with tasks map[common.Hash]*stateTask // Set of tasks currently queued for retrieval diff --git a/trie/encoding.go b/trie/encoding.go index e96a786e40..221fa6d3aa 100644 --- a/trie/encoding.go +++ b/trie/encoding.go @@ -83,7 +83,7 @@ func hexToKeybytes(hex []byte) []byte { if len(hex)&1 != 0 { panic("can't convert hex key of odd length") } - key := make([]byte, (len(hex)+1)/2) + key := make([]byte, len(hex)/2) decodeNibbles(hex, key) return key } diff --git a/trie/sync.go b/trie/sync.go index 4ae975d042..ccec80c9e3 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -68,19 +68,19 @@ func newSyncMemBatch() *syncMemBatch { } } -// TrieSync is the main state trie synchronisation scheduler, which provides yet +// Sync is the main state trie synchronisation scheduler, which provides yet // unknown trie hashes to retrieve, accepts node data associated with said hashes // and reconstructs the trie step by step until all is done. -type TrieSync struct { +type Sync struct { database DatabaseReader // Persistent database to check for existing entries membatch *syncMemBatch // Memory buffer to avoid frequest database writes requests map[common.Hash]*request // Pending requests pertaining to a key hash queue *prque.Prque // Priority queue with the pending requests } -// NewTrieSync creates a new trie data download scheduler. -func NewTrieSync(root common.Hash, database DatabaseReader, callback LeafCallback) *TrieSync { - ts := &TrieSync{ +// NewSync creates a new trie data download scheduler. +func NewSync(root common.Hash, database DatabaseReader, callback LeafCallback) *Sync { + ts := &Sync{ database: database, membatch: newSyncMemBatch(), requests: make(map[common.Hash]*request), @@ -91,7 +91,7 @@ func NewTrieSync(root common.Hash, database DatabaseReader, callback LeafCallbac } // AddSubTrie registers a new trie to the sync code, rooted at the designated parent. -func (s *TrieSync) AddSubTrie(root common.Hash, depth int, parent common.Hash, callback LeafCallback) { +func (s *Sync) AddSubTrie(root common.Hash, depth int, parent common.Hash, callback LeafCallback) { // Short circuit if the trie is empty or already known if root == emptyRoot { return @@ -126,7 +126,7 @@ func (s *TrieSync) AddSubTrie(root common.Hash, depth int, parent common.Hash, c // interpreted as a trie node, but rather accepted and stored into the database // as is. This method's goal is to support misc state metadata retrievals (e.g. // contract code). -func (s *TrieSync) AddRawEntry(hash common.Hash, depth int, parent common.Hash) { +func (s *Sync) AddRawEntry(hash common.Hash, depth int, parent common.Hash) { // Short circuit if the entry is empty or already known if hash == emptyState { return @@ -156,7 +156,7 @@ func (s *TrieSync) AddRawEntry(hash common.Hash, depth int, parent common.Hash) } // Missing retrieves the known missing nodes from the trie for retrieval. -func (s *TrieSync) Missing(max int) []common.Hash { +func (s *Sync) Missing(max int) []common.Hash { requests := []common.Hash{} for !s.queue.Empty() && (max == 0 || len(requests) < max) { requests = append(requests, s.queue.PopItem().(common.Hash)) @@ -167,7 +167,7 @@ func (s *TrieSync) Missing(max int) []common.Hash { // Process injects a batch of retrieved trie nodes data, returning if something // was committed to the database and also the index of an entry if processing of // it failed. -func (s *TrieSync) Process(results []SyncResult) (bool, int, error) { +func (s *Sync) Process(results []SyncResult) (bool, int, error) { committed := false for i, item := range results { @@ -213,7 +213,7 @@ func (s *TrieSync) Process(results []SyncResult) (bool, int, error) { // Commit flushes the data stored in the internal membatch out to persistent // storage, returning the number of items written and any occurred error. -func (s *TrieSync) Commit(dbw ethdb.Putter) (int, error) { +func (s *Sync) Commit(dbw ethdb.Putter) (int, error) { // Dump the membatch into a database dbw for i, key := range s.membatch.order { if err := dbw.Put(key[:], s.membatch.batch[key]); err != nil { @@ -228,14 +228,14 @@ func (s *TrieSync) Commit(dbw ethdb.Putter) (int, error) { } // Pending returns the number of state entries currently pending for download. -func (s *TrieSync) Pending() int { +func (s *Sync) Pending() int { return len(s.requests) } // schedule inserts a new state retrieval request into the fetch queue. If there // is already a pending request for this node, the new request will be discarded // and only a parent reference added to the old one. -func (s *TrieSync) schedule(req *request) { +func (s *Sync) schedule(req *request) { // If we're already requesting this node, add a new reference and stop if old, ok := s.requests[req.hash]; ok { old.parents = append(old.parents, req.parents...) @@ -248,7 +248,7 @@ func (s *TrieSync) schedule(req *request) { // children retrieves all the missing children of a state trie entry for future // retrieval scheduling. -func (s *TrieSync) children(req *request, object node) ([]*request, error) { +func (s *Sync) children(req *request, object node) ([]*request, error) { // Gather all the children of the node, irrelevant whether known or not type child struct { node node @@ -310,7 +310,7 @@ func (s *TrieSync) children(req *request, object node) ([]*request, error) { // commit finalizes a retrieval request and stores it into the membatch. If any // of the referencing parent requests complete due to this commit, they are also // committed themselves. -func (s *TrieSync) commit(req *request) (err error) { +func (s *Sync) commit(req *request) (err error) { // Write the node content to the membatch s.membatch.batch[req.hash] = req.data s.membatch.order = append(s.membatch.order, req.hash) diff --git a/trie/sync_test.go b/trie/sync_test.go index 142a6f5b1a..c76779e5c7 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -87,14 +87,14 @@ func checkTrieConsistency(db *Database, root common.Hash) error { } // Tests that an empty trie is not scheduled for syncing. -func TestEmptyTrieSync(t *testing.T) { +func TestEmptySync(t *testing.T) { dbA := NewDatabase(ethdb.NewMemDatabase()) dbB := NewDatabase(ethdb.NewMemDatabase()) emptyA, _ := New(common.Hash{}, dbA) emptyB, _ := New(emptyRoot, dbB) for i, trie := range []*Trie{emptyA, emptyB} { - if req := NewTrieSync(trie.Hash(), ethdb.NewMemDatabase(), nil).Missing(1); len(req) != 0 { + if req := NewSync(trie.Hash(), ethdb.NewMemDatabase(), nil).Missing(1); len(req) != 0 { t.Errorf("test %d: content requested for empty trie: %v", i, req) } } @@ -102,17 +102,17 @@ func TestEmptyTrieSync(t *testing.T) { // Tests that given a root hash, a trie can sync iteratively on a single thread, // requesting retrieval tasks and returning all of them in one go. -func TestIterativeTrieSyncIndividual(t *testing.T) { testIterativeTrieSync(t, 1) } -func TestIterativeTrieSyncBatched(t *testing.T) { testIterativeTrieSync(t, 100) } +func TestIterativeSyncIndividual(t *testing.T) { testIterativeSync(t, 1) } +func TestIterativeSyncBatched(t *testing.T) { testIterativeSync(t, 100) } -func testIterativeTrieSync(t *testing.T, batch int) { +func testIterativeSync(t *testing.T, batch int) { // Create a random trie to copy srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) - sched := NewTrieSync(srcTrie.Hash(), diskdb, nil) + sched := NewSync(srcTrie.Hash(), diskdb, nil) queue := append([]common.Hash{}, sched.Missing(batch)...) for len(queue) > 0 { @@ -138,14 +138,14 @@ func testIterativeTrieSync(t *testing.T, batch int) { // Tests that the trie scheduler can correctly reconstruct the state even if only // partial results are returned, and the others sent only later. -func TestIterativeDelayedTrieSync(t *testing.T) { +func TestIterativeDelayedSync(t *testing.T) { // Create a random trie to copy srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) - sched := NewTrieSync(srcTrie.Hash(), diskdb, nil) + sched := NewSync(srcTrie.Hash(), diskdb, nil) queue := append([]common.Hash{}, sched.Missing(10000)...) for len(queue) > 0 { @@ -173,17 +173,17 @@ func TestIterativeDelayedTrieSync(t *testing.T) { // Tests that given a root hash, a trie can sync iteratively on a single thread, // requesting retrieval tasks and returning all of them in one go, however in a // random order. -func TestIterativeRandomTrieSyncIndividual(t *testing.T) { testIterativeRandomTrieSync(t, 1) } -func TestIterativeRandomTrieSyncBatched(t *testing.T) { testIterativeRandomTrieSync(t, 100) } +func TestIterativeRandomSyncIndividual(t *testing.T) { testIterativeRandomSync(t, 1) } +func TestIterativeRandomSyncBatched(t *testing.T) { testIterativeRandomSync(t, 100) } -func testIterativeRandomTrieSync(t *testing.T, batch int) { +func testIterativeRandomSync(t *testing.T, batch int) { // Create a random trie to copy srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) - sched := NewTrieSync(srcTrie.Hash(), diskdb, nil) + sched := NewSync(srcTrie.Hash(), diskdb, nil) queue := make(map[common.Hash]struct{}) for _, hash := range sched.Missing(batch) { @@ -217,14 +217,14 @@ func testIterativeRandomTrieSync(t *testing.T, batch int) { // Tests that the trie scheduler can correctly reconstruct the state even if only // partial results are returned (Even those randomly), others sent only later. -func TestIterativeRandomDelayedTrieSync(t *testing.T) { +func TestIterativeRandomDelayedSync(t *testing.T) { // Create a random trie to copy srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) - sched := NewTrieSync(srcTrie.Hash(), diskdb, nil) + sched := NewSync(srcTrie.Hash(), diskdb, nil) queue := make(map[common.Hash]struct{}) for _, hash := range sched.Missing(10000) { @@ -264,14 +264,14 @@ func TestIterativeRandomDelayedTrieSync(t *testing.T) { // Tests that a trie sync will not request nodes multiple times, even if they // have such references. -func TestDuplicateAvoidanceTrieSync(t *testing.T) { +func TestDuplicateAvoidanceSync(t *testing.T) { // Create a random trie to copy srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) - sched := NewTrieSync(srcTrie.Hash(), diskdb, nil) + sched := NewSync(srcTrie.Hash(), diskdb, nil) queue := append([]common.Hash{}, sched.Missing(0)...) requested := make(map[common.Hash]struct{}) @@ -304,14 +304,14 @@ func TestDuplicateAvoidanceTrieSync(t *testing.T) { // Tests that at any point in time during a sync, only complete sub-tries are in // the database. -func TestIncompleteTrieSync(t *testing.T) { +func TestIncompleteSync(t *testing.T) { // Create a random trie to copy srcDb, srcTrie, _ := makeTestTrie() // Create a destination trie and sync with the scheduler diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) - sched := NewTrieSync(srcTrie.Hash(), diskdb, nil) + sched := NewSync(srcTrie.Hash(), diskdb, nil) added := []common.Hash{} queue := append([]common.Hash{}, sched.Missing(1)...)