diff --git a/Dockerfile b/Dockerfile index 947f045e59..eae8924997 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,16 @@ -FROM alpine:3.6 +# Build Geth in a stock Go builder container +FROM golang:1.9-alpine as builder + +RUN apk add --no-cache make gcc musl-dev linux-headers ADD . /go-ethereum -RUN \ - apk add --no-cache git go make gcc musl-dev linux-headers && \ - (cd go-ethereum && make geth) && \ - cp go-ethereum/build/bin/geth /usr/local/bin/ && \ - apk del git go make gcc musl-dev linux-headers && \ - rm -rf /go-ethereum +RUN cd /go-ethereum && make geth -EXPOSE 8545 30303 30303/udp +# Pull Geth into a second stage deploy alpine container +FROM alpine:latest +RUN apk add --no-cache ca-certificates +COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/ + +EXPOSE 8545 8546 30303 30303/udp ENTRYPOINT ["geth"] diff --git a/VERSION b/VERSION index bd8bf882d0..943f9cbc4e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.7.0 +1.7.1 diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 497a0b9023..3f53edeca2 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -22,6 +22,7 @@ import ( "fmt" "math/big" "sync" + "time" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi/bind" @@ -294,6 +295,21 @@ func (b *SimulatedBackend) SubscribeFilterLogs(ctx context.Context, q ethereum.F func (b *SimulatedBackend) FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]*types.Log, error) { return nil, errLogsUnsupported } +// JumpTimeInSeconds adds skip seconds to the clock +func (b *SimulatedBackend) AdjustTime(adjustment time.Duration) error { + b.mu.Lock() + defer b.mu.Unlock() + blocks, _ := core.GenerateChain(b.config, b.blockchain.CurrentBlock(), b.database, 1, func(number int, block *core.BlockGen) { + for _, tx := range b.pendingBlock.Transactions() { + block.AddTx(tx) + } + block.OffsetTime(int64(adjustment.Seconds())) + }) + b.pendingBlock = blocks[0] + b.pendingState, _ = state.New(b.pendingBlock.Root(), state.NewDatabase(b.database)) + + return nil +} // callmsg implements core.Message to allow passing it as a transaction simulator. type callmsg struct { diff --git a/bmt/bmt.go b/bmt/bmt.go new file mode 100644 index 0000000000..d62365bb1f --- /dev/null +++ b/bmt/bmt.go @@ -0,0 +1,562 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package bmt provides a binary merkle tree implementation +package bmt + +import ( + "fmt" + "hash" + "io" + "strings" + "sync" + "sync/atomic" +) + +/* +Binary Merkle Tree Hash is a hash function over arbitrary datachunks of limited size +It is defined as the root hash of the binary merkle tree built over fixed size segments +of the underlying chunk using any base hash function (e.g keccak 256 SHA3) + +It is used as the chunk hash function in swarm which in turn is the basis for the +128 branching swarm hash http://swarm-guide.readthedocs.io/en/latest/architecture.html#swarm-hash + +The BMT is optimal for providing compact inclusion proofs, i.e. prove that a +segment is a substring of a chunk starting at a particular offset +The size of the underlying segments is fixed at 32 bytes (called the resolution +of the BMT hash), the EVM word size to optimize for on-chain BMT verification +as well as the hash size optimal for inclusion proofs in the merkle tree of the swarm hash. + +Two implementations are provided: + +* RefHasher is optimized for code simplicity and meant as a reference implementation +* Hasher is optimized for speed taking advantage of concurrency with minimalistic + control structure to coordinate the concurrent routines + It implements the ChunkHash interface as well as the go standard hash.Hash interface + +*/ + +const ( + // DefaultSegmentCount is the maximum number of segments of the underlying chunk + DefaultSegmentCount = 128 // Should be equal to storage.DefaultBranches + // DefaultPoolSize is the maximum number of bmt trees used by the hashers, i.e, + // the maximum number of concurrent BMT hashing operations performed by the same hasher + DefaultPoolSize = 8 +) + +// BaseHasher is a hash.Hash constructor function used for the base hash of the BMT. +type BaseHasher func() hash.Hash + +// Hasher a reusable hasher for fixed maximum size chunks representing a BMT +// implements the hash.Hash interface +// reuse pool of Tree-s for amortised memory allocation and resource control +// supports order-agnostic concurrent segment writes +// as well as sequential read and write +// can not be called concurrently on more than one chunk +// can be further appended after Sum +// Reset gives back the Tree to the pool and guaranteed to leave +// the tree and itself in a state reusable for hashing a new chunk +type Hasher struct { + pool *TreePool // BMT resource pool + bmt *Tree // prebuilt BMT resource for flowcontrol and proofs + blocksize int // segment size (size of hash) also for hash.Hash + count int // segment count + size int // for hash.Hash same as hashsize + cur int // cursor position for righmost currently open chunk + segment []byte // the rightmost open segment (not complete) + depth int // index of last level + result chan []byte // result channel + hash []byte // to record the result + max int32 // max segments for SegmentWriter interface + blockLength []byte // The block length that needes to be added in Sum +} + +// New creates a reusable Hasher +// implements the hash.Hash interface +// pulls a new Tree from a resource pool for hashing each chunk +func New(p *TreePool) *Hasher { + return &Hasher{ + pool: p, + depth: depth(p.SegmentCount), + size: p.SegmentSize, + blocksize: p.SegmentSize, + count: p.SegmentCount, + result: make(chan []byte), + } +} + +// Node is a reuseable segment hasher representing a node in a BMT +// it allows for continued writes after a Sum +// and is left in completely reusable state after Reset +type Node struct { + level, index int // position of node for information/logging only + initial bool // first and last node + root bool // whether the node is root to a smaller BMT + isLeft bool // whether it is left side of the parent double segment + unbalanced bool // indicates if a node has only the left segment + parent *Node // BMT connections + state int32 // atomic increment impl concurrent boolean toggle + left, right []byte +} + +// NewNode constructor for segment hasher nodes in the BMT +func NewNode(level, index int, parent *Node) *Node { + return &Node{ + parent: parent, + level: level, + index: index, + initial: index == 0, + isLeft: index%2 == 0, + } +} + +// TreePool provides a pool of Trees used as resources by Hasher +// a Tree popped from the pool is guaranteed to have clean state +// for hashing a new chunk +// Hasher Reset releases the Tree to the pool +type TreePool struct { + lock sync.Mutex + c chan *Tree + hasher BaseHasher + SegmentSize int + SegmentCount int + Capacity int + count int +} + +// NewTreePool creates a Tree pool with hasher, segment size, segment count and capacity +// on GetTree it reuses free Trees or creates a new one if size is not reached +func NewTreePool(hasher BaseHasher, segmentCount, capacity int) *TreePool { + return &TreePool{ + c: make(chan *Tree, capacity), + hasher: hasher, + SegmentSize: hasher().Size(), + SegmentCount: segmentCount, + Capacity: capacity, + } +} + +// Drain drains the pool uptil it has no more than n resources +func (self *TreePool) Drain(n int) { + self.lock.Lock() + defer self.lock.Unlock() + for len(self.c) > n { + <-self.c + self.count-- + } +} + +// Reserve is blocking until it returns an available Tree +// it reuses free Trees or creates a new one if size is not reached +func (self *TreePool) Reserve() *Tree { + self.lock.Lock() + defer self.lock.Unlock() + var t *Tree + if self.count == self.Capacity { + return <-self.c + } + select { + case t = <-self.c: + default: + t = NewTree(self.hasher, self.SegmentSize, self.SegmentCount) + self.count++ + } + return t +} + +// Release gives back a Tree to the pool. +// This Tree is guaranteed to be in reusable state +// does not need locking +func (self *TreePool) Release(t *Tree) { + self.c <- t // can never fail but... +} + +// Tree is a reusable control structure representing a BMT +// organised in a binary tree +// Hasher uses a TreePool to pick one for each chunk hash +// the Tree is 'locked' while not in the pool +type Tree struct { + leaves []*Node +} + +// Draw draws the BMT (badly) +func (self *Tree) Draw(hash []byte, d int) string { + var left, right []string + var anc []*Node + for i, n := range self.leaves { + left = append(left, fmt.Sprintf("%v", hashstr(n.left))) + if i%2 == 0 { + anc = append(anc, n.parent) + } + right = append(right, fmt.Sprintf("%v", hashstr(n.right))) + } + anc = self.leaves + var hashes [][]string + for l := 0; len(anc) > 0; l++ { + var nodes []*Node + hash := []string{""} + for i, n := range anc { + hash = append(hash, fmt.Sprintf("%v|%v", hashstr(n.left), hashstr(n.right))) + if i%2 == 0 && n.parent != nil { + nodes = append(nodes, n.parent) + } + } + hash = append(hash, "") + hashes = append(hashes, hash) + anc = nodes + } + hashes = append(hashes, []string{"", fmt.Sprintf("%v", hashstr(hash)), ""}) + total := 60 + del := " " + var rows []string + for i := len(hashes) - 1; i >= 0; i-- { + var textlen int + hash := hashes[i] + for _, s := range hash { + textlen += len(s) + } + if total < textlen { + total = textlen + len(hash) + } + delsize := (total - textlen) / (len(hash) - 1) + if delsize > len(del) { + delsize = len(del) + } + row := fmt.Sprintf("%v: %v", len(hashes)-i-1, strings.Join(hash, del[:delsize])) + rows = append(rows, row) + + } + rows = append(rows, strings.Join(left, " ")) + rows = append(rows, strings.Join(right, " ")) + return strings.Join(rows, "\n") + "\n" +} + +// NewTree initialises the Tree by building up the nodes of a BMT +// segment size is stipulated to be the size of the hash +// segmentCount needs to be positive integer and does not need to be +// a power of two and can even be an odd number +// segmentSize * segmentCount determines the maximum chunk size +// hashed using the tree +func NewTree(hasher BaseHasher, segmentSize, segmentCount int) *Tree { + n := NewNode(0, 0, nil) + n.root = true + prevlevel := []*Node{n} + // iterate over levels and creates 2^level nodes + level := 1 + count := 2 + for d := 1; d <= depth(segmentCount); d++ { + nodes := make([]*Node, count) + for i := 0; i < len(nodes); i++ { + var parent *Node + parent = prevlevel[i/2] + t := NewNode(level, i, parent) + nodes[i] = t + } + prevlevel = nodes + level++ + count *= 2 + } + // the datanode level is the nodes on the last level where + return &Tree{ + leaves: prevlevel, + } +} + +// methods needed by hash.Hash + +// Size returns the size +func (self *Hasher) Size() int { + return self.size +} + +// BlockSize returns the block size +func (self *Hasher) BlockSize() int { + return self.blocksize +} + +// Sum returns the hash of the buffer +// hash.Hash interface Sum method appends the byte slice to the underlying +// data before it calculates and returns the hash of the chunk +func (self *Hasher) Sum(b []byte) (r []byte) { + t := self.bmt + i := self.cur + n := t.leaves[i] + j := i + // must run strictly before all nodes calculate + // datanodes are guaranteed to have a parent + if len(self.segment) > self.size && i > 0 && n.parent != nil { + n = n.parent + } else { + i *= 2 + } + d := self.finalise(n, i) + self.writeSegment(j, self.segment, d) + c := <-self.result + self.releaseTree() + + // sha3(length + BMT(pure_chunk)) + if self.blockLength == nil { + return c + } + res := self.pool.hasher() + res.Reset() + res.Write(self.blockLength) + res.Write(c) + return res.Sum(nil) +} + +// Hasher implements the SwarmHash interface + +// Hash waits for the hasher result and returns it +// caller must call this on a BMT Hasher being written to +func (self *Hasher) Hash() []byte { + return <-self.result +} + +// Hasher implements the io.Writer interface + +// Write fills the buffer to hash +// with every full segment complete launches a hasher go routine +// that shoots up the BMT +func (self *Hasher) Write(b []byte) (int, error) { + l := len(b) + if l <= 0 { + return 0, nil + } + s := self.segment + i := self.cur + count := (self.count + 1) / 2 + need := self.count*self.size - self.cur*2*self.size + size := self.size + if need > size { + size *= 2 + } + if l < need { + need = l + } + // calculate missing bit to complete current open segment + rest := size - len(s) + if need < rest { + rest = need + } + s = append(s, b[:rest]...) + need -= rest + // read full segments and the last possibly partial segment + for need > 0 && i < count-1 { + // push all finished chunks we read + self.writeSegment(i, s, self.depth) + need -= size + if need < 0 { + size += need + } + s = b[rest : rest+size] + rest += size + i++ + } + self.segment = s + self.cur = i + // otherwise, we can assume len(s) == 0, so all buffer is read and chunk is not yet full + return l, nil +} + +// Hasher implements the io.ReaderFrom interface + +// ReadFrom reads from io.Reader and appends to the data to hash using Write +// it reads so that chunk to hash is maximum length or reader reaches EOF +// caller must Reset the hasher prior to call +func (self *Hasher) ReadFrom(r io.Reader) (m int64, err error) { + bufsize := self.size*self.count - self.size*self.cur - len(self.segment) + buf := make([]byte, bufsize) + var read int + for { + var n int + n, err = r.Read(buf) + read += n + if err == io.EOF || read == len(buf) { + hash := self.Sum(buf[:n]) + if read == len(buf) { + err = NewEOC(hash) + } + break + } + if err != nil { + break + } + n, err = self.Write(buf[:n]) + if err != nil { + break + } + } + return int64(read), err +} + +// Reset needs to be called before writing to the hasher +func (self *Hasher) Reset() { + self.getTree() + self.blockLength = nil +} + +// Hasher implements the SwarmHash interface + +// ResetWithLength needs to be called before writing to the hasher +// the argument is supposed to be the byte slice binary representation of +// the legth of the data subsumed under the hash +func (self *Hasher) ResetWithLength(l []byte) { + self.Reset() + self.blockLength = l + +} + +// Release gives back the Tree to the pool whereby it unlocks +// it resets tree, segment and index +func (self *Hasher) releaseTree() { + if self.bmt != nil { + n := self.bmt.leaves[self.cur] + for ; n != nil; n = n.parent { + n.unbalanced = false + if n.parent != nil { + n.root = false + } + } + self.pool.Release(self.bmt) + self.bmt = nil + + } + self.cur = 0 + self.segment = nil +} + +func (self *Hasher) writeSegment(i int, s []byte, d int) { + h := self.pool.hasher() + n := self.bmt.leaves[i] + + if len(s) > self.size && n.parent != nil { + go func() { + h.Reset() + h.Write(s) + s = h.Sum(nil) + + if n.root { + self.result <- s + return + } + self.run(n.parent, h, d, n.index, s) + }() + return + } + go self.run(n, h, d, i*2, s) +} + +func (self *Hasher) run(n *Node, h hash.Hash, d int, i int, s []byte) { + isLeft := i%2 == 0 + for { + if isLeft { + n.left = s + } else { + n.right = s + } + if !n.unbalanced && n.toggle() { + return + } + if !n.unbalanced || !isLeft || i == 0 && d == 0 { + h.Reset() + h.Write(n.left) + h.Write(n.right) + s = h.Sum(nil) + + } else { + s = append(n.left, n.right...) + } + + self.hash = s + if n.root { + self.result <- s + return + } + + isLeft = n.isLeft + n = n.parent + i++ + } +} + +// getTree obtains a BMT resource by reserving one from the pool +func (self *Hasher) getTree() *Tree { + if self.bmt != nil { + return self.bmt + } + t := self.pool.Reserve() + self.bmt = t + return t +} + +// atomic bool toggle implementing a concurrent reusable 2-state object +// atomic addint with %2 implements atomic bool toggle +// it returns true if the toggler just put it in the active/waiting state +func (self *Node) toggle() bool { + return atomic.AddInt32(&self.state, 1)%2 == 1 +} + +func hashstr(b []byte) string { + end := len(b) + if end > 4 { + end = 4 + } + return fmt.Sprintf("%x", b[:end]) +} + +func depth(n int) (d int) { + for l := (n - 1) / 2; l > 0; l /= 2 { + d++ + } + return d +} + +// finalise is following the zigzags on the tree belonging +// to the final datasegment +func (self *Hasher) finalise(n *Node, i int) (d int) { + isLeft := i%2 == 0 + for { + // when the final segment's path is going via left segments + // the incoming data is pushed to the parent upon pulling the left + // we do not need toogle the state since this condition is + // detectable + n.unbalanced = isLeft + n.right = nil + if n.initial { + n.root = true + return d + } + isLeft = n.isLeft + n = n.parent + d++ + } +} + +// EOC (end of chunk) implements the error interface +type EOC struct { + Hash []byte // read the hash of the chunk off the error +} + +// Error returns the error string +func (self *EOC) Error() string { + return fmt.Sprintf("hasher limit reached, chunk hash: %x", self.Hash) +} + +// NewEOC creates new end of chunk error with the hash +func NewEOC(hash []byte) *EOC { + return &EOC{hash} +} diff --git a/bmt/bmt_r.go b/bmt/bmt_r.go new file mode 100644 index 0000000000..649093ee3a --- /dev/null +++ b/bmt/bmt_r.go @@ -0,0 +1,85 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// simple nonconcurrent reference implementation for hashsize segment based +// Binary Merkle tree hash on arbitrary but fixed maximum chunksize +// +// This implementation does not take advantage of any paralellisms and uses +// far more memory than necessary, but it is easy to see that it is correct. +// It can be used for generating test cases for optimized implementations. +// see testBMTHasherCorrectness function in bmt_test.go +package bmt + +import ( + "hash" +) + +// RefHasher is the non-optimized easy to read reference implementation of BMT +type RefHasher struct { + span int + section int + cap int + h hash.Hash +} + +// NewRefHasher returns a new RefHasher +func NewRefHasher(hasher BaseHasher, count int) *RefHasher { + h := hasher() + hashsize := h.Size() + maxsize := hashsize * count + c := 2 + for ; c < count; c *= 2 { + } + if c > 2 { + c /= 2 + } + return &RefHasher{ + section: 2 * hashsize, + span: c * hashsize, + cap: maxsize, + h: h, + } +} + +// Hash returns the BMT hash of the byte slice +// implements the SwarmHash interface +func (rh *RefHasher) Hash(d []byte) []byte { + if len(d) > rh.cap { + d = d[:rh.cap] + } + + return rh.hash(d, rh.span) +} + +func (rh *RefHasher) hash(d []byte, s int) []byte { + l := len(d) + left := d + var right []byte + if l > rh.section { + for ; s >= l; s /= 2 { + } + left = rh.hash(d[:s], s) + right = d[s:] + if l-s > rh.section/2 { + right = rh.hash(right, s) + } + } + defer rh.h.Reset() + rh.h.Write(left) + rh.h.Write(right) + h := rh.h.Sum(nil) + return h +} diff --git a/bmt/bmt_test.go b/bmt/bmt_test.go new file mode 100644 index 0000000000..57df83060a --- /dev/null +++ b/bmt/bmt_test.go @@ -0,0 +1,481 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package bmt + +import ( + "bytes" + crand "crypto/rand" + "fmt" + "hash" + "io" + "math/rand" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/ethereum/go-ethereum/crypto/sha3" +) + +const ( + maxproccnt = 8 +) + +// TestRefHasher tests that the RefHasher computes the expected BMT hash for +// all data lengths between 0 and 256 bytes +func TestRefHasher(t *testing.T) { + hashFunc := sha3.NewKeccak256 + + sha3 := func(data ...[]byte) []byte { + h := hashFunc() + for _, v := range data { + h.Write(v) + } + return h.Sum(nil) + } + + // the test struct is used to specify the expected BMT hash for data + // lengths between "from" and "to" + type test struct { + from int64 + to int64 + expected func([]byte) []byte + } + + var tests []*test + + // all lengths in [0,64] should be: + // + // sha3(data) + // + tests = append(tests, &test{ + from: 0, + to: 64, + expected: func(data []byte) []byte { + return sha3(data) + }, + }) + + // all lengths in [65,96] should be: + // + // sha3( + // sha3(data[:64]) + // data[64:] + // ) + // + tests = append(tests, &test{ + from: 65, + to: 96, + expected: func(data []byte) []byte { + return sha3(sha3(data[:64]), data[64:]) + }, + }) + + // all lengths in [97,128] should be: + // + // sha3( + // sha3(data[:64]) + // sha3(data[64:]) + // ) + // + tests = append(tests, &test{ + from: 97, + to: 128, + expected: func(data []byte) []byte { + return sha3(sha3(data[:64]), sha3(data[64:])) + }, + }) + + // all lengths in [129,160] should be: + // + // sha3( + // sha3( + // sha3(data[:64]) + // sha3(data[64:128]) + // ) + // data[128:] + // ) + // + tests = append(tests, &test{ + from: 129, + to: 160, + expected: func(data []byte) []byte { + return sha3(sha3(sha3(data[:64]), sha3(data[64:128])), data[128:]) + }, + }) + + // all lengths in [161,192] should be: + // + // sha3( + // sha3( + // sha3(data[:64]) + // sha3(data[64:128]) + // ) + // sha3(data[128:]) + // ) + // + tests = append(tests, &test{ + from: 161, + to: 192, + expected: func(data []byte) []byte { + return sha3(sha3(sha3(data[:64]), sha3(data[64:128])), sha3(data[128:])) + }, + }) + + // all lengths in [193,224] should be: + // + // sha3( + // sha3( + // sha3(data[:64]) + // sha3(data[64:128]) + // ) + // sha3( + // sha3(data[128:192]) + // data[192:] + // ) + // ) + // + tests = append(tests, &test{ + from: 193, + to: 224, + expected: func(data []byte) []byte { + return sha3(sha3(sha3(data[:64]), sha3(data[64:128])), sha3(sha3(data[128:192]), data[192:])) + }, + }) + + // all lengths in [225,256] should be: + // + // sha3( + // sha3( + // sha3(data[:64]) + // sha3(data[64:128]) + // ) + // sha3( + // sha3(data[128:192]) + // sha3(data[192:]) + // ) + // ) + // + tests = append(tests, &test{ + from: 225, + to: 256, + expected: func(data []byte) []byte { + return sha3(sha3(sha3(data[:64]), sha3(data[64:128])), sha3(sha3(data[128:192]), sha3(data[192:]))) + }, + }) + + // run the tests + for _, x := range tests { + for length := x.from; length <= x.to; length++ { + t.Run(fmt.Sprintf("%d_bytes", length), func(t *testing.T) { + data := make([]byte, length) + if _, err := io.ReadFull(crand.Reader, data); err != nil && err != io.EOF { + t.Fatal(err) + } + expected := x.expected(data) + actual := NewRefHasher(hashFunc, 128).Hash(data) + if !bytes.Equal(actual, expected) { + t.Fatalf("expected %x, got %x", expected, actual) + } + }) + } + } +} + +func testDataReader(l int) (r io.Reader) { + return io.LimitReader(crand.Reader, int64(l)) +} + +func TestHasherCorrectness(t *testing.T) { + err := testHasher(testBaseHasher) + if err != nil { + t.Fatal(err) + } +} + +func testHasher(f func(BaseHasher, []byte, int, int) error) error { + tdata := testDataReader(4128) + data := make([]byte, 4128) + tdata.Read(data) + hasher := sha3.NewKeccak256 + size := hasher().Size() + counts := []int{1, 2, 3, 4, 5, 8, 16, 32, 64, 128} + + var err error + for _, count := range counts { + max := count * size + incr := 1 + for n := 0; n <= max+incr; n += incr { + err = f(hasher, data, n, count) + if err != nil { + return err + } + } + } + return nil +} + +func TestHasherReuseWithoutRelease(t *testing.T) { + testHasherReuse(1, t) +} + +func TestHasherReuseWithRelease(t *testing.T) { + testHasherReuse(maxproccnt, t) +} + +func testHasherReuse(i int, t *testing.T) { + hasher := sha3.NewKeccak256 + pool := NewTreePool(hasher, 128, i) + defer pool.Drain(0) + bmt := New(pool) + + for i := 0; i < 500; i++ { + n := rand.Intn(4096) + tdata := testDataReader(n) + data := make([]byte, n) + tdata.Read(data) + + err := testHasherCorrectness(bmt, hasher, data, n, 128) + if err != nil { + t.Fatal(err) + } + } +} + +func TestHasherConcurrency(t *testing.T) { + hasher := sha3.NewKeccak256 + pool := NewTreePool(hasher, 128, maxproccnt) + defer pool.Drain(0) + wg := sync.WaitGroup{} + cycles := 100 + wg.Add(maxproccnt * cycles) + errc := make(chan error) + + for p := 0; p < maxproccnt; p++ { + for i := 0; i < cycles; i++ { + go func() { + bmt := New(pool) + n := rand.Intn(4096) + tdata := testDataReader(n) + data := make([]byte, n) + tdata.Read(data) + err := testHasherCorrectness(bmt, hasher, data, n, 128) + wg.Done() + if err != nil { + errc <- err + } + }() + } + } + go func() { + wg.Wait() + close(errc) + }() + var err error + select { + case <-time.NewTimer(5 * time.Second).C: + err = fmt.Errorf("timed out") + case err = <-errc: + } + if err != nil { + t.Fatal(err) + } +} + +func testBaseHasher(hasher BaseHasher, d []byte, n, count int) error { + pool := NewTreePool(hasher, count, 1) + defer pool.Drain(0) + bmt := New(pool) + return testHasherCorrectness(bmt, hasher, d, n, count) +} + +func testHasherCorrectness(bmt hash.Hash, hasher BaseHasher, d []byte, n, count int) (err error) { + data := d[:n] + rbmt := NewRefHasher(hasher, count) + exp := rbmt.Hash(data) + timeout := time.NewTimer(time.Second) + c := make(chan error) + + go func() { + bmt.Reset() + bmt.Write(data) + got := bmt.Sum(nil) + if !bytes.Equal(got, exp) { + c <- fmt.Errorf("wrong hash: expected %x, got %x", exp, got) + } + close(c) + }() + select { + case <-timeout.C: + err = fmt.Errorf("BMT hash calculation timed out") + case err = <-c: + } + return err +} + +func BenchmarkSHA3_4k(t *testing.B) { benchmarkSHA3(4096, t) } +func BenchmarkSHA3_2k(t *testing.B) { benchmarkSHA3(4096/2, t) } +func BenchmarkSHA3_1k(t *testing.B) { benchmarkSHA3(4096/4, t) } +func BenchmarkSHA3_512b(t *testing.B) { benchmarkSHA3(4096/8, t) } +func BenchmarkSHA3_256b(t *testing.B) { benchmarkSHA3(4096/16, t) } +func BenchmarkSHA3_128b(t *testing.B) { benchmarkSHA3(4096/32, t) } + +func BenchmarkBMTBaseline_4k(t *testing.B) { benchmarkBMTBaseline(4096, t) } +func BenchmarkBMTBaseline_2k(t *testing.B) { benchmarkBMTBaseline(4096/2, t) } +func BenchmarkBMTBaseline_1k(t *testing.B) { benchmarkBMTBaseline(4096/4, t) } +func BenchmarkBMTBaseline_512b(t *testing.B) { benchmarkBMTBaseline(4096/8, t) } +func BenchmarkBMTBaseline_256b(t *testing.B) { benchmarkBMTBaseline(4096/16, t) } +func BenchmarkBMTBaseline_128b(t *testing.B) { benchmarkBMTBaseline(4096/32, t) } + +func BenchmarkRefHasher_4k(t *testing.B) { benchmarkRefHasher(4096, t) } +func BenchmarkRefHasher_2k(t *testing.B) { benchmarkRefHasher(4096/2, t) } +func BenchmarkRefHasher_1k(t *testing.B) { benchmarkRefHasher(4096/4, t) } +func BenchmarkRefHasher_512b(t *testing.B) { benchmarkRefHasher(4096/8, t) } +func BenchmarkRefHasher_256b(t *testing.B) { benchmarkRefHasher(4096/16, t) } +func BenchmarkRefHasher_128b(t *testing.B) { benchmarkRefHasher(4096/32, t) } + +func BenchmarkHasher_4k(t *testing.B) { benchmarkHasher(4096, t) } +func BenchmarkHasher_2k(t *testing.B) { benchmarkHasher(4096/2, t) } +func BenchmarkHasher_1k(t *testing.B) { benchmarkHasher(4096/4, t) } +func BenchmarkHasher_512b(t *testing.B) { benchmarkHasher(4096/8, t) } +func BenchmarkHasher_256b(t *testing.B) { benchmarkHasher(4096/16, t) } +func BenchmarkHasher_128b(t *testing.B) { benchmarkHasher(4096/32, t) } + +func BenchmarkHasherNoReuse_4k(t *testing.B) { benchmarkHasherReuse(1, 4096, t) } +func BenchmarkHasherNoReuse_2k(t *testing.B) { benchmarkHasherReuse(1, 4096/2, t) } +func BenchmarkHasherNoReuse_1k(t *testing.B) { benchmarkHasherReuse(1, 4096/4, t) } +func BenchmarkHasherNoReuse_512b(t *testing.B) { benchmarkHasherReuse(1, 4096/8, t) } +func BenchmarkHasherNoReuse_256b(t *testing.B) { benchmarkHasherReuse(1, 4096/16, t) } +func BenchmarkHasherNoReuse_128b(t *testing.B) { benchmarkHasherReuse(1, 4096/32, t) } + +func BenchmarkHasherReuse_4k(t *testing.B) { benchmarkHasherReuse(16, 4096, t) } +func BenchmarkHasherReuse_2k(t *testing.B) { benchmarkHasherReuse(16, 4096/2, t) } +func BenchmarkHasherReuse_1k(t *testing.B) { benchmarkHasherReuse(16, 4096/4, t) } +func BenchmarkHasherReuse_512b(t *testing.B) { benchmarkHasherReuse(16, 4096/8, t) } +func BenchmarkHasherReuse_256b(t *testing.B) { benchmarkHasherReuse(16, 4096/16, t) } +func BenchmarkHasherReuse_128b(t *testing.B) { benchmarkHasherReuse(16, 4096/32, t) } + +// benchmarks the minimum hashing time for a balanced (for simplicity) BMT +// by doing count/segmentsize parallel hashings of 2*segmentsize bytes +// doing it on n maxproccnt each reusing the base hasher +// the premise is that this is the minimum computation needed for a BMT +// therefore this serves as a theoretical optimum for concurrent implementations +func benchmarkBMTBaseline(n int, t *testing.B) { + tdata := testDataReader(64) + data := make([]byte, 64) + tdata.Read(data) + hasher := sha3.NewKeccak256 + + t.ReportAllocs() + t.ResetTimer() + for i := 0; i < t.N; i++ { + count := int32((n-1)/hasher().Size() + 1) + wg := sync.WaitGroup{} + wg.Add(maxproccnt) + var i int32 + for j := 0; j < maxproccnt; j++ { + go func() { + defer wg.Done() + h := hasher() + for atomic.AddInt32(&i, 1) < count { + h.Reset() + h.Write(data) + h.Sum(nil) + } + }() + } + wg.Wait() + } +} + +func benchmarkHasher(n int, t *testing.B) { + tdata := testDataReader(n) + data := make([]byte, n) + tdata.Read(data) + + size := 1 + hasher := sha3.NewKeccak256 + segmentCount := 128 + pool := NewTreePool(hasher, segmentCount, size) + bmt := New(pool) + + t.ReportAllocs() + t.ResetTimer() + for i := 0; i < t.N; i++ { + bmt.Reset() + bmt.Write(data) + bmt.Sum(nil) + } +} + +func benchmarkHasherReuse(poolsize, n int, t *testing.B) { + tdata := testDataReader(n) + data := make([]byte, n) + tdata.Read(data) + + hasher := sha3.NewKeccak256 + segmentCount := 128 + pool := NewTreePool(hasher, segmentCount, poolsize) + cycles := 200 + + t.ReportAllocs() + t.ResetTimer() + for i := 0; i < t.N; i++ { + wg := sync.WaitGroup{} + wg.Add(cycles) + for j := 0; j < cycles; j++ { + bmt := New(pool) + go func() { + defer wg.Done() + bmt.Reset() + bmt.Write(data) + bmt.Sum(nil) + }() + } + wg.Wait() + } +} + +func benchmarkSHA3(n int, t *testing.B) { + data := make([]byte, n) + tdata := testDataReader(n) + tdata.Read(data) + hasher := sha3.NewKeccak256 + h := hasher() + + t.ReportAllocs() + t.ResetTimer() + for i := 0; i < t.N; i++ { + h.Reset() + h.Write(data) + h.Sum(nil) + } +} + +func benchmarkRefHasher(n int, t *testing.B) { + data := make([]byte, n) + tdata := testDataReader(n) + tdata.Read(data) + hasher := sha3.NewKeccak256 + rbmt := NewRefHasher(hasher, 128) + + t.ReportAllocs() + t.ResetTimer() + for i := 0; i < t.N; i++ { + rbmt.Hash(data) + } +} diff --git a/build/ci-notes.md b/build/ci-notes.md index 7574bfffa2..78e9575c01 100644 --- a/build/ci-notes.md +++ b/build/ci-notes.md @@ -21,18 +21,18 @@ variable which Travis CI makes available to certain builds. We want to build go-ethereum with the most recent version of Go, irrespective of the Go version that is available in the main Ubuntu repository. In order to make this possible, our PPA depends on the ~gophers/ubuntu/archive PPA. Our source package build-depends on -golang-1.8, which is co-installable alongside the regular golang package. PPA dependencies +golang-1.9, which is co-installable alongside the regular golang package. PPA dependencies can be edited at https://launchpad.net/%7Eethereum/+archive/ubuntu/ethereum/+edit-dependencies ## Building Packages Locally (for testing) You need to run Ubuntu to do test packaging. -Add the gophers PPA and install Go 1.8 and Debian packaging tools: +Add the gophers PPA and install Go 1.9 and Debian packaging tools: $ sudo apt-add-repository ppa:gophers/ubuntu/archive $ sudo apt-get update - $ sudo apt-get install build-essential golang-1.8 devscripts debhelper + $ sudo apt-get install build-essential golang-1.9 devscripts debhelper Create the source packages: diff --git a/build/deb.control b/build/deb.control index 7394754ef0..5c9ce6705c 100644 --- a/build/deb.control +++ b/build/deb.control @@ -2,7 +2,7 @@ Source: {{.Name}} Section: science Priority: extra Maintainer: {{.Author}} -Build-Depends: debhelper (>= 8.0.0), golang-1.8 +Build-Depends: debhelper (>= 8.0.0), golang-1.9 Standards-Version: 3.9.5 Homepage: https://ethereum.org Vcs-Git: git://github.com/ethereum/go-ethereum.git diff --git a/build/deb.rules b/build/deb.rules index b3fe5267f3..7a78525139 100644 --- a/build/deb.rules +++ b/build/deb.rules @@ -5,7 +5,7 @@ #export DH_VERBOSE=1 override_dh_auto_build: - build/env.sh /usr/lib/go-1.8/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}} + build/env.sh /usr/lib/go-1.9/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}} override_dh_auto_test: diff --git a/cmd/evm/main.go b/cmd/evm/main.go index a2e3b048e7..6c39cf8b88 100644 --- a/cmd/evm/main.go +++ b/cmd/evm/main.go @@ -143,6 +143,7 @@ func init() { compileCommand, disasmCommand, runCommand, + stateTestCommand, } } diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go new file mode 100644 index 0000000000..3a4cc51c03 --- /dev/null +++ b/cmd/evm/staterunner.go @@ -0,0 +1,119 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "os" + + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/tests" + + cli "gopkg.in/urfave/cli.v1" +) + +var stateTestCommand = cli.Command{ + Action: stateTestCmd, + Name: "statetest", + Usage: "executes the given state tests", + ArgsUsage: "", +} + +type StatetestResult struct { + Name string `json:"name"` + Pass bool `json:"pass"` + Fork string `json:"fork"` + Error string `json:"error,omitempty"` + State *state.Dump `json:"state,omitempty"` +} + +func stateTestCmd(ctx *cli.Context) error { + if len(ctx.Args().First()) == 0 { + return errors.New("path-to-test argument required") + } + // Configure the go-ethereum logger + glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false))) + glogger.Verbosity(log.Lvl(ctx.GlobalInt(VerbosityFlag.Name))) + log.Root().SetHandler(glogger) + + // Configure the EVM logger + config := &vm.LogConfig{ + DisableMemory: ctx.GlobalBool(DisableMemoryFlag.Name), + DisableStack: ctx.GlobalBool(DisableStackFlag.Name), + } + var ( + tracer vm.Tracer + debugger *vm.StructLogger + ) + switch { + case ctx.GlobalBool(MachineFlag.Name): + tracer = NewJSONLogger(config, os.Stderr) + + case ctx.GlobalBool(DebugFlag.Name): + debugger = vm.NewStructLogger(config) + tracer = debugger + + default: + debugger = vm.NewStructLogger(config) + } + // Load the test content from the input file + src, err := ioutil.ReadFile(ctx.Args().First()) + if err != nil { + return err + } + var tests map[string]tests.StateTest + if err = json.Unmarshal(src, &tests); err != nil { + return err + } + // Iterate over all the tests, run them and aggregate the results + cfg := vm.Config{ + Tracer: tracer, + Debug: ctx.GlobalBool(DebugFlag.Name) || ctx.GlobalBool(MachineFlag.Name), + } + results := make([]StatetestResult, 0, len(tests)) + for key, test := range tests { + for _, st := range test.Subtests() { + // Run the test and aggregate the result + result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true} + if state, err := test.Run(st, cfg); err != nil { + // Test failed, mark as so and dump any state to aid debugging + result.Pass, result.Error = false, err.Error() + if ctx.GlobalBool(DumpFlag.Name) && state != nil { + dump := state.RawDump() + result.State = &dump + } + } + results = append(results, *result) + + // Print any structured logs collected + if ctx.GlobalBool(DebugFlag.Name) { + if debugger != nil { + fmt.Fprintln(os.Stderr, "#### TRACE ####") + vm.WriteTrace(os.Stderr, debugger.StructLogs()) + } + } + } + } + out, _ := json.MarshalIndent(results, "", " ") + fmt.Println(string(out)) + return nil +} diff --git a/cmd/puppeth/module_ethstats.go b/cmd/puppeth/module_ethstats.go index 5d3fa5fc03..6ce662f65f 100644 --- a/cmd/puppeth/module_ethstats.go +++ b/cmd/puppeth/module_ethstats.go @@ -42,7 +42,7 @@ RUN \ WORKDIR /eth-netstats EXPOSE 3000 -RUN echo 'module.exports = {trusted: [{{.Trusted}}], banned: [{{.Banned}}]};' > lib/utils/config.js +RUN echo 'module.exports = {trusted: [{{.Trusted}}], banned: [{{.Banned}}], reserved: ["yournode"]};' > lib/utils/config.js CMD ["npm", "start"] ` diff --git a/cmd/puppeth/module_node.go b/cmd/puppeth/module_node.go index 9fe97c892d..8f912f9ebd 100644 --- a/cmd/puppeth/module_node.go +++ b/cmd/puppeth/module_node.go @@ -30,7 +30,7 @@ import ( // nodeDockerfile is the Dockerfile required to run an Ethereum node. var nodeDockerfile = ` -FROM ethereum/client-go:alpine-develop +FROM ethereum/client-go:latest ADD genesis.json /genesis.json {{if .Unlock}} @@ -38,9 +38,9 @@ ADD genesis.json /genesis.json ADD signer.pass /signer.pass {{end}} RUN \ - echo '/geth init /genesis.json' > geth.sh && \{{if .Unlock}} + echo 'geth init /genesis.json' > geth.sh && \{{if .Unlock}} echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> geth.sh && \{{end}} - echo $'/geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .BootV4}}--bootnodesv4 {{.BootV4}}{{end}} {{if .BootV5}}--bootnodesv5 {{.BootV5}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine{{end}}{{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --targetgaslimit {{.GasTarget}} --gasprice {{.GasPrice}}' >> geth.sh + echo $'geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .BootV4}}--bootnodesv4 {{.BootV4}}{{end}} {{if .BootV5}}--bootnodesv5 {{.BootV5}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine{{end}}{{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --targetgaslimit {{.GasTarget}} --gasprice {{.GasPrice}}' >> geth.sh ENTRYPOINT ["/bin/sh", "geth.sh"] ` @@ -197,7 +197,7 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error) // Container available, retrieve its node ID and its genesis json var out []byte - if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 /geth --exec admin.nodeInfo.id attach", network, kind)); err != nil { + if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 geth --exec admin.nodeInfo.id attach", network, kind)); err != nil { return nil, ErrServiceUnreachable } id := bytes.Trim(bytes.TrimSpace(out), "\"") diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 17c258c6c8..23b10c2d74 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -164,7 +164,7 @@ func ImportChain(chain *core.BlockChain, fn string) error { func hasAllBlocks(chain *core.BlockChain, bs []*types.Block) bool { for _, b := range bs { - if !chain.HasBlock(b.Hash()) { + if !chain.HasBlock(b.Hash(), b.NumberU64()) { return false } } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 37ccd06efb..5ab6047e83 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -949,10 +949,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { cfg.NetworkId = ctx.GlobalUint64(NetworkIdFlag.Name) } - // Ethereum needs to know maxPeers to calculate the light server peer ratio. - // TODO(fjl): ensure Ethereum can get MaxPeers from node. - cfg.MaxPeers = ctx.GlobalInt(MaxPeersFlag.Name) - if ctx.GlobalIsSet(CacheFlag.Name) { cfg.DatabaseCache = ctx.GlobalInt(CacheFlag.Name) } diff --git a/common/types.go b/common/types.go index eaf8352fbc..d31bbf741b 100644 --- a/common/types.go +++ b/common/types.go @@ -88,7 +88,7 @@ func (h Hash) MarshalText() ([]byte, error) { return hexutil.Bytes(h[:]).MarshalText() } -// Sets the hash to the value of b. If b is larger than len(h) it will panic +// Sets the hash to the value of b. If b is larger than len(h), 'b' will be cropped (from the left). func (h *Hash) SetBytes(b []byte) { if len(b) > len(h) { b = b[len(b)-HashLength:] @@ -97,7 +97,7 @@ func (h *Hash) SetBytes(b []byte) { copy(h[HashLength-len(b):], b) } -// Set string `s` to h. If s is larger than len(h) it will panic +// Set string `s` to h. If s is larger than len(h) s will be cropped (from left) to fit. func (h *Hash) SetString(s string) { h.SetBytes([]byte(s)) } // Sets h to other diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index b714204458..6a19d449f3 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -36,9 +36,9 @@ import ( // Ethash proof-of-work protocol constants. var ( - frontierBlockReward *big.Int = big.NewInt(5e+18) // Block reward in wei for successfully mining a block - metropolisBlockReward *big.Int = big.NewInt(3e+18) // Block reward in wei for successfully mining a block upward from Metropolis - maxUncles = 2 // Maximum number of uncles allowed in a single block + frontierBlockReward *big.Int = big.NewInt(5e+18) // Block reward in wei for successfully mining a block + byzantiumBlockReward *big.Int = big.NewInt(3e+18) // Block reward in wei for successfully mining a block upward from Byzantium + maxUncles = 2 // Maximum number of uncles allowed in a single block ) // Various error messages to mark blocks invalid. These should be private to @@ -290,8 +290,8 @@ func (ethash *Ethash) verifyHeader(chain consensus.ChainReader, header, parent * func CalcDifficulty(config *params.ChainConfig, time uint64, parent *types.Header) *big.Int { next := new(big.Int).Add(parent.Number, big1) switch { - case config.IsMetropolis(next): - return calcDifficultyMetropolis(time, parent) + case config.IsByzantium(next): + return calcDifficultyByzantium(time, parent) case config.IsHomestead(next): return calcDifficultyHomestead(time, parent) default: @@ -310,10 +310,10 @@ var ( big2999999 = big.NewInt(2999999) ) -// calcDifficultyMetropolis is the difficulty adjustment algorithm. It returns +// calcDifficultyByzantium is the difficulty adjustment algorithm. It returns // the difficulty that a new block should have when created at time given the -// parent block's time and difficulty. The calculation uses the Metropolis rules. -func calcDifficultyMetropolis(time uint64, parent *types.Header) *big.Int { +// parent block's time and difficulty. The calculation uses the Byzantium rules. +func calcDifficultyByzantium(time uint64, parent *types.Header) *big.Int { // https://github.com/ethereum/EIPs/issues/100. // algorithm: // diff = (parent_diff + @@ -530,8 +530,8 @@ var ( func AccumulateRewards(config *params.ChainConfig, state *state.StateDB, header *types.Header, uncles []*types.Header) { // Select the correct block reward based on chain progression blockReward := frontierBlockReward - if config.IsMetropolis(header.Number) { - blockReward = metropolisBlockReward + if config.IsByzantium(header.Number) { + blockReward = byzantiumBlockReward } // Accumulate the rewards for the miner and any included uncles reward := new(big.Int).Set(blockReward) diff --git a/containers/docker/master-alpine/Dockerfile b/containers/docker/master-alpine/Dockerfile index b2467f1a78..44402361d7 100644 --- a/containers/docker/master-alpine/Dockerfile +++ b/containers/docker/master-alpine/Dockerfile @@ -2,7 +2,7 @@ FROM alpine:3.5 RUN \ apk add --update go git make gcc musl-dev linux-headers ca-certificates && \ - git clone --depth 1 --branch release/1.6 https://github.com/ethereum/go-ethereum && \ + git clone --depth 1 --branch release/1.7 https://github.com/ethereum/go-ethereum && \ (cd go-ethereum && make geth) && \ cp go-ethereum/build/bin/geth /geth && \ apk del go git make gcc musl-dev linux-headers && \ diff --git a/containers/docker/master-ubuntu/Dockerfile b/containers/docker/master-ubuntu/Dockerfile index 877ae94e99..cc4fad10c4 100644 --- a/containers/docker/master-ubuntu/Dockerfile +++ b/containers/docker/master-ubuntu/Dockerfile @@ -3,7 +3,7 @@ FROM ubuntu:xenial RUN \ apt-get update && apt-get upgrade -q -y && \ apt-get install -y --no-install-recommends golang git make gcc libc-dev ca-certificates && \ - git clone --depth 1 --branch release/1.5 https://github.com/ethereum/go-ethereum && \ + git clone --depth 1 --branch release/1.7 https://github.com/ethereum/go-ethereum && \ (cd go-ethereum && make geth) && \ cp go-ethereum/build/bin/geth /geth && \ apt-get remove -y golang git make gcc libc-dev && apt autoremove -y && apt-get clean && \ diff --git a/contracts/chequebook/cheque_test.go b/contracts/chequebook/cheque_test.go index 5f6a54a1c1..b7555d0815 100644 --- a/contracts/chequebook/cheque_test.go +++ b/contracts/chequebook/cheque_test.go @@ -232,8 +232,8 @@ func TestDeposit(t *testing.T) { balance := new(big.Int).SetUint64(42) chbook.Deposit(balance) backend.Commit() - if chbook.balance.Cmp(balance) != 0 { - t.Fatalf("expected balance %v, got %v", balance, chbook.balance) + if chbook.Balance().Cmp(balance) != 0 { + t.Fatalf("expected balance %v, got %v", balance, chbook.Balance()) } amount := common.Big1 @@ -243,8 +243,8 @@ func TestDeposit(t *testing.T) { } backend.Commit() exp := new(big.Int).SetUint64(41) - if chbook.balance.Cmp(exp) != 0 { - t.Fatalf("expected balance %v, got %v", exp, chbook.balance) + if chbook.Balance().Cmp(exp) != 0 { + t.Fatalf("expected balance %v, got %v", exp, chbook.Balance()) } // autodeposit on each issue @@ -259,8 +259,8 @@ func TestDeposit(t *testing.T) { t.Fatalf("expected no error, got %v", err) } backend.Commit() - if chbook.balance.Cmp(balance) != 0 { - t.Fatalf("expected balance %v, got %v", balance, chbook.balance) + if chbook.Balance().Cmp(balance) != 0 { + t.Fatalf("expected balance %v, got %v", balance, chbook.Balance()) } // autodeposit off @@ -277,11 +277,11 @@ func TestDeposit(t *testing.T) { backend.Commit() exp = new(big.Int).SetUint64(40) - if chbook.balance.Cmp(exp) != 0 { - t.Fatalf("expected balance %v, got %v", exp, chbook.balance) + if chbook.Balance().Cmp(exp) != 0 { + t.Fatalf("expected balance %v, got %v", exp, chbook.Balance()) } - // autodeposit every 10ms if new cheque issued + // autodeposit every 30ms if new cheque issued interval := 30 * time.Millisecond chbook.AutoDeposit(interval, common.Big1, balance) _, err = chbook.Issue(addr1, amount) @@ -296,14 +296,14 @@ func TestDeposit(t *testing.T) { backend.Commit() exp = new(big.Int).SetUint64(38) - if chbook.balance.Cmp(exp) != 0 { - t.Fatalf("expected balance %v, got %v", exp, chbook.balance) + if chbook.Balance().Cmp(exp) != 0 { + t.Fatalf("expected balance %v, got %v", exp, chbook.Balance()) } time.Sleep(3 * interval) backend.Commit() - if chbook.balance.Cmp(balance) != 0 { - t.Fatalf("expected balance %v, got %v", balance, chbook.balance) + if chbook.Balance().Cmp(balance) != 0 { + t.Fatalf("expected balance %v, got %v", balance, chbook.Balance()) } exp = new(big.Int).SetUint64(40) @@ -319,8 +319,8 @@ func TestDeposit(t *testing.T) { } time.Sleep(3 * interval) backend.Commit() - if chbook.balance.Cmp(exp) != 0 { - t.Fatalf("expected balance %v, got %v", exp, chbook.balance) + if chbook.Balance().Cmp(exp) != 0 { + t.Fatalf("expected balance %v, got %v", exp, chbook.Balance()) } _, err = chbook.Issue(addr1, amount) @@ -330,8 +330,8 @@ func TestDeposit(t *testing.T) { time.Sleep(1 * interval) backend.Commit() - if chbook.balance.Cmp(balance) != 0 { - t.Fatalf("expected balance %v, got %v", balance, chbook.balance) + if chbook.Balance().Cmp(balance) != 0 { + t.Fatalf("expected balance %v, got %v", balance, chbook.Balance()) } chbook.AutoDeposit(1*interval, common.Big0, balance) @@ -352,8 +352,8 @@ func TestDeposit(t *testing.T) { backend.Commit() exp = new(big.Int).SetUint64(39) - if chbook.balance.Cmp(exp) != 0 { - t.Fatalf("expected balance %v, got %v", exp, chbook.balance) + if chbook.Balance().Cmp(exp) != 0 { + t.Fatalf("expected balance %v, got %v", exp, chbook.Balance()) } } diff --git a/core/asm/lexer.go b/core/asm/lexer.go index d784e5d504..a34b2cbd8f 100644 --- a/core/asm/lexer.go +++ b/core/asm/lexer.go @@ -145,7 +145,7 @@ func (l *lexer) ignore() { // Accepts checks whether the given input matches the next rune func (l *lexer) accept(valid string) bool { - if strings.IndexRune(valid, l.next()) >= 0 { + if strings.ContainsRune(valid, l.next()) { return true } @@ -157,7 +157,7 @@ func (l *lexer) accept(valid string) bool { // acceptRun will continue to advance the seeker until valid // can no longer be met. func (l *lexer) acceptRun(valid string) { - for strings.IndexRune(valid, l.next()) >= 0 { + for strings.ContainsRune(valid, l.next()) { } l.backup() } @@ -166,7 +166,7 @@ func (l *lexer) acceptRun(valid string) { // to advance the seeker until the rune has been found. func (l *lexer) acceptRunUntil(until rune) bool { // Continues running until a rune is found - for i := l.next(); strings.IndexRune(string(until), i) == -1; i = l.next() { + for i := l.next(); !strings.ContainsRune(string(until), i); i = l.next() { if i == 0 { return false } diff --git a/core/blockchain.go b/core/blockchain.go index f3ca4e08cf..235c1f3da2 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -23,7 +23,6 @@ import ( "io" "math/big" mrand "math/rand" - "runtime" "sync" "sync/atomic" "time" @@ -81,7 +80,6 @@ type BlockChain struct { hc *HeaderChain chainDb ethdb.Database - rmTxFeed event.Feed rmLogsFeed event.Feed chainFeed event.Feed chainSideFeed event.Feed @@ -516,10 +514,13 @@ func (bc *BlockChain) GetBodyRLP(hash common.Hash) rlp.RawValue { return body } -// HasBlock checks if a block is fully present in the database or not, caching -// it if present. -func (bc *BlockChain) HasBlock(hash common.Hash) bool { - return bc.GetBlockByHash(hash) != nil +// HasBlock checks if a block is fully present in the database or not. +func (bc *BlockChain) HasBlock(hash common.Hash, number uint64) bool { + if bc.blockCache.Contains(hash) { + return true + } + ok, _ := bc.chainDb.Has(blockBodyKey(hash, number)) + return ok } // HasBlockAndState checks if a block and associated state trie is fully present @@ -694,114 +695,73 @@ func SetReceiptsData(config *params.ChainConfig, block *types.Block, receipts ty // InsertReceiptChain attempts to complete an already existing header chain with // transaction and receipt data. -// XXX should this be moved to the test? func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain []types.Receipts) (int, error) { + bc.wg.Add(1) + defer bc.wg.Done() + // Do a sanity check that the provided chain is actually ordered and linked for i := 1; i < len(blockChain); i++ { if blockChain[i].NumberU64() != blockChain[i-1].NumberU64()+1 || blockChain[i].ParentHash() != blockChain[i-1].Hash() { - // Chain broke ancestry, log a messge (programming error) and skip insertion log.Error("Non contiguous receipt insert", "number", blockChain[i].Number(), "hash", blockChain[i].Hash(), "parent", blockChain[i].ParentHash(), "prevnumber", blockChain[i-1].Number(), "prevhash", blockChain[i-1].Hash()) - return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, blockChain[i-1].NumberU64(), blockChain[i-1].Hash().Bytes()[:4], i, blockChain[i].NumberU64(), blockChain[i].Hash().Bytes()[:4], blockChain[i].ParentHash().Bytes()[:4]) } } - // Pre-checks passed, start the block body and receipt imports - bc.wg.Add(1) - defer bc.wg.Done() - // Collect some import statistics to report on - stats := struct{ processed, ignored int32 }{} - start := time.Now() + var ( + stats = struct{ processed, ignored int32 }{} + start = time.Now() + bytes = 0 + batch = bc.chainDb.NewBatch() + ) + for i, block := range blockChain { + receipts := receiptChain[i] + // Short circuit insertion if shutting down or processing failed + if atomic.LoadInt32(&bc.procInterrupt) == 1 { + return 0, nil + } + // Short circuit if the owner header is unknown + if !bc.HasHeader(block.Hash(), block.NumberU64()) { + return i, fmt.Errorf("containing header #%d [%x…] unknown", block.Number(), block.Hash().Bytes()[:4]) + } + // Skip if the entire data is already known + if bc.HasBlock(block.Hash(), block.NumberU64()) { + stats.ignored++ + continue + } + // Compute all the non-consensus fields of the receipts + SetReceiptsData(bc.config, block, receipts) + // Write all the data out into the database + if err := WriteBody(batch, block.Hash(), block.NumberU64(), block.Body()); err != nil { + return i, fmt.Errorf("failed to write block body: %v", err) + } + if err := WriteBlockReceipts(batch, block.Hash(), block.NumberU64(), receipts); err != nil { + return i, fmt.Errorf("failed to write block receipts: %v", err) + } + if err := WriteTxLookupEntries(batch, block); err != nil { + return i, fmt.Errorf("failed to write lookup metadata: %v", err) + } + stats.processed++ - // Create the block importing task queue and worker functions - tasks := make(chan int, len(blockChain)) - for i := 0; i < len(blockChain) && i < len(receiptChain); i++ { - tasks <- i - } - close(tasks) - - errs, failed := make([]error, len(tasks)), int32(0) - process := func(worker int) { - for index := range tasks { - block, receipts := blockChain[index], receiptChain[index] - - // Short circuit insertion if shutting down or processing failed - if atomic.LoadInt32(&bc.procInterrupt) == 1 { - return + if batch.ValueSize() >= ethdb.IdealBatchSize { + if err := batch.Write(); err != nil { + return 0, err } - if atomic.LoadInt32(&failed) > 0 { - return - } - // Short circuit if the owner header is unknown - if !bc.HasHeader(block.Hash()) { - errs[index] = fmt.Errorf("containing header #%d [%x…] unknown", block.Number(), block.Hash().Bytes()[:4]) - atomic.AddInt32(&failed, 1) - return - } - // Skip if the entire data is already known - if bc.HasBlock(block.Hash()) { - atomic.AddInt32(&stats.ignored, 1) - continue - } - // Compute all the non-consensus fields of the receipts - SetReceiptsData(bc.config, block, receipts) - // Write all the data out into the database - if err := WriteBody(bc.chainDb, block.Hash(), block.NumberU64(), block.Body()); err != nil { - errs[index] = fmt.Errorf("failed to write block body: %v", err) - atomic.AddInt32(&failed, 1) - log.Crit("Failed to write block body", "err", err) - return - } - if err := WriteBlockReceipts(bc.chainDb, block.Hash(), block.NumberU64(), receipts); err != nil { - errs[index] = fmt.Errorf("failed to write block receipts: %v", err) - atomic.AddInt32(&failed, 1) - log.Crit("Failed to write block receipts", "err", err) - return - } - if err := WriteMipmapBloom(bc.chainDb, block.NumberU64(), receipts); err != nil { - errs[index] = fmt.Errorf("failed to write log blooms: %v", err) - atomic.AddInt32(&failed, 1) - log.Crit("Failed to write log blooms", "err", err) - return - } - if err := WriteTxLookupEntries(bc.chainDb, block); err != nil { - errs[index] = fmt.Errorf("failed to write lookup metadata: %v", err) - atomic.AddInt32(&failed, 1) - log.Crit("Failed to write lookup metadata", "err", err) - return - } - atomic.AddInt32(&stats.processed, 1) + bytes += batch.ValueSize() + batch = bc.chainDb.NewBatch() } } - // Start as many worker threads as goroutines allowed - pending := new(sync.WaitGroup) - for i := 0; i < runtime.GOMAXPROCS(0); i++ { - pending.Add(1) - go func(id int) { - defer pending.Done() - process(id) - }(i) - } - pending.Wait() - - // If anything failed, report - if failed > 0 { - for i, err := range errs { - if err != nil { - return i, err - } + if batch.ValueSize() > 0 { + bytes += batch.ValueSize() + if err := batch.Write(); err != nil { + return 0, err } } - if atomic.LoadInt32(&bc.procInterrupt) == 1 { - log.Debug("Premature abort during receipts processing") - return 0, nil - } + // Update the head fast sync block if better bc.mu.Lock() - - head := blockChain[len(errs)-1] + head := blockChain[len(blockChain)-1] if td := bc.GetTd(head.Hash(), head.NumberU64()); td != nil { // Rewind may have occurred, skip in that case if bc.GetTd(bc.currentFastBlock.Hash(), bc.currentFastBlock.NumberU64()).Cmp(td) < 0 { if err := WriteHeadFastBlockHash(bc.chainDb, head.Hash()); err != nil { @@ -812,16 +772,18 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ } bc.mu.Unlock() - // Report some public statistics so the user has a clue what's going on - last := blockChain[len(blockChain)-1] - log.Info("Imported new block receipts", "count", stats.processed, "elapsed", common.PrettyDuration(time.Since(start)), - "number", last.Number(), "hash", last.Hash(), "ignored", stats.ignored) - + log.Info("Imported new block receipts", + "count", stats.processed, + "elapsed", common.PrettyDuration(time.Since(start)), + "bytes", bytes, + "number", head.Number(), + "hash", head.Hash(), + "ignored", stats.ignored) return 0, nil } // WriteBlock writes the block to the chain. -func (bc *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err error) { +func (bc *BlockChain) WriteBlockAndState(block *types.Block, receipts []*types.Receipt, state *state.StateDB) (status WriteStatus, err error) { bc.wg.Add(1) defer bc.wg.Done() @@ -834,7 +796,7 @@ func (bc *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err er bc.mu.Lock() defer bc.mu.Unlock() - if bc.HasBlock(block.Hash()) { + if bc.HasBlock(block.Hash(), block.NumberU64()) { log.Trace("Block existed", "hash", block.Hash()) return } @@ -844,10 +806,18 @@ func (bc *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err er // Irrelevant of the canonical status, write the block itself to the database if err := bc.hc.WriteTd(block.Hash(), block.NumberU64(), externTd); err != nil { - log.Crit("Failed to write block total difficulty", "err", err) + return NonStatTy, err } - if err := WriteBlock(bc.chainDb, block); err != nil { - log.Crit("Failed to write block contents", "err", err) + // Write other block data using a batch. + batch := bc.chainDb.NewBatch() + if err := WriteBlock(batch, block); err != nil { + return NonStatTy, err + } + if _, err := state.CommitTo(batch, bc.config.IsEIP158(block.Number())); err != nil { + return NonStatTy, err + } + if err := WriteBlockReceipts(batch, block.Hash(), block.NumberU64(), receipts); err != nil { + return NonStatTy, err } // If the total difficulty is higher than our known, add it to the canonical chain @@ -860,20 +830,46 @@ func (bc *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err er return NonStatTy, err } } - bc.insert(block) // Insert the block as the new head of the chain + // Write the positional metadata for transaction and receipt lookups + if err := WriteTxLookupEntries(batch, block); err != nil { + return NonStatTy, err + } + // Write hash preimages + if err := WritePreimages(bc.chainDb, block.NumberU64(), state.Preimages()); err != nil { + return NonStatTy, err + } status = CanonStatTy } else { status = SideStatTy } + if err := batch.Write(); err != nil { + return NonStatTy, err + } + // Set new head. + if status == CanonStatTy { + bc.insert(block) + } bc.futureBlocks.Remove(block.Hash()) - - return + return status, nil } -// InsertChain will attempt to insert the given chain in to the canonical chain or, otherwise, create a fork. If an error is returned -// it will return the index number of the failing block as well an error describing what went wrong (for possible errors see core/errors.go). +// InsertChain attempts to insert the given batch of blocks in to the canonical +// chain or, otherwise, create a fork. If an error is returned it will return +// the index number of the failing block as well an error describing what went +// wrong. +// +// After insertion is done, all accumulated events will be fired. func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { + n, events, logs, err := bc.insertChain(chain) + bc.PostChainEvents(events, logs) + return n, err +} + +// insertChain will execute the actual chain insertion and event aggregation. The +// only reason this method exists as a separate one is to make locking cleaner +// with deferred statements. +func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*types.Log, error) { // Do a sanity check that the provided chain is actually ordered and linked for i := 1; i < len(chain); i++ { if chain[i].NumberU64() != chain[i-1].NumberU64()+1 || chain[i].ParentHash() != chain[i-1].Hash() { @@ -881,7 +877,7 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { log.Error("Non contiguous block insert", "number", chain[i].Number(), "hash", chain[i].Hash(), "parent", chain[i].ParentHash(), "prevnumber", chain[i-1].Number(), "prevhash", chain[i-1].Hash()) - return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, chain[i-1].NumberU64(), + return 0, nil, nil, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, chain[i-1].NumberU64(), chain[i-1].Hash().Bytes()[:4], i, chain[i].NumberU64(), chain[i].Hash().Bytes()[:4], chain[i].ParentHash().Bytes()[:4]) } } @@ -898,6 +894,7 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { var ( stats = insertStats{startTime: mclock.Now()} events = make([]interface{}, 0, len(chain)) + lastCanon *types.Block coalescedLogs []*types.Log ) // Start the parallel header verifier @@ -921,7 +918,7 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { // If the header is a banned one, straight out abort if BadHashes[block.Hash()] { bc.reportBlock(block, nil, ErrBlacklistedHash) - return i, ErrBlacklistedHash + return i, events, coalescedLogs, ErrBlacklistedHash } // Wait for the block's verification to complete bstart := time.Now() @@ -942,7 +939,7 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { // if given. max := big.NewInt(time.Now().Unix() + maxTimeFutureBlocks) if block.Time().Cmp(max) > 0 { - return i, fmt.Errorf("future block: %v > %v", block.Time(), max) + return i, events, coalescedLogs, fmt.Errorf("future block: %v > %v", block.Time(), max) } bc.futureBlocks.Add(block.Hash(), block) stats.queued++ @@ -956,7 +953,7 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { } bc.reportBlock(block, nil, err) - return i, err + return i, events, coalescedLogs, err } // Create a new statedb using the parent block and report an // error if it fails. @@ -968,64 +965,35 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { } state, err := state.New(parent.Root(), bc.stateCache) if err != nil { - return i, err + return i, events, coalescedLogs, err } // Process block using the parent state as reference point. receipts, logs, usedGas, err := bc.processor.Process(block, state, bc.vmConfig) if err != nil { bc.reportBlock(block, receipts, err) - return i, err + return i, events, coalescedLogs, err } // Validate the state using the default validator err = bc.Validator().ValidateState(block, parent, state, receipts, usedGas) if err != nil { bc.reportBlock(block, receipts, err) - return i, err + return i, events, coalescedLogs, err } - // Write state changes to database - if _, err = state.CommitTo(bc.chainDb, bc.config.IsEIP158(block.Number())); err != nil { - return i, err - } - - // coalesce logs for later processing - coalescedLogs = append(coalescedLogs, logs...) - - if err = WriteBlockReceipts(bc.chainDb, block.Hash(), block.NumberU64(), receipts); err != nil { - return i, err - } - - // write the block to the chain and get the status - status, err := bc.WriteBlock(block) + // Write the block to the chain and get the status. + status, err := bc.WriteBlockAndState(block, receipts, state) if err != nil { - return i, err + return i, events, coalescedLogs, err } - switch status { case CanonStatTy: log.Debug("Inserted new block", "number", block.Number(), "hash", block.Hash(), "uncles", len(block.Uncles()), "txs", len(block.Transactions()), "gas", block.GasUsed(), "elapsed", common.PrettyDuration(time.Since(bstart))) + coalescedLogs = append(coalescedLogs, logs...) blockInsertTimer.UpdateSince(bstart) events = append(events, ChainEvent{block, block.Hash(), logs}) - // We need some control over the mining operation. Acquiring locks and waiting - // for the miner to create new block takes too long and in most cases isn't - // even necessary. - if bc.LastBlockHash() == block.Hash() { - events = append(events, ChainHeadEvent{block}) - } + lastCanon = block - // Write the positional metadata for transaction and receipt lookups - if err := WriteTxLookupEntries(bc.chainDb, block); err != nil { - return i, err - } - // Write map map bloom filters - if err := WriteMipmapBloom(bc.chainDb, block.NumberU64(), receipts); err != nil { - return i, err - } - // Write hash preimages - if err := WritePreimages(bc.chainDb, block.NumberU64(), state.Preimages()); err != nil { - return i, err - } case SideStatTy: log.Debug("Inserted forked block", "number", block.Number(), "hash", block.Hash(), "diff", block.Difficulty(), "elapsed", common.PrettyDuration(time.Since(bstart)), "txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles())) @@ -1037,9 +1005,11 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { stats.usedGas += usedGas.Uint64() stats.report(chain, i) } - go bc.PostChainEvents(events, coalescedLogs) - - return 0, nil + // Append a single chain head event if we've progressed the chain + if lastCanon != nil && bc.LastBlockHash() == lastCanon.Hash() { + events = append(events, ChainHeadEvent{lastCanon}) + } + return 0, events, coalescedLogs, nil } // insertStats tracks and reports on block insertion. @@ -1179,11 +1149,6 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { if err := WriteTxLookupEntries(bc.chainDb, block); err != nil { return err } - // Write map map bloom filters - receipts := GetBlockReceipts(bc.chainDb, block.Hash(), block.NumberU64()) - if err := WriteMipmapBloom(bc.chainDb, block.NumberU64(), receipts); err != nil { - return err - } addedTxs = append(addedTxs, block.Transactions()...) } @@ -1194,15 +1159,9 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { for _, tx := range diff { DeleteTxLookupEntry(bc.chainDb, tx.Hash()) } - // Must be posted in a goroutine because of the transaction pool trying - // to acquire the chain manager lock - if len(diff) > 0 { - go bc.rmTxFeed.Send(RemovedTransactionEvent{diff}) - } if len(deletedLogs) > 0 { go bc.rmLogsFeed.Send(RemovedLogsEvent{deletedLogs}) } - if len(oldChain) > 0 { go func() { for _, block := range oldChain { @@ -1379,8 +1338,8 @@ func (bc *BlockChain) GetHeaderByHash(hash common.Hash) *types.Header { // HasHeader checks if a block header is present in the database or not, caching // it if present. -func (bc *BlockChain) HasHeader(hash common.Hash) bool { - return bc.hc.HasHeader(hash) +func (bc *BlockChain) HasHeader(hash common.Hash, number uint64) bool { + return bc.hc.HasHeader(hash, number) } // GetBlockHashesFromHash retrieves a number of block hashes starting at a given @@ -1401,11 +1360,6 @@ func (bc *BlockChain) Config() *params.ChainConfig { return bc.config } // Engine retrieves the blockchain's consensus engine. func (bc *BlockChain) Engine() consensus.Engine { return bc.engine } -// SubscribeRemovedTxEvent registers a subscription of RemovedTransactionEvent. -func (bc *BlockChain) SubscribeRemovedTxEvent(ch chan<- RemovedTransactionEvent) event.Subscription { - return bc.scope.Track(bc.rmTxFeed.Subscribe(ch)) -} - // SubscribeRemovedLogsEvent registers a subscription of RemovedLogsEvent. func (bc *BlockChain) SubscribeRemovedLogsEvent(ch chan<- RemovedLogsEvent) event.Subscription { return bc.scope.Track(bc.rmLogsFeed.Subscribe(ch)) diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 470974a0a7..cb1df8d4b0 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -928,14 +928,14 @@ func TestReorgSideEvent(t *testing.T) { replacementBlocks, _ := GenerateChain(gspec.Config, genesis, db, 4, func(i int, gen *BlockGen) { tx, err := types.SignTx(types.NewContractCreation(gen.TxNonce(addr1), new(big.Int), big.NewInt(1000000), new(big.Int), nil), signer, key1) if i == 2 { - gen.OffsetTime(-1) + gen.OffsetTime(-9) } if err != nil { t.Fatalf("failed to create tx: %v", err) } gen.AddTx(tx) }) - chainSideCh := make(chan ChainSideEvent) + chainSideCh := make(chan ChainSideEvent, 64) blockchain.SubscribeChainSideEvent(chainSideCh) if _, err := blockchain.InsertChain(replacementBlocks); err != nil { t.Fatalf("failed to insert chain: %v", err) diff --git a/core/bloombits/doc.go b/core/bloombits/doc.go new file mode 100644 index 0000000000..3d159e74f7 --- /dev/null +++ b/core/bloombits/doc.go @@ -0,0 +1,18 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package bloombits implements bloom filtering on batches of data. +package bloombits diff --git a/core/bloombits/generator.go b/core/bloombits/generator.go new file mode 100644 index 0000000000..540085450d --- /dev/null +++ b/core/bloombits/generator.go @@ -0,0 +1,87 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package bloombits + +import ( + "errors" + + "github.com/ethereum/go-ethereum/core/types" +) + +// errSectionOutOfBounds is returned if the user tried to add more bloom filters +// to the batch than available space, or if tries to retrieve above the capacity, +var errSectionOutOfBounds = errors.New("section out of bounds") + +// Generator takes a number of bloom filters and generates the rotated bloom bits +// to be used for batched filtering. +type Generator struct { + blooms [types.BloomBitLength][]byte // Rotated blooms for per-bit matching + sections uint // Number of sections to batch together + nextBit uint // Next bit to set when adding a bloom +} + +// NewGenerator creates a rotated bloom generator that can iteratively fill a +// batched bloom filter's bits. +func NewGenerator(sections uint) (*Generator, error) { + if sections%8 != 0 { + return nil, errors.New("section count not multiple of 8") + } + b := &Generator{sections: sections} + for i := 0; i < types.BloomBitLength; i++ { + b.blooms[i] = make([]byte, sections/8) + } + return b, nil +} + +// AddBloom takes a single bloom filter and sets the corresponding bit column +// in memory accordingly. +func (b *Generator) AddBloom(index uint, bloom types.Bloom) error { + // Make sure we're not adding more bloom filters than our capacity + if b.nextBit >= b.sections { + return errSectionOutOfBounds + } + if b.nextBit != index { + return errors.New("bloom filter with unexpected index") + } + // Rotate the bloom and insert into our collection + byteIndex := b.nextBit / 8 + bitMask := byte(1) << byte(7-b.nextBit%8) + + for i := 0; i < types.BloomBitLength; i++ { + bloomByteIndex := types.BloomByteLength - 1 - i/8 + bloomBitMask := byte(1) << byte(i%8) + + if (bloom[bloomByteIndex] & bloomBitMask) != 0 { + b.blooms[i][byteIndex] |= bitMask + } + } + b.nextBit++ + + return nil +} + +// Bitset returns the bit vector belonging to the given bit index after all +// blooms have been added. +func (b *Generator) Bitset(idx uint) ([]byte, error) { + if b.nextBit != b.sections { + return nil, errors.New("bloom not fully generated yet") + } + if idx >= b.sections { + return nil, errSectionOutOfBounds + } + return b.blooms[idx], nil +} diff --git a/core/bloombits/generator_test.go b/core/bloombits/generator_test.go new file mode 100644 index 0000000000..f9bcef96e0 --- /dev/null +++ b/core/bloombits/generator_test.go @@ -0,0 +1,60 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package bloombits + +import ( + "bytes" + "math/rand" + "testing" + + "github.com/ethereum/go-ethereum/core/types" +) + +// Tests that batched bloom bits are correctly rotated from the input bloom +// filters. +func TestGenerator(t *testing.T) { + // Generate the input and the rotated output + var input, output [types.BloomBitLength][types.BloomByteLength]byte + + for i := 0; i < types.BloomBitLength; i++ { + for j := 0; j < types.BloomBitLength; j++ { + bit := byte(rand.Int() % 2) + + input[i][j/8] |= bit << byte(7-j%8) + output[types.BloomBitLength-1-j][i/8] |= bit << byte(7-i%8) + } + } + // Crunch the input through the generator and verify the result + gen, err := NewGenerator(types.BloomBitLength) + if err != nil { + t.Fatalf("failed to create bloombit generator: %v", err) + } + for i, bloom := range input { + if err := gen.AddBloom(uint(i), bloom); err != nil { + t.Fatalf("bloom %d: failed to add: %v", i, err) + } + } + for i, want := range output { + have, err := gen.Bitset(uint(i)) + if err != nil { + t.Fatalf("output %d: failed to retrieve bits: %v", i, err) + } + if !bytes.Equal(have, want[:]) { + t.Errorf("output %d: bit vector mismatch have %x, want %x", i, have, want) + } + } +} diff --git a/core/bloombits/matcher.go b/core/bloombits/matcher.go new file mode 100644 index 0000000000..f3ed405a61 --- /dev/null +++ b/core/bloombits/matcher.go @@ -0,0 +1,615 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package bloombits + +import ( + "bytes" + "errors" + "math" + "sort" + "sync" + "sync/atomic" + "time" + + "github.com/ethereum/go-ethereum/common/bitutil" + "github.com/ethereum/go-ethereum/crypto" +) + +// bloomIndexes represents the bit indexes inside the bloom filter that belong +// to some key. +type bloomIndexes [3]uint + +// calcBloomIndexes returns the bloom filter bit indexes belonging to the given key. +func calcBloomIndexes(b []byte) bloomIndexes { + b = crypto.Keccak256(b) + + var idxs bloomIndexes + for i := 0; i < len(idxs); i++ { + idxs[i] = (uint(b[2*i])<<8)&2047 + uint(b[2*i+1]) + } + return idxs +} + +// partialMatches with a non-nil vector represents a section in which some sub- +// matchers have already found potential matches. Subsequent sub-matchers will +// binary AND their matches with this vector. If vector is nil, it represents a +// section to be processed by the first sub-matcher. +type partialMatches struct { + section uint64 + bitset []byte +} + +// Retrieval represents a request for retrieval task assignments for a given +// bit with the given number of fetch elements, or a response for such a request. +// It can also have the actual results set to be used as a delivery data struct. +type Retrieval struct { + Bit uint + Sections []uint64 + Bitsets [][]byte +} + +// Matcher is a pipelined system of schedulers and logic matchers which perform +// binary AND/OR operations on the bit-streams, creating a stream of potential +// blocks to inspect for data content. +type Matcher struct { + sectionSize uint64 // Size of the data batches to filter on + + filters [][]bloomIndexes // Filter the system is matching for + schedulers map[uint]*scheduler // Retrieval schedulers for loading bloom bits + + retrievers chan chan uint // Retriever processes waiting for bit allocations + counters chan chan uint // Retriever processes waiting for task count reports + retrievals chan chan *Retrieval // Retriever processes waiting for task allocations + deliveries chan *Retrieval // Retriever processes waiting for task response deliveries + + running uint32 // Atomic flag whether a session is live or not +} + +// NewMatcher creates a new pipeline for retrieving bloom bit streams and doing +// address and topic filtering on them. +func NewMatcher(sectionSize uint64, filters [][][]byte) *Matcher { + // Create the matcher instance + m := &Matcher{ + sectionSize: sectionSize, + schedulers: make(map[uint]*scheduler), + retrievers: make(chan chan uint), + counters: make(chan chan uint), + retrievals: make(chan chan *Retrieval), + deliveries: make(chan *Retrieval), + } + // Calculate the bloom bit indexes for the groups we're interested in + m.filters = nil + + for _, filter := range filters { + bloomBits := make([]bloomIndexes, len(filter)) + for i, clause := range filter { + bloomBits[i] = calcBloomIndexes(clause) + } + m.filters = append(m.filters, bloomBits) + } + // For every bit, create a scheduler to load/download the bit vectors + for _, bloomIndexLists := range m.filters { + for _, bloomIndexList := range bloomIndexLists { + for _, bloomIndex := range bloomIndexList { + m.addScheduler(bloomIndex) + } + } + } + return m +} + +// addScheduler adds a bit stream retrieval scheduler for the given bit index if +// it has not existed before. If the bit is already selected for filtering, the +// existing scheduler can be used. +func (m *Matcher) addScheduler(idx uint) { + if _, ok := m.schedulers[idx]; ok { + return + } + m.schedulers[idx] = newScheduler(idx) +} + +// Start starts the matching process and returns a stream of bloom matches in +// a given range of blocks. If there are no more matches in the range, the result +// channel is closed. +func (m *Matcher) Start(begin, end uint64, results chan uint64) (*MatcherSession, error) { + // Make sure we're not creating concurrent sessions + if atomic.SwapUint32(&m.running, 1) == 1 { + return nil, errors.New("matcher already running") + } + defer atomic.StoreUint32(&m.running, 0) + + // Initiate a new matching round + session := &MatcherSession{ + matcher: m, + quit: make(chan struct{}), + kill: make(chan struct{}), + } + for _, scheduler := range m.schedulers { + scheduler.reset() + } + sink := m.run(begin, end, cap(results), session) + + // Read the output from the result sink and deliver to the user + session.pend.Add(1) + go func() { + defer session.pend.Done() + defer close(results) + + for { + select { + case <-session.quit: + return + + case res, ok := <-sink: + // New match result found + if !ok { + return + } + // Calculate the first and last blocks of the section + sectionStart := res.section * m.sectionSize + + first := sectionStart + if begin > first { + first = begin + } + last := sectionStart + m.sectionSize - 1 + if end < last { + last = end + } + // Iterate over all the blocks in the section and return the matching ones + for i := first; i <= last; i++ { + // Skip the entire byte if no matches are found inside + next := res.bitset[(i-sectionStart)/8] + if next == 0 { + i += 7 + continue + } + // Some bit it set, do the actual submatching + if bit := 7 - i%8; next&(1<= req.section }) + requests[req.bit] = append(queue[:index], append([]uint64{req.section}, queue[index:]...)...) + + // If it's a new bit and we have waiting fetchers, allocate to them + if len(queue) == 0 { + assign(req.bit) + } + + case fetcher := <-retrievers: + // New retriever arrived, find the lowest section-ed bit to assign + bit, best := uint(0), uint64(math.MaxUint64) + for idx := range unallocs { + if requests[idx][0] < best { + bit, best = idx, requests[idx][0] + } + } + // Stop tracking this bit (and alloc notifications if no more work is available) + delete(unallocs, bit) + if len(unallocs) == 0 { + retrievers = nil + } + allocs++ + fetcher <- bit + + case fetcher := <-m.counters: + // New task count request arrives, return number of items + fetcher <- uint(len(requests[<-fetcher])) + + case fetcher := <-m.retrievals: + // New fetcher waiting for tasks to retrieve, assign + task := <-fetcher + if want := len(task.Sections); want >= len(requests[task.Bit]) { + task.Sections = requests[task.Bit] + delete(requests, task.Bit) + } else { + task.Sections = append(task.Sections[:0], requests[task.Bit][:want]...) + requests[task.Bit] = append(requests[task.Bit][:0], requests[task.Bit][want:]...) + } + fetcher <- task + + // If anything was left unallocated, try to assign to someone else + if len(requests[task.Bit]) > 0 { + assign(task.Bit) + } + + case result := <-m.deliveries: + // New retrieval task response from fetcher, split out missing sections and + // deliver complete ones + var ( + sections = make([]uint64, 0, len(result.Sections)) + bitsets = make([][]byte, 0, len(result.Bitsets)) + missing = make([]uint64, 0, len(result.Sections)) + ) + for i, bitset := range result.Bitsets { + if len(bitset) == 0 { + missing = append(missing, result.Sections[i]) + continue + } + sections = append(sections, result.Sections[i]) + bitsets = append(bitsets, bitset) + } + m.schedulers[result.Bit].deliver(sections, bitsets) + allocs-- + + // Reschedule missing sections and allocate bit if newly available + if len(missing) > 0 { + queue := requests[result.Bit] + for _, section := range missing { + index := sort.Search(len(queue), func(i int) bool { return queue[i] >= section }) + queue = append(queue[:index], append([]uint64{section}, queue[index:]...)...) + } + requests[result.Bit] = queue + + if len(queue) == len(missing) { + assign(result.Bit) + } + } + // If we're in the process of shutting down, terminate + if allocs == 0 && shutdown == nil { + return + } + } + } +} + +// MatcherSession is returned by a started matcher to be used as a terminator +// for the actively running matching operation. +type MatcherSession struct { + matcher *Matcher + + quit chan struct{} // Quit channel to request pipeline termination + kill chan struct{} // Term channel to signal non-graceful forced shutdown + pend sync.WaitGroup +} + +// Close stops the matching process and waits for all subprocesses to terminate +// before returning. The timeout may be used for graceful shutdown, allowing the +// currently running retrievals to complete before this time. +func (s *MatcherSession) Close(timeout time.Duration) { + // Bail out if the matcher is not running + select { + case <-s.quit: + return + default: + } + // Signal termination and wait for all goroutines to tear down + close(s.quit) + time.AfterFunc(timeout, func() { close(s.kill) }) + s.pend.Wait() +} + +// AllocateRetrieval assigns a bloom bit index to a client process that can either +// immediately reuest and fetch the section contents assigned to this bit or wait +// a little while for more sections to be requested. +func (s *MatcherSession) AllocateRetrieval() (uint, bool) { + fetcher := make(chan uint) + + select { + case <-s.quit: + return 0, false + case s.matcher.retrievers <- fetcher: + bit, ok := <-fetcher + return bit, ok + } +} + +// PendingSections returns the number of pending section retrievals belonging to +// the given bloom bit index. +func (s *MatcherSession) PendingSections(bit uint) int { + fetcher := make(chan uint) + + select { + case <-s.quit: + return 0 + case s.matcher.counters <- fetcher: + fetcher <- bit + return int(<-fetcher) + } +} + +// AllocateSections assigns all or part of an already allocated bit-task queue +// to the requesting process. +func (s *MatcherSession) AllocateSections(bit uint, count int) []uint64 { + fetcher := make(chan *Retrieval) + + select { + case <-s.quit: + return nil + case s.matcher.retrievals <- fetcher: + task := &Retrieval{ + Bit: bit, + Sections: make([]uint64, count), + } + fetcher <- task + return (<-fetcher).Sections + } +} + +// DeliverSections delivers a batch of section bit-vectors for a specific bloom +// bit index to be injected into the processing pipeline. +func (s *MatcherSession) DeliverSections(bit uint, sections []uint64, bitsets [][]byte) { + select { + case <-s.kill: + return + case s.matcher.deliveries <- &Retrieval{Bit: bit, Sections: sections, Bitsets: bitsets}: + } +} + +// Multiplex polls the matcher session for rerieval tasks and multiplexes it into +// the reuested retrieval queue to be serviced together with other sessions. +// +// This method will block for the lifetime of the session. Even after termination +// of the session, any request in-flight need to be responded to! Empty responses +// are fine though in that case. +func (s *MatcherSession) Multiplex(batch int, wait time.Duration, mux chan chan *Retrieval) { + for { + // Allocate a new bloom bit index to retrieve data for, stopping when done + bit, ok := s.AllocateRetrieval() + if !ok { + return + } + // Bit allocated, throttle a bit if we're below our batch limit + if s.PendingSections(bit) < batch { + select { + case <-s.quit: + // Session terminating, we can't meaningfully service, abort + s.AllocateSections(bit, 0) + s.DeliverSections(bit, []uint64{}, [][]byte{}) + return + + case <-time.After(wait): + // Throttling up, fetch whatever's available + } + } + // Allocate as much as we can handle and request servicing + sections := s.AllocateSections(bit, batch) + request := make(chan *Retrieval) + + select { + case <-s.quit: + // Session terminating, we can't meaningfully service, abort + s.DeliverSections(bit, sections, make([][]byte, len(sections))) + return + + case mux <- request: + // Retrieval accepted, something must arrive before we're aborting + request <- &Retrieval{Bit: bit, Sections: sections} + + result := <-request + s.DeliverSections(result.Bit, result.Sections, result.Bitsets) + } + } +} diff --git a/core/bloombits/matcher_test.go b/core/bloombits/matcher_test.go new file mode 100644 index 0000000000..f0198c4e38 --- /dev/null +++ b/core/bloombits/matcher_test.go @@ -0,0 +1,242 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package bloombits + +import ( + "math/rand" + "sync/atomic" + "testing" + "time" +) + +const testSectionSize = 4096 + +// Tests the matcher pipeline on a single continuous workflow without interrupts. +func TestMatcherContinuous(t *testing.T) { + testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 100000, false, 75) + testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 100000, false, 81) + testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 10000, false, 36) +} + +// Tests the matcher pipeline on a constantly interrupted and resumed work pattern +// with the aim of ensuring data items are requested only once. +func TestMatcherIntermittent(t *testing.T) { + testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 100000, true, 75) + testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 100000, true, 81) + testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 10000, true, 36) +} + +// Tests the matcher pipeline on random input to hopefully catch anomalies. +func TestMatcherRandom(t *testing.T) { + for i := 0; i < 10; i++ { + testMatcherBothModes(t, makeRandomIndexes([]int{1}, 50), 10000, 0) + testMatcherBothModes(t, makeRandomIndexes([]int{3}, 50), 10000, 0) + testMatcherBothModes(t, makeRandomIndexes([]int{2, 2, 2}, 20), 10000, 0) + testMatcherBothModes(t, makeRandomIndexes([]int{5, 5, 5}, 50), 10000, 0) + testMatcherBothModes(t, makeRandomIndexes([]int{4, 4, 4}, 20), 10000, 0) + } +} + +// Tests that matching on everything doesn't crash (special case internally). +func TestWildcardMatcher(t *testing.T) { + testMatcherBothModes(t, nil, 10000, 0) +} + +// makeRandomIndexes generates a random filter system, composed on multiple filter +// criteria, each having one bloom list component for the address and arbitrarilly +// many topic bloom list components. +func makeRandomIndexes(lengths []int, max int) [][]bloomIndexes { + res := make([][]bloomIndexes, len(lengths)) + for i, topics := range lengths { + res[i] = make([]bloomIndexes, topics) + for j := 0; j < topics; j++ { + for k := 0; k < len(res[i][j]); k++ { + res[i][j][k] = uint(rand.Intn(max-1) + 2) + } + } + } + return res +} + +// testMatcherDiffBatches runs the given matches test in single-delivery and also +// in batches delivery mode, verifying that all kinds of deliveries are handled +// correctly withn. +func testMatcherDiffBatches(t *testing.T, filter [][]bloomIndexes, blocks uint64, intermittent bool, retrievals uint32) { + singleton := testMatcher(t, filter, blocks, intermittent, retrievals, 1) + batched := testMatcher(t, filter, blocks, intermittent, retrievals, 16) + + if singleton != batched { + t.Errorf("filter = %v blocks = %v intermittent = %v: request count mismatch, %v in signleton vs. %v in batched mode", filter, blocks, intermittent, singleton, batched) + } +} + +// testMatcherBothModes runs the given matcher test in both continuous as well as +// in intermittent mode, verifying that the request counts match each other. +func testMatcherBothModes(t *testing.T, filter [][]bloomIndexes, blocks uint64, retrievals uint32) { + continuous := testMatcher(t, filter, blocks, false, retrievals, 16) + intermittent := testMatcher(t, filter, blocks, true, retrievals, 16) + + if continuous != intermittent { + t.Errorf("filter = %v blocks = %v: request count mismatch, %v in continuous vs. %v in intermittent mode", filter, blocks, continuous, intermittent) + } +} + +// testMatcher is a generic tester to run the given matcher test and return the +// number of requests made for cross validation between different modes. +func testMatcher(t *testing.T, filter [][]bloomIndexes, blocks uint64, intermittent bool, retrievals uint32, maxReqCount int) uint32 { + // Create a new matcher an simulate our explicit random bitsets + matcher := NewMatcher(testSectionSize, nil) + matcher.filters = filter + + for _, rule := range filter { + for _, topic := range rule { + for _, bit := range topic { + matcher.addScheduler(bit) + } + } + } + // Track the number of retrieval requests made + var requested uint32 + + // Start the matching session for the filter and the retriver goroutines + quit := make(chan struct{}) + matches := make(chan uint64, 16) + + session, err := matcher.Start(0, blocks-1, matches) + if err != nil { + t.Fatalf("failed to stat matcher session: %v", err) + } + startRetrievers(session, quit, &requested, maxReqCount) + + // Iterate over all the blocks and verify that the pipeline produces the correct matches + for i := uint64(0); i < blocks; i++ { + if expMatch3(filter, i) { + match, ok := <-matches + if !ok { + t.Errorf("filter = %v blocks = %v intermittent = %v: expected #%v, results channel closed", filter, blocks, intermittent, i) + return 0 + } + if match != i { + t.Errorf("filter = %v blocks = %v intermittent = %v: expected #%v, got #%v", filter, blocks, intermittent, i, match) + } + // If we're testing intermittent mode, abort and restart the pipeline + if intermittent { + session.Close(time.Second) + close(quit) + + quit = make(chan struct{}) + matches = make(chan uint64, 16) + + session, err = matcher.Start(i+1, blocks-1, matches) + if err != nil { + t.Fatalf("failed to stat matcher session: %v", err) + } + startRetrievers(session, quit, &requested, maxReqCount) + } + } + } + // Ensure the result channel is torn down after the last block + match, ok := <-matches + if ok { + t.Errorf("filter = %v blocks = %v intermittent = %v: expected closed channel, got #%v", filter, blocks, intermittent, match) + } + // Clean up the session and ensure we match the expected retrieval count + session.Close(time.Second) + close(quit) + + if retrievals != 0 && requested != retrievals { + t.Errorf("filter = %v blocks = %v intermittent = %v: request count mismatch, have #%v, want #%v", filter, blocks, intermittent, requested, retrievals) + } + return requested +} + +// startRetrievers starts a batch of goroutines listening for section requests +// and serving them. +func startRetrievers(session *MatcherSession, quit chan struct{}, retrievals *uint32, batch int) { + requests := make(chan chan *Retrieval) + + for i := 0; i < 10; i++ { + // Start a multiplexer to test multiple threaded execution + go session.Multiplex(batch, 100*time.Microsecond, requests) + + // Start a services to match the above multiplexer + go func() { + for { + // Wait for a service request or a shutdown + select { + case <-quit: + return + + case request := <-requests: + task := <-request + + task.Bitsets = make([][]byte, len(task.Sections)) + for i, section := range task.Sections { + if rand.Int()%4 != 0 { // Handle occasional missing deliveries + task.Bitsets[i] = generateBitset(task.Bit, section) + atomic.AddUint32(retrievals, 1) + } + } + request <- task + } + } + }() + } +} + +// generateBitset generates the rotated bitset for the given bloom bit and section +// numbers. +func generateBitset(bit uint, section uint64) []byte { + bitset := make([]byte, testSectionSize/8) + for i := 0; i < len(bitset); i++ { + for b := 0; b < 8; b++ { + blockIdx := section*testSectionSize + uint64(i*8+b) + bitset[i] += bitset[i] + if (blockIdx % uint64(bit)) == 0 { + bitset[i]++ + } + } + } + return bitset +} + +func expMatch1(filter bloomIndexes, i uint64) bool { + for _, ii := range filter { + if (i % uint64(ii)) != 0 { + return false + } + } + return true +} + +func expMatch2(filter []bloomIndexes, i uint64) bool { + for _, ii := range filter { + if expMatch1(ii, i) { + return true + } + } + return false +} + +func expMatch3(filter [][]bloomIndexes, i uint64) bool { + for _, ii := range filter { + if !expMatch2(ii, i) { + return false + } + } + return true +} diff --git a/core/bloombits/scheduler.go b/core/bloombits/scheduler.go new file mode 100644 index 0000000000..6449c7465a --- /dev/null +++ b/core/bloombits/scheduler.go @@ -0,0 +1,181 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package bloombits + +import ( + "sync" +) + +// request represents a bloom retrieval task to prioritize and pull from the local +// database or remotely from the network. +type request struct { + section uint64 // Section index to retrieve the a bit-vector from + bit uint // Bit index within the section to retrieve the vector of +} + +// response represents the state of a requested bit-vector through a scheduler. +type response struct { + cached []byte // Cached bits to dedup multiple requests + done chan struct{} // Channel to allow waiting for completion +} + +// scheduler handles the scheduling of bloom-filter retrieval operations for +// entire section-batches belonging to a single bloom bit. Beside scheduling the +// retrieval operations, this struct also deduplicates the requests and caches +// the results to minimize network/database overhead even in complex filtering +// scenarios. +type scheduler struct { + bit uint // Index of the bit in the bloom filter this scheduler is responsible for + responses map[uint64]*response // Currently pending retrieval requests or already cached responses + lock sync.Mutex // Lock protecting the responses from concurrent access +} + +// newScheduler creates a new bloom-filter retrieval scheduler for a specific +// bit index. +func newScheduler(idx uint) *scheduler { + return &scheduler{ + bit: idx, + responses: make(map[uint64]*response), + } +} + +// run creates a retrieval pipeline, receiving section indexes from sections and +// returning the results in the same order through the done channel. Concurrent +// runs of the same scheduler are allowed, leading to retrieval task deduplication. +func (s *scheduler) run(sections chan uint64, dist chan *request, done chan []byte, quit chan struct{}, wg *sync.WaitGroup) { + // Create a forwarder channel between requests and responses of the same size as + // the distribution channel (since that will block the pipeline anyway). + pend := make(chan uint64, cap(dist)) + + // Start the pipeline schedulers to forward between user -> distributor -> user + wg.Add(2) + go s.scheduleRequests(sections, dist, pend, quit, wg) + go s.scheduleDeliveries(pend, done, quit, wg) +} + +// reset cleans up any leftovers from previous runs. This is required before a +// restart to ensure the no previously requested but never delivered state will +// cause a lockup. +func (s *scheduler) reset() { + s.lock.Lock() + defer s.lock.Unlock() + + for section, res := range s.responses { + if res.cached == nil { + delete(s.responses, section) + } + } +} + +// scheduleRequests reads section retrieval requests from the input channel, +// deduplicates the stream and pushes unique retrieval tasks into the distribution +// channel for a database or network layer to honour. +func (s *scheduler) scheduleRequests(reqs chan uint64, dist chan *request, pend chan uint64, quit chan struct{}, wg *sync.WaitGroup) { + // Clean up the goroutine and pipeline when done + defer wg.Done() + defer close(pend) + + // Keep reading and scheduling section requests + for { + select { + case <-quit: + return + + case section, ok := <-reqs: + // New section retrieval requested + if !ok { + return + } + // Deduplicate retrieval requests + unique := false + + s.lock.Lock() + if s.responses[section] == nil { + s.responses[section] = &response{ + done: make(chan struct{}), + } + unique = true + } + s.lock.Unlock() + + // Schedule the section for retrieval and notify the deliverer to expect this section + if unique { + select { + case <-quit: + return + case dist <- &request{bit: s.bit, section: section}: + } + } + select { + case <-quit: + return + case pend <- section: + } + } + } +} + +// scheduleDeliveries reads section acceptance notifications and waits for them +// to be delivered, pushing them into the output data buffer. +func (s *scheduler) scheduleDeliveries(pend chan uint64, done chan []byte, quit chan struct{}, wg *sync.WaitGroup) { + // Clean up the goroutine and pipeline when done + defer wg.Done() + defer close(done) + + // Keep reading notifications and scheduling deliveries + for { + select { + case <-quit: + return + + case idx, ok := <-pend: + // New section retrieval pending + if !ok { + return + } + // Wait until the request is honoured + s.lock.Lock() + res := s.responses[idx] + s.lock.Unlock() + + select { + case <-quit: + return + case <-res.done: + } + // Deliver the result + select { + case <-quit: + return + case done <- res.cached: + } + } + } +} + +// deliver is called by the request distributor when a reply to a request arrives. +func (s *scheduler) deliver(sections []uint64, data [][]byte) { + s.lock.Lock() + defer s.lock.Unlock() + + for i, section := range sections { + if res := s.responses[section]; res != nil && res.cached == nil { // Avoid non-requests and double deliveries + res.cached = data[i] + close(res.done) + } + } +} diff --git a/core/bloombits/scheduler_test.go b/core/bloombits/scheduler_test.go new file mode 100644 index 0000000000..8a159c2370 --- /dev/null +++ b/core/bloombits/scheduler_test.go @@ -0,0 +1,105 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package bloombits + +import ( + "bytes" + "math/big" + "math/rand" + "sync" + "sync/atomic" + "testing" + "time" +) + +// Tests that the scheduler can deduplicate and forward retrieval requests to +// underlying fetchers and serve responses back, irrelevant of the concurrency +// of the requesting clients or serving data fetchers. +func TestSchedulerSingleClientSingleFetcher(t *testing.T) { testScheduler(t, 1, 1, 5000) } +func TestSchedulerSingleClientMultiFetcher(t *testing.T) { testScheduler(t, 1, 10, 5000) } +func TestSchedulerMultiClientSingleFetcher(t *testing.T) { testScheduler(t, 10, 1, 5000) } +func TestSchedulerMultiClientMultiFetcher(t *testing.T) { testScheduler(t, 10, 10, 5000) } + +func testScheduler(t *testing.T, clients int, fetchers int, requests int) { + f := newScheduler(0) + + // Create a batch of handler goroutines that respond to bloom bit requests and + // deliver them to the scheduler. + var fetchPend sync.WaitGroup + fetchPend.Add(fetchers) + defer fetchPend.Wait() + + fetch := make(chan *request, 16) + defer close(fetch) + + var delivered uint32 + for i := 0; i < fetchers; i++ { + go func() { + defer fetchPend.Done() + + for req := range fetch { + time.Sleep(time.Duration(rand.Intn(int(100 * time.Microsecond)))) + atomic.AddUint32(&delivered, 1) + + f.deliver([]uint64{ + req.section + uint64(requests), // Non-requested data (ensure it doesn't go out of bounds) + req.section, // Requested data + req.section, // Duplicated data (ensure it doesn't double close anything) + }, [][]byte{ + []byte{}, + new(big.Int).SetUint64(req.section).Bytes(), + new(big.Int).SetUint64(req.section).Bytes(), + }) + } + }() + } + // Start a batch of goroutines to concurrently run scheduling tasks + quit := make(chan struct{}) + + var pend sync.WaitGroup + pend.Add(clients) + + for i := 0; i < clients; i++ { + go func() { + defer pend.Done() + + in := make(chan uint64, 16) + out := make(chan []byte, 16) + + f.run(in, fetch, out, quit, &pend) + + go func() { + for j := 0; j < requests; j++ { + in <- uint64(j) + } + close(in) + }() + + for j := 0; j < requests; j++ { + bits := <-out + if want := new(big.Int).SetUint64(uint64(j)).Bytes(); !bytes.Equal(bits, want) { + t.Errorf("vector %d: delivered content mismatch: have %x, want %x", j, bits, want) + } + } + }() + } + pend.Wait() + + if have := atomic.LoadUint32(&delivered); int(have) != requests { + t.Errorf("request count mismatch: have %v, want %v", have, requests) + } +} diff --git a/core/chain_indexer.go b/core/chain_indexer.go index 9a88a5b1b8..f4c207dcca 100644 --- a/core/chain_indexer.go +++ b/core/chain_indexer.go @@ -42,9 +42,8 @@ type ChainIndexerBackend interface { // will ensure a sequential order of headers. Process(header *types.Header) - // Commit finalizes the section metadata and stores it into the database. This - // interface will usually be a batch writer. - Commit(db ethdb.Database) error + // Commit finalizes the section metadata and stores it into the database. + Commit() error } // ChainIndexer does a post-processing job for equally sized sections of the @@ -102,9 +101,10 @@ func NewChainIndexer(chainDb, indexDb ethdb.Database, backend ChainIndexerBacken } // Start creates a goroutine to feed chain head events into the indexer for -// cascading background processing. -func (c *ChainIndexer) Start(currentHeader *types.Header, eventMux *event.TypeMux) { - go c.eventLoop(currentHeader, eventMux) +// cascading background processing. Children do not need to be started, they +// are notified about new events by their parents. +func (c *ChainIndexer) Start(currentHeader *types.Header, chainEventer func(ch chan<- ChainEvent) event.Subscription) { + go c.eventLoop(currentHeader, chainEventer) } // Close tears down all goroutines belonging to the indexer and returns any error @@ -125,6 +125,12 @@ func (c *ChainIndexer) Close() error { errs = append(errs, err) } } + // Close all children + for _, child := range c.children { + if err := child.Close(); err != nil { + errs = append(errs, err) + } + } // Return any failures switch { case len(errs) == 0: @@ -141,12 +147,12 @@ func (c *ChainIndexer) Close() error { // eventLoop is a secondary - optional - event loop of the indexer which is only // started for the outermost indexer to push chain head events into a processing // queue. -func (c *ChainIndexer) eventLoop(currentHeader *types.Header, eventMux *event.TypeMux) { +func (c *ChainIndexer) eventLoop(currentHeader *types.Header, chainEventer func(ch chan<- ChainEvent) event.Subscription) { // Mark the chain indexer as active, requiring an additional teardown atomic.StoreUint32(&c.active, 1) - // Subscribe to chain head events - sub := eventMux.Subscribe(ChainEvent{}) + events := make(chan ChainEvent, 10) + sub := chainEventer(events) defer sub.Unsubscribe() // Fire the initial new head event to start any outstanding processing @@ -163,14 +169,14 @@ func (c *ChainIndexer) eventLoop(currentHeader *types.Header, eventMux *event.Ty errc <- nil return - case ev, ok := <-sub.Chan(): + case ev, ok := <-events: // Received a new event, ensure it's not nil (closing) and update if !ok { errc := <-c.quit errc <- nil return } - header := ev.Data.(ChainEvent).Block.Header() + header := ev.Block.Header() if header.ParentHash != prevHash { c.newHead(FindCommonAncestor(c.chainDb, prevHeader, header).Number.Uint64(), true) } @@ -226,8 +232,10 @@ func (c *ChainIndexer) newHead(head uint64, reorg bool) { // updateLoop is the main event loop of the indexer which pushes chain segments // down into the processing backend. func (c *ChainIndexer) updateLoop() { - var updated time.Time - + var ( + updating bool + updated time.Time + ) for { select { case errc := <-c.quit: @@ -242,6 +250,7 @@ func (c *ChainIndexer) updateLoop() { // Periodically print an upgrade log message to the user if time.Since(updated) > 8*time.Second { if c.knownSections > c.storedSections+1 { + updating = true c.log.Info("Upgrading chain index", "percentage", c.storedSections*100/c.knownSections) } updated = time.Now() @@ -255,12 +264,19 @@ func (c *ChainIndexer) updateLoop() { // Process the newly defined section in the background c.lock.Unlock() newHead, err := c.processSection(section, oldHead) + if err != nil { + c.log.Error("Section processing failed", "error", err) + } c.lock.Lock() // If processing succeeded and no reorgs occcurred, mark the section completed if err == nil && oldHead == c.sectionHead(section-1) { c.setSectionHead(section, newHead) c.setValidSections(section + 1) + if c.storedSections == c.knownSections && updating { + updating = false + c.log.Info("Finished upgrading chain index") + } c.cascadedHead = c.storedSections*c.sectionSize - 1 for _, child := range c.children { @@ -311,7 +327,8 @@ func (c *ChainIndexer) processSection(section uint64, lastHead common.Hash) (com c.backend.Process(header) lastHead = header.Hash() } - if err := c.backend.Commit(c.chainDb); err != nil { + if err := c.backend.Commit(); err != nil { + c.log.Error("Section commit failed", "error", err) return common.Hash{}, err } return lastHead, nil diff --git a/core/chain_indexer_test.go b/core/chain_indexer_test.go index 780e46e433..b761e8a5b2 100644 --- a/core/chain_indexer_test.go +++ b/core/chain_indexer_test.go @@ -58,7 +58,6 @@ func testChainIndexer(t *testing.T, count int) { ) backends[i] = &testChainIndexBackend{t: t, processCh: make(chan uint64)} backends[i].indexer = NewChainIndexer(db, ethdb.NewTable(db, string([]byte{byte(i)})), backends[i], sectionSize, confirmsReq, 0, fmt.Sprintf("indexer-%d", i)) - defer backends[i].indexer.Close() if sections, _, _ := backends[i].indexer.Sections(); sections != 0 { t.Fatalf("Canonical section count mismatch: have %v, want %v", sections, 0) @@ -67,6 +66,7 @@ func testChainIndexer(t *testing.T, count int) { backends[i-1].indexer.AddChildIndexer(backends[i].indexer) } } + defer backends[0].indexer.Close() // parent indexer shuts down children // notify pings the root indexer about a new head or reorg, then expect // processed blocks if a section is processable notify := func(headNum, failNum uint64, reorg bool) { @@ -226,7 +226,7 @@ func (b *testChainIndexBackend) Process(header *types.Header) { } } -func (b *testChainIndexBackend) Commit(db ethdb.Database) error { +func (b *testChainIndexBackend) Commit() error { if b.headerCnt != b.indexer.sectionSize { b.t.Error("Not enough headers processed") } diff --git a/core/dao.go b/core/dao.go deleted file mode 100644 index ff42a0e9d3..0000000000 --- a/core/dao.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package core - -import ( - "bytes" - "fmt" - "math/big" - - "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/params" -) - -// ValidateDAOHeaderExtraData validates the extra-data field of a block header to -// ensure it conforms to DAO hard-fork rules. -// -// DAO hard-fork extension to the header validity: -// a) if the node is no-fork, do not accept blocks in the [fork, fork+10) range -// with the fork specific extra-data set -// b) if the node is pro-fork, require blocks in the specific range to have the -// unique extra-data set. -func ValidateDAOHeaderExtraData(config *params.ChainConfig, header *types.Header) error { - // Short circuit validation if the node doesn't care about the DAO fork - if config.DAOForkBlock == nil { - return nil - } - // Make sure the block is within the fork's modified extra-data range - limit := new(big.Int).Add(config.DAOForkBlock, params.DAOForkExtraRange) - if header.Number.Cmp(config.DAOForkBlock) < 0 || header.Number.Cmp(limit) >= 0 { - return nil - } - // Depending whether we support or oppose the fork, validate the extra-data contents - if config.DAOForkSupport { - if !bytes.Equal(header.Extra, params.DAOForkBlockExtra) { - return fmt.Errorf("DAO pro-fork bad block extra-data: 0x%x", header.Extra) - } - } else { - if bytes.Equal(header.Extra, params.DAOForkBlockExtra) { - return fmt.Errorf("DAO no-fork bad block extra-data: 0x%x", header.Extra) - } - } - // All ok, header has the same extra-data we expect - return nil -} - -// ApplyDAOHardFork modifies the state database according to the DAO hard-fork -// rules, transferring all balances of a set of DAO accounts to a single refund -// contract. -func ApplyDAOHardFork(statedb *state.StateDB) { - // Retrieve the contract to refund balances into - if !statedb.Exist(params.DAORefundContract) { - statedb.CreateAccount(params.DAORefundContract) - } - - // Move every DAO account and extra-balance account funds into the refund contract - for _, addr := range params.DAODrainList() { - statedb.AddBalance(params.DAORefundContract, statedb.GetBalance(addr)) - statedb.SetBalance(addr, new(big.Int)) - } -} diff --git a/core/dao_test.go b/core/dao_test.go index d6e11d78ae..b9898ff7c8 100644 --- a/core/dao_test.go +++ b/core/dao_test.go @@ -40,14 +40,22 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Create the concurrent, conflicting two nodes proDb, _ := ethdb.NewMemDatabase() gspec.MustCommit(proDb) - proConf := ¶ms.ChainConfig{HomesteadBlock: big.NewInt(0), DAOForkBlock: forkBlock, DAOForkSupport: true} - proBc, _ := NewBlockChain(proDb, proConf, ethash.NewFaker(), vm.Config{}) + + proConf := *params.TestChainConfig + proConf.DAOForkBlock = forkBlock + proConf.DAOForkSupport = true + + proBc, _ := NewBlockChain(proDb, &proConf, ethash.NewFaker(), vm.Config{}) defer proBc.Stop() conDb, _ := ethdb.NewMemDatabase() gspec.MustCommit(conDb) - conConf := ¶ms.ChainConfig{HomesteadBlock: big.NewInt(0), DAOForkBlock: forkBlock, DAOForkSupport: false} - conBc, _ := NewBlockChain(conDb, conConf, ethash.NewFaker(), vm.Config{}) + + conConf := *params.TestChainConfig + conConf.DAOForkBlock = forkBlock + conConf.DAOForkSupport = false + + conBc, _ := NewBlockChain(conDb, &conConf, ethash.NewFaker(), vm.Config{}) defer conBc.Stop() if _, err := proBc.InsertChain(prefix); err != nil { @@ -61,7 +69,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Create a pro-fork block, and try to feed into the no-fork chain db, _ = ethdb.NewMemDatabase() gspec.MustCommit(db) - bc, _ := NewBlockChain(db, conConf, ethash.NewFaker(), vm.Config{}) + bc, _ := NewBlockChain(db, &conConf, ethash.NewFaker(), vm.Config{}) defer bc.Stop() blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64())) @@ -71,19 +79,19 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks); err != nil { t.Fatalf("failed to import contra-fork chain for expansion: %v", err) } - blocks, _ = GenerateChain(proConf, conBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) + blocks, _ = GenerateChain(&proConf, conBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) if _, err := conBc.InsertChain(blocks); err == nil { t.Fatalf("contra-fork chain accepted pro-fork block: %v", blocks[0]) } // Create a proper no-fork block for the contra-forker - blocks, _ = GenerateChain(conConf, conBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) + blocks, _ = GenerateChain(&conConf, conBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) if _, err := conBc.InsertChain(blocks); err != nil { t.Fatalf("contra-fork chain didn't accepted no-fork block: %v", err) } // Create a no-fork block, and try to feed into the pro-fork chain db, _ = ethdb.NewMemDatabase() gspec.MustCommit(db) - bc, _ = NewBlockChain(db, proConf, ethash.NewFaker(), vm.Config{}) + bc, _ = NewBlockChain(db, &proConf, ethash.NewFaker(), vm.Config{}) defer bc.Stop() blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64())) @@ -93,12 +101,12 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks); err != nil { t.Fatalf("failed to import pro-fork chain for expansion: %v", err) } - blocks, _ = GenerateChain(conConf, proBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) + blocks, _ = GenerateChain(&conConf, proBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) if _, err := proBc.InsertChain(blocks); err == nil { t.Fatalf("pro-fork chain accepted contra-fork block: %v", blocks[0]) } // Create a proper pro-fork block for the pro-forker - blocks, _ = GenerateChain(proConf, proBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) + blocks, _ = GenerateChain(&proConf, proBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) if _, err := proBc.InsertChain(blocks); err != nil { t.Fatalf("pro-fork chain didn't accepted pro-fork block: %v", err) } @@ -106,7 +114,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Verify that contra-forkers accept pro-fork extra-datas after forking finishes db, _ = ethdb.NewMemDatabase() gspec.MustCommit(db) - bc, _ := NewBlockChain(db, conConf, ethash.NewFaker(), vm.Config{}) + bc, _ := NewBlockChain(db, &conConf, ethash.NewFaker(), vm.Config{}) defer bc.Stop() blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64())) @@ -116,14 +124,14 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks); err != nil { t.Fatalf("failed to import contra-fork chain for expansion: %v", err) } - blocks, _ = GenerateChain(proConf, conBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) + blocks, _ = GenerateChain(&proConf, conBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) if _, err := conBc.InsertChain(blocks); err != nil { t.Fatalf("contra-fork chain didn't accept pro-fork block post-fork: %v", err) } // Verify that pro-forkers accept contra-fork extra-datas after forking finishes db, _ = ethdb.NewMemDatabase() gspec.MustCommit(db) - bc, _ = NewBlockChain(db, proConf, ethash.NewFaker(), vm.Config{}) + bc, _ = NewBlockChain(db, &proConf, ethash.NewFaker(), vm.Config{}) defer bc.Stop() blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64())) @@ -133,7 +141,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks); err != nil { t.Fatalf("failed to import pro-fork chain for expansion: %v", err) } - blocks, _ = GenerateChain(conConf, proBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) + blocks, _ = GenerateChain(&conConf, proBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) if _, err := proBc.InsertChain(blocks); err != nil { t.Fatalf("pro-fork chain didn't accept contra-fork block post-fork: %v", err) } diff --git a/core/database_util.go b/core/database_util.go index 6971113944..1730a048e7 100644 --- a/core/database_util.go +++ b/core/database_util.go @@ -23,7 +23,6 @@ import ( "errors" "fmt" "math/big" - "sync" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -34,24 +33,36 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) +// DatabaseReader wraps the Get method of a backing data store. +type DatabaseReader interface { + Get(key []byte) (value []byte, err error) +} + +// DatabaseDeleter wraps the Delete method of a backing data store. +type DatabaseDeleter interface { + Delete(key []byte) error +} + var ( headHeaderKey = []byte("LastHeader") headBlockKey = []byte("LastBlock") headFastKey = []byte("LastFast") - headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header - tdSuffix = []byte("t") // headerPrefix + num (uint64 big endian) + hash + tdSuffix -> td - numSuffix = []byte("n") // headerPrefix + num (uint64 big endian) + numSuffix -> hash - blockHashPrefix = []byte("H") // blockHashPrefix + hash -> num (uint64 big endian) - bodyPrefix = []byte("b") // bodyPrefix + num (uint64 big endian) + hash -> block body - blockReceiptsPrefix = []byte("r") // blockReceiptsPrefix + num (uint64 big endian) + hash -> block receipts - lookupPrefix = []byte("l") // lookupPrefix + hash -> transaction/receipt lookup metadata - preimagePrefix = "secure-key-" // preimagePrefix + hash -> preimage + // Data item prefixes (use single byte to avoid mixing data types, avoid `i`). + headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header + tdSuffix = []byte("t") // headerPrefix + num (uint64 big endian) + hash + tdSuffix -> td + numSuffix = []byte("n") // headerPrefix + num (uint64 big endian) + numSuffix -> hash + blockHashPrefix = []byte("H") // blockHashPrefix + hash -> num (uint64 big endian) + bodyPrefix = []byte("b") // bodyPrefix + num (uint64 big endian) + hash -> block body + blockReceiptsPrefix = []byte("r") // blockReceiptsPrefix + num (uint64 big endian) + hash -> block receipts + lookupPrefix = []byte("l") // lookupPrefix + hash -> transaction/receipt lookup metadata + bloomBitsPrefix = []byte("B") // bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash -> bloom bits - mipmapPre = []byte("mipmap-log-bloom-") - MIPMapLevels = []uint64{1000000, 500000, 100000, 50000, 1000} + preimagePrefix = "secure-key-" // preimagePrefix + hash -> preimage + configPrefix = []byte("ethereum-config-") // config prefix for the db - configPrefix = []byte("ethereum-config-") // config prefix for the db + // Chain index prefixes (use `i` + single byte to avoid mixing data types). + BloomBitsIndexPrefix = []byte("iB") // BloomBitsIndexPrefix is the data table of a chain indexer to track its progress // used by old db, now only used for conversion oldReceiptsPrefix = []byte("receipts-") @@ -59,8 +70,6 @@ var ( ErrChainConfigNotFound = errors.New("ChainConfig not found") // general config not found error - mipmapBloomMu sync.Mutex // protect against race condition when updating mipmap blooms - preimageCounter = metrics.NewCounter("db/preimage/total") preimageHitCounter = metrics.NewCounter("db/preimage/hits") ) @@ -81,7 +90,7 @@ func encodeBlockNumber(number uint64) []byte { } // GetCanonicalHash retrieves a hash assigned to a canonical block number. -func GetCanonicalHash(db ethdb.Database, number uint64) common.Hash { +func GetCanonicalHash(db DatabaseReader, number uint64) common.Hash { data, _ := db.Get(append(append(headerPrefix, encodeBlockNumber(number)...), numSuffix...)) if len(data) == 0 { return common.Hash{} @@ -95,7 +104,7 @@ const missingNumber = uint64(0xffffffffffffffff) // GetBlockNumber returns the block number assigned to a block hash // if the corresponding header is present in the database -func GetBlockNumber(db ethdb.Database, hash common.Hash) uint64 { +func GetBlockNumber(db DatabaseReader, hash common.Hash) uint64 { data, _ := db.Get(append(blockHashPrefix, hash.Bytes()...)) if len(data) != 8 { return missingNumber @@ -108,7 +117,7 @@ func GetBlockNumber(db ethdb.Database, hash common.Hash) uint64 { // last block hash is only updated upon a full block import, the last header // hash is updated already at header import, allowing head tracking for the // light synchronization mechanism. -func GetHeadHeaderHash(db ethdb.Database) common.Hash { +func GetHeadHeaderHash(db DatabaseReader) common.Hash { data, _ := db.Get(headHeaderKey) if len(data) == 0 { return common.Hash{} @@ -117,7 +126,7 @@ func GetHeadHeaderHash(db ethdb.Database) common.Hash { } // GetHeadBlockHash retrieves the hash of the current canonical head block. -func GetHeadBlockHash(db ethdb.Database) common.Hash { +func GetHeadBlockHash(db DatabaseReader) common.Hash { data, _ := db.Get(headBlockKey) if len(data) == 0 { return common.Hash{} @@ -129,7 +138,7 @@ func GetHeadBlockHash(db ethdb.Database) common.Hash { // fast synchronization. The difference between this and GetHeadBlockHash is that // whereas the last block hash is only updated upon a full block import, the last // fast hash is updated when importing pre-processed blocks. -func GetHeadFastBlockHash(db ethdb.Database) common.Hash { +func GetHeadFastBlockHash(db DatabaseReader) common.Hash { data, _ := db.Get(headFastKey) if len(data) == 0 { return common.Hash{} @@ -139,14 +148,14 @@ func GetHeadFastBlockHash(db ethdb.Database) common.Hash { // GetHeaderRLP retrieves a block header in its raw RLP database encoding, or nil // if the header's not found. -func GetHeaderRLP(db ethdb.Database, hash common.Hash, number uint64) rlp.RawValue { - data, _ := db.Get(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) +func GetHeaderRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue { + data, _ := db.Get(headerKey(hash, number)) return data } // GetHeader retrieves the block header corresponding to the hash, nil if none // found. -func GetHeader(db ethdb.Database, hash common.Hash, number uint64) *types.Header { +func GetHeader(db DatabaseReader, hash common.Hash, number uint64) *types.Header { data := GetHeaderRLP(db, hash, number) if len(data) == 0 { return nil @@ -160,14 +169,22 @@ func GetHeader(db ethdb.Database, hash common.Hash, number uint64) *types.Header } // GetBodyRLP retrieves the block body (transactions and uncles) in RLP encoding. -func GetBodyRLP(db ethdb.Database, hash common.Hash, number uint64) rlp.RawValue { - data, _ := db.Get(append(append(bodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) +func GetBodyRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue { + data, _ := db.Get(blockBodyKey(hash, number)) return data } +func headerKey(hash common.Hash, number uint64) []byte { + return append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...) +} + +func blockBodyKey(hash common.Hash, number uint64) []byte { + return append(append(bodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...) +} + // GetBody retrieves the block body (transactons, uncles) corresponding to the // hash, nil if none found. -func GetBody(db ethdb.Database, hash common.Hash, number uint64) *types.Body { +func GetBody(db DatabaseReader, hash common.Hash, number uint64) *types.Body { data := GetBodyRLP(db, hash, number) if len(data) == 0 { return nil @@ -182,7 +199,7 @@ func GetBody(db ethdb.Database, hash common.Hash, number uint64) *types.Body { // GetTd retrieves a block's total difficulty corresponding to the hash, nil if // none found. -func GetTd(db ethdb.Database, hash common.Hash, number uint64) *big.Int { +func GetTd(db DatabaseReader, hash common.Hash, number uint64) *big.Int { data, _ := db.Get(append(append(append(headerPrefix, encodeBlockNumber(number)...), hash[:]...), tdSuffix...)) if len(data) == 0 { return nil @@ -201,7 +218,7 @@ func GetTd(db ethdb.Database, hash common.Hash, number uint64) *big.Int { // // Note, due to concurrent download of header and block body the header and thus // canonical hash can be stored in the database but the body data not (yet). -func GetBlock(db ethdb.Database, hash common.Hash, number uint64) *types.Block { +func GetBlock(db DatabaseReader, hash common.Hash, number uint64) *types.Block { // Retrieve the block header and body contents header := GetHeader(db, hash, number) if header == nil { @@ -217,7 +234,7 @@ func GetBlock(db ethdb.Database, hash common.Hash, number uint64) *types.Block { // GetBlockReceipts retrieves the receipts generated by the transactions included // in a block given by its hash. -func GetBlockReceipts(db ethdb.Database, hash common.Hash, number uint64) types.Receipts { +func GetBlockReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Receipts { data, _ := db.Get(append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash[:]...)) if len(data) == 0 { return nil @@ -236,7 +253,7 @@ func GetBlockReceipts(db ethdb.Database, hash common.Hash, number uint64) types. // GetTxLookupEntry retrieves the positional metadata associated with a transaction // hash to allow retrieving the transaction or receipt by hash. -func GetTxLookupEntry(db ethdb.Database, hash common.Hash) (common.Hash, uint64, uint64) { +func GetTxLookupEntry(db DatabaseReader, hash common.Hash) (common.Hash, uint64, uint64) { // Load the positional metadata from disk and bail if it fails data, _ := db.Get(append(lookupPrefix, hash.Bytes()...)) if len(data) == 0 { @@ -253,7 +270,7 @@ func GetTxLookupEntry(db ethdb.Database, hash common.Hash) (common.Hash, uint64, // GetTransaction retrieves a specific transaction from the database, along with // its added positional metadata. -func GetTransaction(db ethdb.Database, hash common.Hash) (*types.Transaction, common.Hash, uint64, uint64) { +func GetTransaction(db DatabaseReader, hash common.Hash) (*types.Transaction, common.Hash, uint64, uint64) { // Retrieve the lookup metadata and resolve the transaction from the body blockHash, blockNumber, txIndex := GetTxLookupEntry(db, hash) @@ -288,7 +305,7 @@ func GetTransaction(db ethdb.Database, hash common.Hash) (*types.Transaction, co // GetReceipt retrieves a specific transaction receipt from the database, along with // its added positional metadata. -func GetReceipt(db ethdb.Database, hash common.Hash) (*types.Receipt, common.Hash, uint64, uint64) { +func GetReceipt(db DatabaseReader, hash common.Hash) (*types.Receipt, common.Hash, uint64, uint64) { // Retrieve the lookup metadata and resolve the receipt from the receipts blockHash, blockNumber, receiptIndex := GetTxLookupEntry(db, hash) @@ -313,8 +330,20 @@ func GetReceipt(db ethdb.Database, hash common.Hash) (*types.Receipt, common.Has return (*types.Receipt)(&receipt), common.Hash{}, 0, 0 } +// GetBloomBits retrieves the compressed bloom bit vector belonging to the given +// section and bit index from the. +func GetBloomBits(db DatabaseReader, bit uint, section uint64, head common.Hash) []byte { + key := append(append(bloomBitsPrefix, make([]byte, 10)...), head.Bytes()...) + + binary.BigEndian.PutUint16(key[1:], uint16(bit)) + binary.BigEndian.PutUint64(key[3:], section) + + bits, _ := db.Get(key) + return bits +} + // WriteCanonicalHash stores the canonical hash for the given block number. -func WriteCanonicalHash(db ethdb.Database, hash common.Hash, number uint64) error { +func WriteCanonicalHash(db ethdb.Putter, hash common.Hash, number uint64) error { key := append(append(headerPrefix, encodeBlockNumber(number)...), numSuffix...) if err := db.Put(key, hash.Bytes()); err != nil { log.Crit("Failed to store number to hash mapping", "err", err) @@ -323,7 +352,7 @@ func WriteCanonicalHash(db ethdb.Database, hash common.Hash, number uint64) erro } // WriteHeadHeaderHash stores the head header's hash. -func WriteHeadHeaderHash(db ethdb.Database, hash common.Hash) error { +func WriteHeadHeaderHash(db ethdb.Putter, hash common.Hash) error { if err := db.Put(headHeaderKey, hash.Bytes()); err != nil { log.Crit("Failed to store last header's hash", "err", err) } @@ -331,7 +360,7 @@ func WriteHeadHeaderHash(db ethdb.Database, hash common.Hash) error { } // WriteHeadBlockHash stores the head block's hash. -func WriteHeadBlockHash(db ethdb.Database, hash common.Hash) error { +func WriteHeadBlockHash(db ethdb.Putter, hash common.Hash) error { if err := db.Put(headBlockKey, hash.Bytes()); err != nil { log.Crit("Failed to store last block's hash", "err", err) } @@ -339,7 +368,7 @@ func WriteHeadBlockHash(db ethdb.Database, hash common.Hash) error { } // WriteHeadFastBlockHash stores the fast head block's hash. -func WriteHeadFastBlockHash(db ethdb.Database, hash common.Hash) error { +func WriteHeadFastBlockHash(db ethdb.Putter, hash common.Hash) error { if err := db.Put(headFastKey, hash.Bytes()); err != nil { log.Crit("Failed to store last fast block's hash", "err", err) } @@ -347,7 +376,7 @@ func WriteHeadFastBlockHash(db ethdb.Database, hash common.Hash) error { } // WriteHeader serializes a block header into the database. -func WriteHeader(db ethdb.Database, header *types.Header) error { +func WriteHeader(db ethdb.Putter, header *types.Header) error { data, err := rlp.EncodeToBytes(header) if err != nil { return err @@ -367,7 +396,7 @@ func WriteHeader(db ethdb.Database, header *types.Header) error { } // WriteBody serializes the body of a block into the database. -func WriteBody(db ethdb.Database, hash common.Hash, number uint64, body *types.Body) error { +func WriteBody(db ethdb.Putter, hash common.Hash, number uint64, body *types.Body) error { data, err := rlp.EncodeToBytes(body) if err != nil { return err @@ -376,7 +405,7 @@ func WriteBody(db ethdb.Database, hash common.Hash, number uint64, body *types.B } // WriteBodyRLP writes a serialized body of a block into the database. -func WriteBodyRLP(db ethdb.Database, hash common.Hash, number uint64, rlp rlp.RawValue) error { +func WriteBodyRLP(db ethdb.Putter, hash common.Hash, number uint64, rlp rlp.RawValue) error { key := append(append(bodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...) if err := db.Put(key, rlp); err != nil { log.Crit("Failed to store block body", "err", err) @@ -385,7 +414,7 @@ func WriteBodyRLP(db ethdb.Database, hash common.Hash, number uint64, rlp rlp.Ra } // WriteTd serializes the total difficulty of a block into the database. -func WriteTd(db ethdb.Database, hash common.Hash, number uint64, td *big.Int) error { +func WriteTd(db ethdb.Putter, hash common.Hash, number uint64, td *big.Int) error { data, err := rlp.EncodeToBytes(td) if err != nil { return err @@ -398,7 +427,7 @@ func WriteTd(db ethdb.Database, hash common.Hash, number uint64, td *big.Int) er } // WriteBlock serializes a block into the database, header and body separately. -func WriteBlock(db ethdb.Database, block *types.Block) error { +func WriteBlock(db ethdb.Putter, block *types.Block) error { // Store the body first to retain database consistency if err := WriteBody(db, block.Hash(), block.NumberU64(), block.Body()); err != nil { return err @@ -413,7 +442,7 @@ func WriteBlock(db ethdb.Database, block *types.Block) error { // WriteBlockReceipts stores all the transaction receipts belonging to a block // as a single receipt slice. This is used during chain reorganisations for // rescheduling dropped transactions. -func WriteBlockReceipts(db ethdb.Database, hash common.Hash, number uint64, receipts types.Receipts) error { +func WriteBlockReceipts(db ethdb.Putter, hash common.Hash, number uint64, receipts types.Receipts) error { // Convert the receipts into their storage form and serialize them storageReceipts := make([]*types.ReceiptForStorage, len(receipts)) for i, receipt := range receipts { @@ -433,9 +462,7 @@ func WriteBlockReceipts(db ethdb.Database, hash common.Hash, number uint64, rece // WriteTxLookupEntries stores a positional metadata for every transaction from // a block, enabling hash based transaction and receipt lookups. -func WriteTxLookupEntries(db ethdb.Database, block *types.Block) error { - batch := db.NewBatch() - +func WriteTxLookupEntries(db ethdb.Putter, block *types.Block) error { // Iterate over each transaction and encode its metadata for i, tx := range block.Transactions() { entry := txLookupEntry{ @@ -447,40 +474,49 @@ func WriteTxLookupEntries(db ethdb.Database, block *types.Block) error { if err != nil { return err } - if err := batch.Put(append(lookupPrefix, tx.Hash().Bytes()...), data); err != nil { + if err := db.Put(append(lookupPrefix, tx.Hash().Bytes()...), data); err != nil { return err } } - // Write the scheduled data into the database - if err := batch.Write(); err != nil { - log.Crit("Failed to store lookup entries", "err", err) - } return nil } +// WriteBloomBits writes the compressed bloom bits vector belonging to the given +// section and bit index. +func WriteBloomBits(db ethdb.Putter, bit uint, section uint64, head common.Hash, bits []byte) { + key := append(append(bloomBitsPrefix, make([]byte, 10)...), head.Bytes()...) + + binary.BigEndian.PutUint16(key[1:], uint16(bit)) + binary.BigEndian.PutUint64(key[3:], section) + + if err := db.Put(key, bits); err != nil { + log.Crit("Failed to store bloom bits", "err", err) + } +} + // DeleteCanonicalHash removes the number to hash canonical mapping. -func DeleteCanonicalHash(db ethdb.Database, number uint64) { +func DeleteCanonicalHash(db DatabaseDeleter, number uint64) { db.Delete(append(append(headerPrefix, encodeBlockNumber(number)...), numSuffix...)) } // DeleteHeader removes all block header data associated with a hash. -func DeleteHeader(db ethdb.Database, hash common.Hash, number uint64) { +func DeleteHeader(db DatabaseDeleter, hash common.Hash, number uint64) { db.Delete(append(blockHashPrefix, hash.Bytes()...)) db.Delete(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) } // DeleteBody removes all block body data associated with a hash. -func DeleteBody(db ethdb.Database, hash common.Hash, number uint64) { +func DeleteBody(db DatabaseDeleter, hash common.Hash, number uint64) { db.Delete(append(append(bodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) } // DeleteTd removes all block total difficulty data associated with a hash. -func DeleteTd(db ethdb.Database, hash common.Hash, number uint64) { +func DeleteTd(db DatabaseDeleter, hash common.Hash, number uint64) { db.Delete(append(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...), tdSuffix...)) } // DeleteBlock removes all block data associated with a hash. -func DeleteBlock(db ethdb.Database, hash common.Hash, number uint64) { +func DeleteBlock(db DatabaseDeleter, hash common.Hash, number uint64) { DeleteBlockReceipts(db, hash, number) DeleteHeader(db, hash, number) DeleteBody(db, hash, number) @@ -488,57 +524,15 @@ func DeleteBlock(db ethdb.Database, hash common.Hash, number uint64) { } // DeleteBlockReceipts removes all receipt data associated with a block hash. -func DeleteBlockReceipts(db ethdb.Database, hash common.Hash, number uint64) { +func DeleteBlockReceipts(db DatabaseDeleter, hash common.Hash, number uint64) { db.Delete(append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) } // DeleteTxLookupEntry removes all transaction data associated with a hash. -func DeleteTxLookupEntry(db ethdb.Database, hash common.Hash) { +func DeleteTxLookupEntry(db DatabaseDeleter, hash common.Hash) { db.Delete(append(lookupPrefix, hash.Bytes()...)) } -// returns a formatted MIP mapped key by adding prefix, canonical number and level -// -// ex. fn(98, 1000) = (prefix || 1000 || 0) -func mipmapKey(num, level uint64) []byte { - lkey := make([]byte, 8) - binary.BigEndian.PutUint64(lkey, level) - key := new(big.Int).SetUint64(num / level * level) - - return append(mipmapPre, append(lkey, key.Bytes()...)...) -} - -// WriteMipmapBloom writes each address included in the receipts' logs to the -// MIP bloom bin. -func WriteMipmapBloom(db ethdb.Database, number uint64, receipts types.Receipts) error { - mipmapBloomMu.Lock() - defer mipmapBloomMu.Unlock() - - batch := db.NewBatch() - for _, level := range MIPMapLevels { - key := mipmapKey(number, level) - bloomDat, _ := db.Get(key) - bloom := types.BytesToBloom(bloomDat) - for _, receipt := range receipts { - for _, log := range receipt.Logs { - bloom.Add(log.Address.Big()) - } - } - batch.Put(key, bloom.Bytes()) - } - if err := batch.Write(); err != nil { - return fmt.Errorf("mipmap write fail for: %d: %v", number, err) - } - return nil -} - -// GetMipmapBloom returns a bloom filter using the number and level as input -// parameters. For available levels see MIPMapLevels. -func GetMipmapBloom(db ethdb.Database, number, level uint64) types.Bloom { - bloomDat, _ := db.Get(mipmapKey(number, level)) - return types.BytesToBloom(bloomDat) -} - // PreimageTable returns a Database instance with the key prefix for preimage entries. func PreimageTable(db ethdb.Database) ethdb.Database { return ethdb.NewTable(db, preimagePrefix) @@ -567,7 +561,7 @@ func WritePreimages(db ethdb.Database, number uint64, preimages map[common.Hash] } // GetBlockChainVersion reads the version number from db. -func GetBlockChainVersion(db ethdb.Database) int { +func GetBlockChainVersion(db DatabaseReader) int { var vsn uint enc, _ := db.Get([]byte("BlockchainVersion")) rlp.DecodeBytes(enc, &vsn) @@ -575,13 +569,13 @@ func GetBlockChainVersion(db ethdb.Database) int { } // WriteBlockChainVersion writes vsn as the version number to db. -func WriteBlockChainVersion(db ethdb.Database, vsn int) { +func WriteBlockChainVersion(db ethdb.Putter, vsn int) { enc, _ := rlp.EncodeToBytes(uint(vsn)) db.Put([]byte("BlockchainVersion"), enc) } // WriteChainConfig writes the chain config settings to the database. -func WriteChainConfig(db ethdb.Database, hash common.Hash, cfg *params.ChainConfig) error { +func WriteChainConfig(db ethdb.Putter, hash common.Hash, cfg *params.ChainConfig) error { // short circuit and ignore if nil config. GetChainConfig // will return a default. if cfg == nil { @@ -597,7 +591,7 @@ func WriteChainConfig(db ethdb.Database, hash common.Hash, cfg *params.ChainConf } // GetChainConfig will fetch the network settings based on the given hash. -func GetChainConfig(db ethdb.Database, hash common.Hash) (*params.ChainConfig, error) { +func GetChainConfig(db DatabaseReader, hash common.Hash) (*params.ChainConfig, error) { jsonChainConfig, _ := db.Get(append(configPrefix, hash[:]...)) if len(jsonChainConfig) == 0 { return nil, ErrChainConfigNotFound @@ -612,7 +606,7 @@ func GetChainConfig(db ethdb.Database, hash common.Hash) (*params.ChainConfig, e } // FindCommonAncestor returns the last common ancestor of two block headers -func FindCommonAncestor(db ethdb.Database, a, b *types.Header) *types.Header { +func FindCommonAncestor(db DatabaseReader, a, b *types.Header) *types.Header { for bn := b.Number.Uint64(); a.Number.Uint64() > bn; { a = GetHeader(db, a.ParentHash, a.Number.Uint64()-1) if a == nil { diff --git a/core/database_util_test.go b/core/database_util_test.go index e91f1b5930..940221a299 100644 --- a/core/database_util_test.go +++ b/core/database_util_test.go @@ -18,17 +18,13 @@ package core import ( "bytes" - "io/ioutil" "math/big" - "os" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" ) @@ -390,107 +386,3 @@ func TestBlockReceiptStorage(t *testing.T) { t.Fatalf("deleted receipts returned: %v", rs) } } - -func TestMipmapBloom(t *testing.T) { - db, _ := ethdb.NewMemDatabase() - - receipt1 := new(types.Receipt) - receipt1.Logs = []*types.Log{ - {Address: common.BytesToAddress([]byte("test"))}, - {Address: common.BytesToAddress([]byte("address"))}, - } - receipt2 := new(types.Receipt) - receipt2.Logs = []*types.Log{ - {Address: common.BytesToAddress([]byte("test"))}, - {Address: common.BytesToAddress([]byte("address1"))}, - } - - WriteMipmapBloom(db, 1, types.Receipts{receipt1}) - WriteMipmapBloom(db, 2, types.Receipts{receipt2}) - - for _, level := range MIPMapLevels { - bloom := GetMipmapBloom(db, 2, level) - if !bloom.Test(new(big.Int).SetBytes([]byte("address1"))) { - t.Error("expected test to be included on level:", level) - } - } - - // reset - db, _ = ethdb.NewMemDatabase() - receipt := new(types.Receipt) - receipt.Logs = []*types.Log{ - {Address: common.BytesToAddress([]byte("test"))}, - } - WriteMipmapBloom(db, 999, types.Receipts{receipt1}) - - receipt = new(types.Receipt) - receipt.Logs = []*types.Log{ - {Address: common.BytesToAddress([]byte("test 1"))}, - } - WriteMipmapBloom(db, 1000, types.Receipts{receipt}) - - bloom := GetMipmapBloom(db, 1000, 1000) - if bloom.TestBytes([]byte("test")) { - t.Error("test should not have been included") - } -} - -func TestMipmapChain(t *testing.T) { - dir, err := ioutil.TempDir("", "mipmap") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - - var ( - db, _ = ethdb.NewLDBDatabase(dir, 0, 0) - key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - addr = crypto.PubkeyToAddress(key1.PublicKey) - addr2 = common.BytesToAddress([]byte("jeff")) - - hash1 = common.BytesToHash([]byte("topic1")) - ) - defer db.Close() - - gspec := &Genesis{ - Config: params.TestChainConfig, - Alloc: GenesisAlloc{addr: {Balance: big.NewInt(1000000)}}, - } - genesis := gspec.MustCommit(db) - chain, receipts := GenerateChain(params.TestChainConfig, genesis, db, 1010, func(i int, gen *BlockGen) { - var receipts types.Receipts - switch i { - case 1: - receipt := types.NewReceipt(nil, false, new(big.Int)) - receipt.Logs = []*types.Log{{Address: addr, Topics: []common.Hash{hash1}}} - gen.AddUncheckedReceipt(receipt) - receipts = types.Receipts{receipt} - case 1000: - receipt := types.NewReceipt(nil, false, new(big.Int)) - receipt.Logs = []*types.Log{{Address: addr2}} - gen.AddUncheckedReceipt(receipt) - receipts = types.Receipts{receipt} - - } - - // store the receipts - WriteMipmapBloom(db, uint64(i+1), receipts) - }) - for i, block := range chain { - WriteBlock(db, block) - if err := WriteCanonicalHash(db, block.Hash(), block.NumberU64()); err != nil { - t.Fatalf("failed to insert block number: %v", err) - } - if err := WriteHeadBlockHash(db, block.Hash()); err != nil { - t.Fatalf("failed to insert block number: %v", err) - } - if err := WriteBlockReceipts(db, block.Hash(), block.NumberU64(), receipts[i]); err != nil { - t.Fatal("error writing block receipts:", err) - } - } - - bloom := GetMipmapBloom(db, 0, 1000) - if bloom.TestBytes(addr2[:]) { - t.Error("address was included in bloom and should not have") - } -} diff --git a/core/error.go b/core/error.go index 9ac4fff514..410eca1e1e 100644 --- a/core/error.go +++ b/core/error.go @@ -28,4 +28,8 @@ var ( // ErrBlacklistedHash is returned if a block to import is on the blacklist. ErrBlacklistedHash = errors.New("blacklisted hash") + + // ErrNonceTooHigh is returned if the nonce of a transaction is higher than the + // next one expected based on the local chain. + ErrNonceTooHigh = errors.New("nonce too high") ) diff --git a/core/headerchain.go b/core/headerchain.go index 6ec44b61d5..0e5215293d 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -267,7 +267,7 @@ func (hc *HeaderChain) InsertHeaderChain(chain []*types.Header, writeHeader WhCa return i, errors.New("aborted") } // If the header's already known, skip it, otherwise store - if hc.GetHeader(header.Hash(), header.Number.Uint64()) != nil { + if hc.HasHeader(header.Hash(), header.Number.Uint64()) { stats.ignored++ continue } @@ -361,10 +361,13 @@ func (hc *HeaderChain) GetHeaderByHash(hash common.Hash) *types.Header { return hc.GetHeader(hash, hc.GetBlockNumber(hash)) } -// HasHeader checks if a block header is present in the database or not, caching -// it if present. -func (hc *HeaderChain) HasHeader(hash common.Hash) bool { - return hc.GetHeaderByHash(hash) != nil +// HasHeader checks if a block header is present in the database or not. +func (hc *HeaderChain) HasHeader(hash common.Hash, number uint64) bool { + if hc.numberCache.Contains(hash) || hc.headerCache.Contains(hash) { + return true + } + ok, _ := hc.chainDb.Has(headerKey(hash, number)) + return ok } // GetHeaderByNumber retrieves a block header from the database by number, diff --git a/core/state/sync.go b/core/state/sync.go index 2c29d706aa..28fcf6ae05 100644 --- a/core/state/sync.go +++ b/core/state/sync.go @@ -18,60 +18,24 @@ package state import ( "bytes" - "math/big" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" ) -// StateSync is the main state synchronisation scheduler, which provides yet the -// unknown state hashes to retrieve, accepts node data associated with said hashes -// and reconstructs the state database step by step until all is done. -type StateSync trie.TrieSync - // NewStateSync create a new state trie download scheduler. -func NewStateSync(root common.Hash, database trie.DatabaseReader) *StateSync { +func NewStateSync(root common.Hash, database trie.DatabaseReader) *trie.TrieSync { var syncer *trie.TrieSync - callback := func(leaf []byte, parent common.Hash) error { - var obj struct { - Nonce uint64 - Balance *big.Int - Root common.Hash - CodeHash []byte - } + var obj Account if err := rlp.Decode(bytes.NewReader(leaf), &obj); err != nil { return err } syncer.AddSubTrie(obj.Root, 64, parent, nil) syncer.AddRawEntry(common.BytesToHash(obj.CodeHash), 64, parent) - return nil } syncer = trie.NewTrieSync(root, database, callback) - return (*StateSync)(syncer) -} - -// Missing retrieves the known missing nodes from the state trie for retrieval. -func (s *StateSync) Missing(max int) []common.Hash { - return (*trie.TrieSync)(s).Missing(max) -} - -// Process injects a batch of retrieved trie nodes data, returning if something -// was committed to the memcache and also the index of an entry if processing of -// it failed. -func (s *StateSync) Process(list []trie.SyncResult) (bool, int, error) { - return (*trie.TrieSync)(s).Process(list) -} - -// Commit flushes the data stored in the internal memcache out to persistent -// storage, returning th enumber of items written and any occurred error. -func (s *StateSync) Commit(dbw trie.DatabaseWriter) (int, error) { - return (*trie.TrieSync)(s).Commit(dbw) -} - -// Pending returns the number of state entries currently pending for download. -func (s *StateSync) Pending() int { - return (*trie.TrieSync)(s).Pending() + return syncer } diff --git a/core/state_processor.go b/core/state_processor.go index 4115eab8cb..689c83785f 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -105,7 +105,7 @@ func ApplyTransaction(config *params.ChainConfig, bc *BlockChain, author *common // Update the state with pending changes var root []byte - if config.IsMetropolis(header.Number) { + if config.IsByzantium(header.Number) { statedb.Finalise(true) } else { root = statedb.IntermediateRoot(config.IsEIP158(header.Number)).Bytes() diff --git a/core/state_transition.go b/core/state_transition.go index bab4540be8..e7a0685893 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -18,7 +18,6 @@ package core import ( "errors" - "fmt" "math/big" "github.com/ethereum/go-ethereum/common" @@ -197,8 +196,11 @@ func (st *StateTransition) preCheck() error { // Make sure this transaction's nonce is correct if msg.CheckNonce() { - if n := st.state.GetNonce(sender.Address()); n != msg.Nonce() { - return fmt.Errorf("invalid nonce: have %d, expected %d", msg.Nonce(), n) + nonce := st.state.GetNonce(sender.Address()) + if nonce < msg.Nonce() { + return ErrNonceTooHigh + } else if nonce > msg.Nonce() { + return ErrNonceTooLow } } return st.buyGas() diff --git a/core/tx_list.go b/core/tx_list.go index 0d87c20bc2..2935929d72 100644 --- a/core/tx_list.go +++ b/core/tx_list.go @@ -298,6 +298,7 @@ func (l *txList) Filter(costLimit, gasLimit *big.Int) (types.Transactions, types // If the list was strict, filter anything above the lowest nonce var invalids types.Transactions + if l.strict && len(removed) > 0 { lowest := uint64(math.MaxUint64) for _, tx := range removed { @@ -435,6 +436,7 @@ func (l *txPricedList) Cap(threshold *big.Int, local *accountSet) types.Transact } // Stop the discards if we've reached the threshold if tx.GasPrice().Cmp(threshold) >= 0 { + save = append(save, tx) break } // Non stale transaction found, discard unless local diff --git a/core/tx_pool.go b/core/tx_pool.go index d835c94d1a..0ad7651795 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -19,6 +19,7 @@ package core import ( "errors" "fmt" + "math" "math/big" "sort" "sync" @@ -105,10 +106,11 @@ var ( // blockChain provides the state of blockchain and current gas limit to do // some pre checks in tx pool and event subscribers. type blockChain interface { - State() (*state.StateDB, error) - GasLimit() *big.Int + CurrentBlock() *types.Block + GetBlock(hash common.Hash, number uint64) *types.Block + StateAt(root common.Hash) (*state.StateDB, error) + SubscribeChainHeadEvent(ch chan<- ChainHeadEvent) event.Subscription - SubscribeRemovedTxEvent(ch chan<- RemovedTransactionEvent) event.Subscription } // TxPoolConfig are the configuration parameters of the transaction pool. @@ -174,18 +176,19 @@ func (config *TxPoolConfig) sanitize() TxPoolConfig { type TxPool struct { config TxPoolConfig chainconfig *params.ChainConfig - blockChain blockChain - pendingState *state.ManagedState + chain blockChain gasPrice *big.Int txFeed event.Feed scope event.SubscriptionScope chainHeadCh chan ChainHeadEvent chainHeadSub event.Subscription - rmTxCh chan RemovedTransactionEvent - rmTxSub event.Subscription signer types.Signer mu sync.RWMutex + currentState *state.StateDB // Current state in the blockchain head + pendingState *state.ManagedState // Pending state tracking virtual nonces + currentMaxGas *big.Int // Current gas limit for transaction caps + locals *accountSet // Set of local transaction to exepmt from evicion rules journal *txJournal // Journal of local transaction to back up to disk @@ -202,28 +205,26 @@ type TxPool struct { // NewTxPool creates a new transaction pool to gather, sort and filter inbound // trnsactions from the network. -func NewTxPool(config TxPoolConfig, chainconfig *params.ChainConfig, blockChain blockChain) *TxPool { +func NewTxPool(config TxPoolConfig, chainconfig *params.ChainConfig, chain blockChain) *TxPool { // Sanitize the input to ensure no vulnerable gas prices are set config = (&config).sanitize() // Create the transaction pool with its initial settings pool := &TxPool{ - config: config, - chainconfig: chainconfig, - blockChain: blockChain, - signer: types.NewEIP155Signer(chainconfig.ChainId), - pending: make(map[common.Address]*txList), - queue: make(map[common.Address]*txList), - beats: make(map[common.Address]time.Time), - all: make(map[common.Hash]*types.Transaction), - chainHeadCh: make(chan ChainHeadEvent, chainHeadChanSize), - rmTxCh: make(chan RemovedTransactionEvent, rmTxChanSize), - gasPrice: new(big.Int).SetUint64(config.PriceLimit), - pendingState: nil, + config: config, + chainconfig: chainconfig, + chain: chain, + signer: types.NewEIP155Signer(chainconfig.ChainId), + pending: make(map[common.Address]*txList), + queue: make(map[common.Address]*txList), + beats: make(map[common.Address]time.Time), + all: make(map[common.Hash]*types.Transaction), + chainHeadCh: make(chan ChainHeadEvent, chainHeadChanSize), + gasPrice: new(big.Int).SetUint64(config.PriceLimit), } pool.locals = newAccountSet(pool.signer) pool.priced = newTxPricedList(&pool.all) - pool.reset() + pool.reset(nil, chain.CurrentBlock().Header()) // If local transactions and journaling is enabled, load from disk if !config.NoLocals && config.Journal != "" { @@ -237,8 +238,8 @@ func NewTxPool(config TxPoolConfig, chainconfig *params.ChainConfig, blockChain } } // Subscribe events from blockchain - pool.chainHeadSub = pool.blockChain.SubscribeChainHeadEvent(pool.chainHeadCh) - pool.rmTxSub = pool.blockChain.SubscribeRemovedTxEvent(pool.rmTxCh) + pool.chainHeadSub = pool.chain.SubscribeChainHeadEvent(pool.chainHeadCh) + // Start the event loop and return pool.wg.Add(1) go pool.loop() @@ -264,31 +265,28 @@ func (pool *TxPool) loop() { journal := time.NewTicker(pool.config.Rejournal) defer journal.Stop() + // Track the previous head headers for transaction reorgs + head := pool.chain.CurrentBlock() + // Keep waiting for and reacting to the various events for { select { // Handle ChainHeadEvent case ev := <-pool.chainHeadCh: - pool.mu.Lock() if ev.Block != nil { + pool.mu.Lock() if pool.chainconfig.IsHomestead(ev.Block.Number()) { pool.homestead = true } + pool.reset(head.Header(), ev.Block.Header()) + head = ev.Block + pool.mu.Unlock() } - pool.reset() - pool.mu.Unlock() // Be unsubscribed due to system stopped case <-pool.chainHeadSub.Err(): return - // Handle RemovedTransactionEvent - case ev := <-pool.rmTxCh: - pool.addTxs(ev.Txs, false) - // Be unsubscribed due to system stopped - case <-pool.rmTxSub.Err(): - return - // Handle stats reporting ticks case <-report.C: pool.mu.RLock() @@ -333,28 +331,85 @@ func (pool *TxPool) loop() { // lockedReset is a wrapper around reset to allow calling it in a thread safe // manner. This method is only ever used in the tester! -func (pool *TxPool) lockedReset() { +func (pool *TxPool) lockedReset(oldHead, newHead *types.Header) { pool.mu.Lock() defer pool.mu.Unlock() - pool.reset() + pool.reset(oldHead, newHead) } // reset retrieves the current state of the blockchain and ensures the content // of the transaction pool is valid with regard to the chain state. -func (pool *TxPool) reset() { - currentState, err := pool.blockChain.State() +func (pool *TxPool) reset(oldHead, newHead *types.Header) { + // If we're reorging an old state, reinject all dropped transactions + var reinject types.Transactions + + if oldHead != nil && oldHead.Hash() != newHead.ParentHash { + // If the reorg is too deep, avoid doing it (will happen during fast sync) + oldNum := oldHead.Number.Uint64() + newNum := newHead.Number.Uint64() + + if depth := uint64(math.Abs(float64(oldNum) - float64(newNum))); depth > 64 { + log.Warn("Skipping deep transaction reorg", "depth", depth) + } else { + // Reorg seems shallow enough to pull in all transactions into memory + var discarded, included types.Transactions + + var ( + rem = pool.chain.GetBlock(oldHead.Hash(), oldHead.Number.Uint64()) + add = pool.chain.GetBlock(newHead.Hash(), newHead.Number.Uint64()) + ) + for rem.NumberU64() > add.NumberU64() { + discarded = append(discarded, rem.Transactions()...) + if rem = pool.chain.GetBlock(rem.ParentHash(), rem.NumberU64()-1); rem == nil { + log.Error("Unrooted old chain seen by tx pool", "block", oldHead.Number, "hash", oldHead.Hash()) + return + } + } + for add.NumberU64() > rem.NumberU64() { + included = append(included, add.Transactions()...) + if add = pool.chain.GetBlock(add.ParentHash(), add.NumberU64()-1); add == nil { + log.Error("Unrooted new chain seen by tx pool", "block", newHead.Number, "hash", newHead.Hash()) + return + } + } + for rem.Hash() != add.Hash() { + discarded = append(discarded, rem.Transactions()...) + if rem = pool.chain.GetBlock(rem.ParentHash(), rem.NumberU64()-1); rem == nil { + log.Error("Unrooted old chain seen by tx pool", "block", oldHead.Number, "hash", oldHead.Hash()) + return + } + included = append(included, add.Transactions()...) + if add = pool.chain.GetBlock(add.ParentHash(), add.NumberU64()-1); add == nil { + log.Error("Unrooted new chain seen by tx pool", "block", newHead.Number, "hash", newHead.Hash()) + return + } + } + reinject = types.TxDifference(discarded, included) + } + } + // Initialize the internal state to the current head + if newHead == nil { + newHead = pool.chain.CurrentBlock().Header() // Special case during testing + } + statedb, err := pool.chain.StateAt(newHead.Root) if err != nil { - log.Error("Failed reset txpool state", "err", err) + log.Error("Failed to reset txpool state", "err", err) return } - pool.pendingState = state.ManageState(currentState) + pool.currentState = statedb + pool.pendingState = state.ManageState(statedb) + pool.currentMaxGas = newHead.GasLimit + + // Inject any transactions discarded due to reorgs + log.Debug("Reinjecting stale transactions", "count", len(reinject)) + pool.addTxsLocked(reinject, false) // validate the pool of pending transactions, this will remove // any transactions that have been included in the block or // have been invalidated because of another transaction (e.g. // higher gas price) - pool.demoteUnexecutables(currentState) + pool.demoteUnexecutables() // Update all accounts to the latest known pending nonce for addr, list := range pool.pending { @@ -363,16 +418,16 @@ func (pool *TxPool) reset() { } // Check the queue and move transactions over to the pending if possible // or remove those that have become invalid - pool.promoteExecutables(currentState, nil) + pool.promoteExecutables(nil) } // Stop terminates the transaction pool. func (pool *TxPool) Stop() { // Unsubscribe all subscriptions registered from txpool pool.scope.Close() + // Unsubscribe subscriptions registered from blockchain pool.chainHeadSub.Unsubscribe() - pool.rmTxSub.Unsubscribe() pool.wg.Wait() if pool.journal != nil { @@ -442,8 +497,8 @@ func (pool *TxPool) stats() (int, int) { // Content retrieves the data content of the transaction pool, returning all the // pending as well as queued transactions, grouped by account and sorted by nonce. func (pool *TxPool) Content() (map[common.Address]types.Transactions, map[common.Address]types.Transactions) { - pool.mu.RLock() - defer pool.mu.RUnlock() + pool.mu.Lock() + defer pool.mu.Unlock() pending := make(map[common.Address]types.Transactions) for addr, list := range pool.pending { @@ -499,7 +554,7 @@ func (pool *TxPool) validateTx(tx *types.Transaction, local bool) error { return ErrNegativeValue } // Ensure the transaction doesn't exceed the current block limit gas. - if pool.blockChain.GasLimit().Cmp(tx.Gas()) < 0 { + if pool.currentMaxGas.Cmp(tx.Gas()) < 0 { return ErrGasLimit } // Make sure the transaction is signed properly @@ -513,16 +568,12 @@ func (pool *TxPool) validateTx(tx *types.Transaction, local bool) error { return ErrUnderpriced } // Ensure the transaction adheres to nonce ordering - currentState, err := pool.blockChain.State() - if err != nil { - return err - } - if currentState.GetNonce(from) > tx.Nonce() { + if pool.currentState.GetNonce(from) > tx.Nonce() { return ErrNonceTooLow } // Transactor should have enough funds to cover the costs // cost == V + GP * GL - if currentState.GetBalance(from).Cmp(tx.Cost()) < 0 { + if pool.currentState.GetBalance(from).Cmp(tx.Cost()) < 0 { return ErrInsufficientFunds } intrGas := IntrinsicGas(tx.Data(), tx.To() == nil, pool.homestead) @@ -721,12 +772,8 @@ func (pool *TxPool) addTx(tx *types.Transaction, local bool) error { } // If we added a new transaction, run promotion checks and return if !replace { - state, err := pool.blockChain.State() - if err != nil { - return err - } from, _ := types.Sender(pool.signer, tx) // already validated - pool.promoteExecutables(state, []common.Address{from}) + pool.promoteExecutables([]common.Address{from}) } return nil } @@ -736,6 +783,12 @@ func (pool *TxPool) addTxs(txs []*types.Transaction, local bool) error { pool.mu.Lock() defer pool.mu.Unlock() + return pool.addTxsLocked(txs, local) +} + +// addTxsLocked attempts to queue a batch of transactions if they are valid, +// whilst assuming the transaction pool lock is already held. +func (pool *TxPool) addTxsLocked(txs []*types.Transaction, local bool) error { // Add the batch of transaction, tracking the accepted ones dirty := make(map[common.Address]struct{}) for _, tx := range txs { @@ -748,15 +801,11 @@ func (pool *TxPool) addTxs(txs []*types.Transaction, local bool) error { } // Only reprocess the internal state if something was actually added if len(dirty) > 0 { - state, err := pool.blockChain.State() - if err != nil { - return err - } addrs := make([]common.Address, 0, len(dirty)) for addr, _ := range dirty { addrs = append(addrs, addr) } - pool.promoteExecutables(state, addrs) + pool.promoteExecutables(addrs) } return nil } @@ -770,24 +819,6 @@ func (pool *TxPool) Get(hash common.Hash) *types.Transaction { return pool.all[hash] } -// Remove removes the transaction with the given hash from the pool. -func (pool *TxPool) Remove(hash common.Hash) { - pool.mu.Lock() - defer pool.mu.Unlock() - - pool.removeTx(hash) -} - -// RemoveBatch removes all given transactions from the pool. -func (pool *TxPool) RemoveBatch(txs types.Transactions) { - pool.mu.Lock() - defer pool.mu.Unlock() - - for _, tx := range txs { - pool.removeTx(tx.Hash()) - } -} - // removeTx removes a single transaction from the queue, moving all subsequent // transactions back to the future queue. func (pool *TxPool) removeTx(hash common.Hash) { @@ -834,9 +865,7 @@ func (pool *TxPool) removeTx(hash common.Hash) { // promoteExecutables moves transactions that have become processable from the // future queue to the set of pending transactions. During this process, all // invalidated transactions (low nonce, low balance) are deleted. -func (pool *TxPool) promoteExecutables(state *state.StateDB, accounts []common.Address) { - gaslimit := pool.blockChain.GasLimit() - +func (pool *TxPool) promoteExecutables(accounts []common.Address) { // Gather all the accounts potentially needing updates if accounts == nil { accounts = make([]common.Address, 0, len(pool.queue)) @@ -851,14 +880,14 @@ func (pool *TxPool) promoteExecutables(state *state.StateDB, accounts []common.A continue // Just in case someone calls with a non existing account } // Drop all transactions that are deemed too old (low nonce) - for _, tx := range list.Forward(state.GetNonce(addr)) { + for _, tx := range list.Forward(pool.currentState.GetNonce(addr)) { hash := tx.Hash() log.Trace("Removed old queued transaction", "hash", hash) delete(pool.all, hash) pool.priced.Removed() } // Drop all transactions that are too costly (low balance or out of gas) - drops, _ := list.Filter(state.GetBalance(addr), gaslimit) + drops, _ := list.Filter(pool.currentState.GetBalance(addr), pool.currentMaxGas) for _, tx := range drops { hash := tx.Hash() log.Trace("Removed unpayable queued transaction", "hash", hash) @@ -1003,12 +1032,10 @@ func (pool *TxPool) promoteExecutables(state *state.StateDB, accounts []common.A // demoteUnexecutables removes invalid and processed transactions from the pools // executable/pending queue and any subsequent transactions that become unexecutable // are moved back into the future queue. -func (pool *TxPool) demoteUnexecutables(state *state.StateDB) { - gaslimit := pool.blockChain.GasLimit() - +func (pool *TxPool) demoteUnexecutables() { // Iterate over all accounts and demote any non-executable transactions for addr, list := range pool.pending { - nonce := state.GetNonce(addr) + nonce := pool.currentState.GetNonce(addr) // Drop all transactions that are deemed too old (low nonce) for _, tx := range list.Forward(nonce) { @@ -1018,7 +1045,7 @@ func (pool *TxPool) demoteUnexecutables(state *state.StateDB) { pool.priced.Removed() } // Drop all transactions that are too costly (low balance or out of gas), and queue any invalids back for later - drops, invalids := list.Filter(state.GetBalance(addr), gaslimit) + drops, invalids := list.Filter(pool.currentState.GetBalance(addr), pool.currentMaxGas) for _, tx := range drops { hash := tx.Hash() log.Trace("Removed unpayable pending transaction", "hash", hash) @@ -1031,6 +1058,14 @@ func (pool *TxPool) demoteUnexecutables(state *state.StateDB) { log.Trace("Demoting pending transaction", "hash", hash) pool.enqueueTx(hash, tx) } + // If there's a gap in front, warn (should never happen) and postpone all transactions + if list.Len() > 0 && list.txs.Get(nonce) == nil { + for _, tx := range list.Cap(0) { + hash := tx.Hash() + log.Error("Demoting invalidated transaction", "hash", hash) + pool.enqueueTx(hash, tx) + } + } // Delete the entire queue entry if it became empty. if list.Empty() { delete(pool.pending, addr) diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index ee1ddd4bbc..17d7368774 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -48,25 +48,26 @@ type testBlockChain struct { statedb *state.StateDB gasLimit *big.Int chainHeadFeed *event.Feed - rmTxFeed *event.Feed } -func (bc *testBlockChain) State() (*state.StateDB, error) { +func (bc *testBlockChain) CurrentBlock() *types.Block { + return types.NewBlock(&types.Header{ + GasLimit: bc.gasLimit, + }, nil, nil, nil) +} + +func (bc *testBlockChain) GetBlock(hash common.Hash, number uint64) *types.Block { + return bc.CurrentBlock() +} + +func (bc *testBlockChain) StateAt(common.Hash) (*state.StateDB, error) { return bc.statedb, nil } -func (bc *testBlockChain) GasLimit() *big.Int { - return new(big.Int).Set(bc.gasLimit) -} - func (bc *testBlockChain) SubscribeChainHeadEvent(ch chan<- ChainHeadEvent) event.Subscription { return bc.chainHeadFeed.Subscribe(ch) } -func (bc *testBlockChain) SubscribeRemovedTxEvent(ch chan<- RemovedTransactionEvent) event.Subscription { - return bc.rmTxFeed.Subscribe(ch) -} - func transaction(nonce uint64, gaslimit *big.Int, key *ecdsa.PrivateKey) *types.Transaction { return pricedTransaction(nonce, gaslimit, big.NewInt(1), key) } @@ -79,7 +80,7 @@ func pricedTransaction(nonce uint64, gaslimit, gasprice *big.Int, key *ecdsa.Pri func setupTxPool() (*TxPool, *ecdsa.PrivateKey) { db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} + blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} key, _ := crypto.GenerateKey() pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) @@ -159,7 +160,7 @@ func TestStateChangeDuringPoolReset(t *testing.T) { // setup pool with 2 transaction in it statedb.SetBalance(address, new(big.Int).SetUint64(params.Ether)) - blockchain := &testChain{&testBlockChain{statedb, big.NewInt(1000000000), new(event.Feed), new(event.Feed)}, address, &trigger} + blockchain := &testChain{&testBlockChain{statedb, big.NewInt(1000000000), new(event.Feed)}, address, &trigger} tx0 := transaction(0, big.NewInt(100000), key) tx1 := transaction(1, big.NewInt(100000), key) @@ -182,7 +183,7 @@ func TestStateChangeDuringPoolReset(t *testing.T) { // trigger state change in the background trigger = true - pool.lockedReset() + pool.lockedReset(nil, nil) pendingTx, err := pool.Pending() if err != nil { @@ -205,20 +206,20 @@ func TestInvalidTransactions(t *testing.T) { tx := transaction(0, big.NewInt(100), key) from, _ := deriveSender(tx) - currentState, _ := pool.blockChain.State() - currentState.AddBalance(from, big.NewInt(1)) + + pool.currentState.AddBalance(from, big.NewInt(1)) if err := pool.AddRemote(tx); err != ErrInsufficientFunds { t.Error("expected", ErrInsufficientFunds) } balance := new(big.Int).Add(tx.Value(), new(big.Int).Mul(tx.Gas(), tx.GasPrice())) - currentState.AddBalance(from, balance) + pool.currentState.AddBalance(from, balance) if err := pool.AddRemote(tx); err != ErrIntrinsicGas { t.Error("expected", ErrIntrinsicGas, "got", err) } - currentState.SetNonce(from, 1) - currentState.AddBalance(from, big.NewInt(0xffffffffffffff)) + pool.currentState.SetNonce(from, 1) + pool.currentState.AddBalance(from, big.NewInt(0xffffffffffffff)) tx = transaction(0, big.NewInt(100000), key) if err := pool.AddRemote(tx); err != ErrNonceTooLow { t.Error("expected", ErrNonceTooLow) @@ -240,21 +241,20 @@ func TestTransactionQueue(t *testing.T) { tx := transaction(0, big.NewInt(100), key) from, _ := deriveSender(tx) - currentState, _ := pool.blockChain.State() - currentState.AddBalance(from, big.NewInt(1000)) - pool.lockedReset() + pool.currentState.AddBalance(from, big.NewInt(1000)) + pool.lockedReset(nil, nil) pool.enqueueTx(tx.Hash(), tx) - pool.promoteExecutables(currentState, []common.Address{from}) + pool.promoteExecutables([]common.Address{from}) if len(pool.pending) != 1 { t.Error("expected valid txs to be 1 is", len(pool.pending)) } tx = transaction(1, big.NewInt(100), key) from, _ = deriveSender(tx) - currentState.SetNonce(from, 2) + pool.currentState.SetNonce(from, 2) pool.enqueueTx(tx.Hash(), tx) - pool.promoteExecutables(currentState, []common.Address{from}) + pool.promoteExecutables([]common.Address{from}) if _, ok := pool.pending[from].txs.items[tx.Nonce()]; ok { t.Error("expected transaction to be in tx pool") } @@ -270,15 +270,14 @@ func TestTransactionQueue(t *testing.T) { tx2 := transaction(10, big.NewInt(100), key) tx3 := transaction(11, big.NewInt(100), key) from, _ = deriveSender(tx1) - currentState, _ = pool.blockChain.State() - currentState.AddBalance(from, big.NewInt(1000)) - pool.lockedReset() + pool.currentState.AddBalance(from, big.NewInt(1000)) + pool.lockedReset(nil, nil) pool.enqueueTx(tx1.Hash(), tx1) pool.enqueueTx(tx2.Hash(), tx2) pool.enqueueTx(tx3.Hash(), tx3) - pool.promoteExecutables(currentState, []common.Address{from}) + pool.promoteExecutables([]common.Address{from}) if len(pool.pending) != 1 { t.Error("expected tx pool to be 1, got", len(pool.pending)) @@ -288,45 +287,13 @@ func TestTransactionQueue(t *testing.T) { } } -func TestRemoveTx(t *testing.T) { - pool, key := setupTxPool() - defer pool.Stop() - - addr := crypto.PubkeyToAddress(key.PublicKey) - currentState, _ := pool.blockChain.State() - currentState.AddBalance(addr, big.NewInt(1)) - - tx1 := transaction(0, big.NewInt(100), key) - tx2 := transaction(2, big.NewInt(100), key) - - pool.promoteTx(addr, tx1.Hash(), tx1) - pool.enqueueTx(tx2.Hash(), tx2) - - if len(pool.queue) != 1 { - t.Error("expected queue to be 1, got", len(pool.queue)) - } - if len(pool.pending) != 1 { - t.Error("expected pending to be 1, got", len(pool.pending)) - } - pool.Remove(tx1.Hash()) - pool.Remove(tx2.Hash()) - - if len(pool.queue) > 0 { - t.Error("expected queue to be 0, got", len(pool.queue)) - } - if len(pool.pending) > 0 { - t.Error("expected pending to be 0, got", len(pool.pending)) - } -} - func TestNegativeValue(t *testing.T) { pool, key := setupTxPool() defer pool.Stop() tx, _ := types.SignTx(types.NewTransaction(0, common.Address{}, big.NewInt(-1), big.NewInt(100), big.NewInt(1), nil), types.HomesteadSigner{}, key) from, _ := deriveSender(tx) - currentState, _ := pool.blockChain.State() - currentState.AddBalance(from, big.NewInt(1)) + pool.currentState.AddBalance(from, big.NewInt(1)) if err := pool.AddRemote(tx); err != ErrNegativeValue { t.Error("expected", ErrNegativeValue, "got", err) } @@ -340,10 +307,10 @@ func TestTransactionChainFork(t *testing.T) { resetState := func() { db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - pool.blockChain = &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} - currentState, _ := pool.blockChain.State() - currentState.AddBalance(addr, big.NewInt(100000000000000)) - pool.lockedReset() + statedb.AddBalance(addr, big.NewInt(100000000000000)) + + pool.chain = &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} + pool.lockedReset(nil, nil) } resetState() @@ -351,7 +318,7 @@ func TestTransactionChainFork(t *testing.T) { if _, err := pool.add(tx, false); err != nil { t.Error("didn't expect error", err) } - pool.RemoveBatch([]*types.Transaction{tx}) + pool.removeTx(tx.Hash()) // reset the pool's internal state resetState() @@ -368,10 +335,10 @@ func TestTransactionDoubleNonce(t *testing.T) { resetState := func() { db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - pool.blockChain = &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} - currentState, _ := pool.blockChain.State() - currentState.AddBalance(addr, big.NewInt(100000000000000)) - pool.lockedReset() + statedb.AddBalance(addr, big.NewInt(100000000000000)) + + pool.chain = &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} + pool.lockedReset(nil, nil) } resetState() @@ -387,8 +354,7 @@ func TestTransactionDoubleNonce(t *testing.T) { if replace, err := pool.add(tx2, false); err != nil || !replace { t.Errorf("second transaction insert failed (%v) or not reported replacement (%v)", err, replace) } - state, _ := pool.blockChain.State() - pool.promoteExecutables(state, []common.Address{addr}) + pool.promoteExecutables([]common.Address{addr}) if pool.pending[addr].Len() != 1 { t.Error("expected 1 pending transactions, got", pool.pending[addr].Len()) } @@ -397,7 +363,7 @@ func TestTransactionDoubleNonce(t *testing.T) { } // Add the third transaction and ensure it's not saved (smaller price) pool.add(tx3, false) - pool.promoteExecutables(state, []common.Address{addr}) + pool.promoteExecutables([]common.Address{addr}) if pool.pending[addr].Len() != 1 { t.Error("expected 1 pending transactions, got", pool.pending[addr].Len()) } @@ -415,8 +381,7 @@ func TestMissingNonce(t *testing.T) { defer pool.Stop() addr := crypto.PubkeyToAddress(key.PublicKey) - currentState, _ := pool.blockChain.State() - currentState.AddBalance(addr, big.NewInt(100000000000000)) + pool.currentState.AddBalance(addr, big.NewInt(100000000000000)) tx := transaction(1, big.NewInt(100000), key) if _, err := pool.add(tx, false); err != nil { t.Error("didn't expect error", err) @@ -432,47 +397,25 @@ func TestMissingNonce(t *testing.T) { } } -func TestNonceRecovery(t *testing.T) { +func TestTransactionNonceRecovery(t *testing.T) { const n = 10 pool, key := setupTxPool() defer pool.Stop() addr := crypto.PubkeyToAddress(key.PublicKey) - currentState, _ := pool.blockChain.State() - currentState.SetNonce(addr, n) - currentState.AddBalance(addr, big.NewInt(100000000000000)) - pool.lockedReset() + pool.currentState.SetNonce(addr, n) + pool.currentState.AddBalance(addr, big.NewInt(100000000000000)) + pool.lockedReset(nil, nil) + tx := transaction(n, big.NewInt(100000), key) if err := pool.AddRemote(tx); err != nil { t.Error(err) } // simulate some weird re-order of transactions and missing nonce(s) - currentState.SetNonce(addr, n-1) - pool.lockedReset() - if fn := pool.pendingState.GetNonce(addr); fn != n+1 { - t.Errorf("expected nonce to be %d, got %d", n+1, fn) - } -} - -func TestRemovedTxEvent(t *testing.T) { - pool, key := setupTxPool() - defer pool.Stop() - - tx := transaction(0, big.NewInt(1000000), key) - from, _ := deriveSender(tx) - currentState, _ := pool.blockChain.State() - currentState.AddBalance(from, big.NewInt(1000000000000)) - pool.lockedReset() - blockChain, _ := pool.blockChain.(*testBlockChain) - blockChain.rmTxFeed.Send(RemovedTransactionEvent{types.Transactions{tx}}) - blockChain.chainHeadFeed.Send(ChainHeadEvent{nil}) - // wait for handling events - <-time.After(500 * time.Millisecond) - if pool.pending[from].Len() != 1 { - t.Error("expected 1 pending tx, got", pool.pending[from].Len()) - } - if len(pool.all) != 1 { - t.Error("expected 1 total transactions, got", len(pool.all)) + pool.currentState.SetNonce(addr, n-1) + pool.lockedReset(nil, nil) + if fn := pool.pendingState.GetNonce(addr); fn != n-1 { + t.Errorf("expected nonce to be %d, got %d", n-1, fn) } } @@ -484,9 +427,7 @@ func TestTransactionDropping(t *testing.T) { defer pool.Stop() account, _ := deriveSender(transaction(0, big.NewInt(0), key)) - - state, _ := pool.blockChain.State() - state.AddBalance(account, big.NewInt(1000)) + pool.currentState.AddBalance(account, big.NewInt(1000)) // Add some pending and some queued transactions var ( @@ -514,7 +455,7 @@ func TestTransactionDropping(t *testing.T) { if len(pool.all) != 6 { t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), 6) } - pool.lockedReset() + pool.lockedReset(nil, nil) if pool.pending[account].Len() != 3 { t.Errorf("pending transaction mismatch: have %d, want %d", pool.pending[account].Len(), 3) } @@ -525,8 +466,8 @@ func TestTransactionDropping(t *testing.T) { t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), 6) } // Reduce the balance of the account, and check that invalidated transactions are dropped - state.AddBalance(account, big.NewInt(-650)) - pool.lockedReset() + pool.currentState.AddBalance(account, big.NewInt(-650)) + pool.lockedReset(nil, nil) if _, ok := pool.pending[account].txs.items[tx0.Nonce()]; !ok { t.Errorf("funded pending transaction missing: %v", tx0) @@ -550,8 +491,8 @@ func TestTransactionDropping(t *testing.T) { t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), 4) } // Reduce the block gas limit, check that invalidated transactions are dropped - pool.blockChain.(*testBlockChain).gasLimit = big.NewInt(100) - pool.lockedReset() + pool.chain.(*testBlockChain).gasLimit = big.NewInt(100) + pool.lockedReset(nil, nil) if _, ok := pool.pending[account].txs.items[tx0.Nonce()]; !ok { t.Errorf("funded pending transaction missing: %v", tx0) @@ -579,9 +520,7 @@ func TestTransactionPostponing(t *testing.T) { defer pool.Stop() account, _ := deriveSender(transaction(0, big.NewInt(0), key)) - - state, _ := pool.blockChain.State() - state.AddBalance(account, big.NewInt(1000)) + pool.currentState.AddBalance(account, big.NewInt(1000)) // Add a batch consecutive pending transactions for validation txns := []*types.Transaction{} @@ -605,7 +544,7 @@ func TestTransactionPostponing(t *testing.T) { if len(pool.all) != len(txns) { t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), len(txns)) } - pool.lockedReset() + pool.lockedReset(nil, nil) if pool.pending[account].Len() != len(txns) { t.Errorf("pending transaction mismatch: have %d, want %d", pool.pending[account].Len(), len(txns)) } @@ -616,8 +555,8 @@ func TestTransactionPostponing(t *testing.T) { t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), len(txns)) } // Reduce the balance of the account, and check that transactions are reorganised - state.AddBalance(account, big.NewInt(-750)) - pool.lockedReset() + pool.currentState.AddBalance(account, big.NewInt(-750)) + pool.lockedReset(nil, nil) if _, ok := pool.pending[account].txs.items[txns[0].Nonce()]; !ok { t.Errorf("tx %d: valid and funded transaction missing from pending pool: %v", 0, txns[0]) @@ -655,10 +594,7 @@ func TestTransactionQueueAccountLimiting(t *testing.T) { defer pool.Stop() account, _ := deriveSender(transaction(0, big.NewInt(0), key)) - - state, _ := pool.blockChain.State() - state.AddBalance(account, big.NewInt(1000000)) - pool.lockedReset() + pool.currentState.AddBalance(account, big.NewInt(1000000)) // Keep queuing up transactions and make sure all above a limit are dropped for i := uint64(1); i <= testTxPoolConfig.AccountQueue+5; i++ { @@ -699,7 +635,7 @@ func testTransactionQueueGlobalLimiting(t *testing.T, nolocals bool) { // Create the pool to test the limit enforcement with db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} + blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} config := testTxPoolConfig config.NoLocals = nolocals @@ -709,12 +645,10 @@ func testTransactionQueueGlobalLimiting(t *testing.T, nolocals bool) { defer pool.Stop() // Create a number of test accounts and fund them (last one will be the local) - state, _ := pool.blockChain.State() - keys := make([]*ecdsa.PrivateKey, 5) for i := 0; i < len(keys); i++ { keys[i], _ = crypto.GenerateKey() - state.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) } local := keys[len(keys)-1] @@ -790,7 +724,7 @@ func testTransactionQueueTimeLimiting(t *testing.T, nolocals bool) { // Create the pool to test the non-expiration enforcement db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} + blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} config := testTxPoolConfig config.Lifetime = time.Second @@ -803,9 +737,8 @@ func testTransactionQueueTimeLimiting(t *testing.T, nolocals bool) { local, _ := crypto.GenerateKey() remote, _ := crypto.GenerateKey() - state, _ := pool.blockChain.State() - state.AddBalance(crypto.PubkeyToAddress(local.PublicKey), big.NewInt(1000000000)) - state.AddBalance(crypto.PubkeyToAddress(remote.PublicKey), big.NewInt(1000000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(local.PublicKey), big.NewInt(1000000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(remote.PublicKey), big.NewInt(1000000000)) // Add the two transactions and ensure they both are queued up if err := pool.AddLocal(pricedTransaction(1, big.NewInt(100000), big.NewInt(1), local)); err != nil { @@ -854,10 +787,7 @@ func TestTransactionPendingLimiting(t *testing.T) { defer pool.Stop() account, _ := deriveSender(transaction(0, big.NewInt(0), key)) - - state, _ := pool.blockChain.State() - state.AddBalance(account, big.NewInt(1000000)) - pool.lockedReset() + pool.currentState.AddBalance(account, big.NewInt(1000000)) // Keep queuing up transactions and make sure all above a limit are dropped for i := uint64(0); i < testTxPoolConfig.AccountQueue+5; i++ { @@ -887,8 +817,7 @@ func testTransactionLimitingEquivalency(t *testing.T, origin uint64) { defer pool1.Stop() account1, _ := deriveSender(transaction(0, big.NewInt(0), key1)) - state1, _ := pool1.blockChain.State() - state1.AddBalance(account1, big.NewInt(1000000)) + pool1.currentState.AddBalance(account1, big.NewInt(1000000)) for i := uint64(0); i < testTxPoolConfig.AccountQueue+5; i++ { if err := pool1.AddRemote(transaction(origin+i, big.NewInt(100000), key1)); err != nil { @@ -900,8 +829,7 @@ func testTransactionLimitingEquivalency(t *testing.T, origin uint64) { defer pool2.Stop() account2, _ := deriveSender(transaction(0, big.NewInt(0), key2)) - state2, _ := pool2.blockChain.State() - state2.AddBalance(account2, big.NewInt(1000000)) + pool2.currentState.AddBalance(account2, big.NewInt(1000000)) txns := []*types.Transaction{} for i := uint64(0); i < testTxPoolConfig.AccountQueue+5; i++ { @@ -934,7 +862,7 @@ func TestTransactionPendingGlobalLimiting(t *testing.T) { // Create the pool to test the limit enforcement with db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} + blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} config := testTxPoolConfig config.GlobalSlots = config.AccountSlots * 10 @@ -943,12 +871,10 @@ func TestTransactionPendingGlobalLimiting(t *testing.T) { defer pool.Stop() // Create a number of test accounts and fund them - state, _ := pool.blockChain.State() - keys := make([]*ecdsa.PrivateKey, 5) for i := 0; i < len(keys); i++ { keys[i], _ = crypto.GenerateKey() - state.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) } // Generate and queue a batch of transactions nonces := make(map[common.Address]uint64) @@ -976,12 +902,12 @@ func TestTransactionPendingGlobalLimiting(t *testing.T) { } } -// Tests that if transactions start being capped, transasctions are also removed from 'all' +// Tests that if transactions start being capped, transactions are also removed from 'all' func TestTransactionCapClearsFromAll(t *testing.T) { // Create the pool to test the limit enforcement with db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} + blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} config := testTxPoolConfig config.AccountSlots = 2 @@ -992,11 +918,9 @@ func TestTransactionCapClearsFromAll(t *testing.T) { defer pool.Stop() // Create a number of test accounts and fund them - state, _ := pool.blockChain.State() - key, _ := crypto.GenerateKey() addr := crypto.PubkeyToAddress(key.PublicKey) - state.AddBalance(addr, big.NewInt(1000000)) + pool.currentState.AddBalance(addr, big.NewInt(1000000)) txs := types.Transactions{} for j := 0; j < int(config.GlobalSlots)*2; j++ { @@ -1016,7 +940,7 @@ func TestTransactionPendingMinimumAllowance(t *testing.T) { // Create the pool to test the limit enforcement with db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} + blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} config := testTxPoolConfig config.GlobalSlots = 0 @@ -1025,12 +949,10 @@ func TestTransactionPendingMinimumAllowance(t *testing.T) { defer pool.Stop() // Create a number of test accounts and fund them - state, _ := pool.blockChain.State() - keys := make([]*ecdsa.PrivateKey, 5) for i := 0; i < len(keys); i++ { keys[i], _ = crypto.GenerateKey() - state.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) } // Generate and queue a batch of transactions nonces := make(map[common.Address]uint64) @@ -1065,18 +987,16 @@ func TestTransactionPoolRepricing(t *testing.T) { // Create the pool to test the pricing enforcement with db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} + blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) defer pool.Stop() // Create a number of test accounts and fund them - state, _ := pool.blockChain.State() - keys := make([]*ecdsa.PrivateKey, 3) for i := 0; i < len(keys); i++ { keys[i], _ = crypto.GenerateKey() - state.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) } // Generate and queue a batch of transactions, both pending and queued txs := types.Transactions{} @@ -1141,6 +1061,64 @@ func TestTransactionPoolRepricing(t *testing.T) { } } +// Tests that setting the transaction pool gas price to a higher value does not +// remove local transactions. +func TestTransactionPoolRepricingKeepsLocals(t *testing.T) { + // Create the pool to test the pricing enforcement with + db, _ := ethdb.NewMemDatabase() + statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} + + pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) + defer pool.Stop() + + // Create a number of test accounts and fund them + keys := make([]*ecdsa.PrivateKey, 3) + for i := 0; i < len(keys); i++ { + keys[i], _ = crypto.GenerateKey() + pool.currentState.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000*1000000)) + } + // Create transaction (both pending and queued) with a linearly growing gasprice + for i := uint64(0); i < 500; i++ { + // Add pending + p_tx := pricedTransaction(i, big.NewInt(100000), big.NewInt(int64(i)), keys[2]) + if err := pool.AddLocal(p_tx); err != nil { + t.Fatal(err) + } + // Add queued + q_tx := pricedTransaction(i+501, big.NewInt(100000), big.NewInt(int64(i)), keys[2]) + if err := pool.AddLocal(q_tx); err != nil { + t.Fatal(err) + } + } + pending, queued := pool.Stats() + expPending, expQueued := 500, 500 + validate := func() { + pending, queued = pool.Stats() + if pending != expPending { + t.Fatalf("pending transactions mismatched: have %d, want %d", pending, expPending) + } + if queued != expQueued { + t.Fatalf("queued transactions mismatched: have %d, want %d", queued, expQueued) + } + + if err := validateTxPoolInternals(pool); err != nil { + t.Fatalf("pool internal state corrupted: %v", err) + } + } + validate() + + // Reprice the pool and check that nothing is dropped + pool.SetGasPrice(big.NewInt(2)) + validate() + + pool.SetGasPrice(big.NewInt(2)) + pool.SetGasPrice(big.NewInt(4)) + pool.SetGasPrice(big.NewInt(8)) + pool.SetGasPrice(big.NewInt(100)) + validate() +} + // Tests that when the pool reaches its global transaction limit, underpriced // transactions are gradually shifted out for more expensive ones and any gapped // pending transactions are moved into te queue. @@ -1150,7 +1128,7 @@ func TestTransactionPoolUnderpricing(t *testing.T) { // Create the pool to test the pricing enforcement with db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} + blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} config := testTxPoolConfig config.GlobalSlots = 2 @@ -1160,12 +1138,10 @@ func TestTransactionPoolUnderpricing(t *testing.T) { defer pool.Stop() // Create a number of test accounts and fund them - state, _ := pool.blockChain.State() - keys := make([]*ecdsa.PrivateKey, 3) for i := 0; i < len(keys); i++ { keys[i], _ = crypto.GenerateKey() - state.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) } // Generate and queue a batch of transactions, both pending and queued txs := types.Transactions{} @@ -1238,16 +1214,14 @@ func TestTransactionReplacement(t *testing.T) { // Create the pool to test the pricing enforcement with db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} + blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) defer pool.Stop() // Create a test account to add transactions with key, _ := crypto.GenerateKey() - - state, _ := pool.blockChain.State() - state.AddBalance(crypto.PubkeyToAddress(key.PublicKey), big.NewInt(1000000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(key.PublicKey), big.NewInt(1000000000)) // Add pending transactions, ensuring the minimum price bump is enforced for replacement (for ultra low prices too) price := int64(100) @@ -1318,7 +1292,7 @@ func testTransactionJournaling(t *testing.T, nolocals bool) { // Create the original pool to inject transaction into the journal db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} + blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} config := testTxPoolConfig config.NoLocals = nolocals @@ -1331,9 +1305,8 @@ func testTransactionJournaling(t *testing.T, nolocals bool) { local, _ := crypto.GenerateKey() remote, _ := crypto.GenerateKey() - statedb, _ = pool.blockChain.State() - statedb.AddBalance(crypto.PubkeyToAddress(local.PublicKey), big.NewInt(1000000000)) - statedb.AddBalance(crypto.PubkeyToAddress(remote.PublicKey), big.NewInt(1000000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(local.PublicKey), big.NewInt(1000000000)) + pool.currentState.AddBalance(crypto.PubkeyToAddress(remote.PublicKey), big.NewInt(1000000000)) // Add three local and a remote transactions and ensure they are queued up if err := pool.AddLocal(pricedTransaction(0, big.NewInt(100000), big.NewInt(1), local)); err != nil { @@ -1361,7 +1334,7 @@ func testTransactionJournaling(t *testing.T, nolocals bool) { // Terminate the old pool, bump the local nonce, create a new pool and ensure relevant transaction survive pool.Stop() statedb.SetNonce(crypto.PubkeyToAddress(local.PublicKey), 1) - blockchain = &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} + blockchain = &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} pool = NewTxPool(config, params.TestChainConfig, blockchain) pending, queued = pool.Stats() @@ -1382,11 +1355,11 @@ func testTransactionJournaling(t *testing.T, nolocals bool) { } // Bump the nonce temporarily and ensure the newly invalidated transaction is removed statedb.SetNonce(crypto.PubkeyToAddress(local.PublicKey), 2) - pool.lockedReset() + pool.lockedReset(nil, nil) time.Sleep(2 * config.Rejournal) pool.Stop() statedb.SetNonce(crypto.PubkeyToAddress(local.PublicKey), 1) - blockchain = &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed), new(event.Feed)} + blockchain = &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)} pool = NewTxPool(config, params.TestChainConfig, blockchain) pending, queued = pool.Stats() @@ -1420,8 +1393,7 @@ func benchmarkPendingDemotion(b *testing.B, size int) { defer pool.Stop() account, _ := deriveSender(transaction(0, big.NewInt(0), key)) - state, _ := pool.blockChain.State() - state.AddBalance(account, big.NewInt(1000000)) + pool.currentState.AddBalance(account, big.NewInt(1000000)) for i := 0; i < size; i++ { tx := transaction(uint64(i), big.NewInt(100000), key) @@ -1430,7 +1402,7 @@ func benchmarkPendingDemotion(b *testing.B, size int) { // Benchmark the speed of pool validation b.ResetTimer() for i := 0; i < b.N; i++ { - pool.demoteUnexecutables(state) + pool.demoteUnexecutables() } } @@ -1446,8 +1418,7 @@ func benchmarkFuturePromotion(b *testing.B, size int) { defer pool.Stop() account, _ := deriveSender(transaction(0, big.NewInt(0), key)) - state, _ := pool.blockChain.State() - state.AddBalance(account, big.NewInt(1000000)) + pool.currentState.AddBalance(account, big.NewInt(1000000)) for i := 0; i < size; i++ { tx := transaction(uint64(1+i), big.NewInt(100000), key) @@ -1456,7 +1427,7 @@ func benchmarkFuturePromotion(b *testing.B, size int) { // Benchmark the speed of pool validation b.ResetTimer() for i := 0; i < b.N; i++ { - pool.promoteExecutables(state, nil) + pool.promoteExecutables(nil) } } @@ -1467,8 +1438,7 @@ func BenchmarkPoolInsert(b *testing.B) { defer pool.Stop() account, _ := deriveSender(transaction(0, big.NewInt(0), key)) - state, _ := pool.blockChain.State() - state.AddBalance(account, big.NewInt(1000000)) + pool.currentState.AddBalance(account, big.NewInt(1000000)) txs := make(types.Transactions, b.N) for i := 0; i < b.N; i++ { @@ -1492,8 +1462,7 @@ func benchmarkPoolBatchInsert(b *testing.B, size int) { defer pool.Stop() account, _ := deriveSender(transaction(0, big.NewInt(0), key)) - state, _ := pool.blockChain.State() - state.AddBalance(account, big.NewInt(1000000)) + pool.currentState.AddBalance(account, big.NewInt(1000000)) batches := make([]types.Transactions, b.N) for i := 0; i < b.N; i++ { diff --git a/core/types/bloom9.go b/core/types/bloom9.go index 60aacc3016..a76b6f33c5 100644 --- a/core/types/bloom9.go +++ b/core/types/bloom9.go @@ -28,10 +28,16 @@ type bytesBacked interface { Bytes() []byte } -const bloomLength = 256 +const ( + // BloomByteLength represents the number of bytes used in a header log bloom. + BloomByteLength = 256 -// Bloom represents a 256 bit bloom filter. -type Bloom [bloomLength]byte + // BloomBitLength represents the number of bits used in a header log bloom. + BloomBitLength = 8 * BloomByteLength +) + +// Bloom represents a 2048 bit bloom filter. +type Bloom [BloomByteLength]byte // BytesToBloom converts a byte slice to a bloom filter. // It panics if b is not of suitable size. @@ -47,7 +53,7 @@ func (b *Bloom) SetBytes(d []byte) { if len(b) < len(d) { panic(fmt.Sprintf("bloom bytes too big %d %d", len(b), len(d))) } - copy(b[bloomLength-len(d):], d) + copy(b[BloomByteLength-len(d):], d) } // Add adds d to the filter. Future calls of Test(d) will return true. diff --git a/core/types/receipt.go b/core/types/receipt.go index 4f3b7357ed..e179fe0cfd 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -79,7 +79,7 @@ func NewReceipt(root []byte, failed bool, cumulativeGasUsed *big.Int) *Receipt { } // EncodeRLP implements rlp.Encoder, and flattens the consensus fields of a receipt -// into an RLP stream. If no post state is present, metropolis fork is assumed. +// into an RLP stream. If no post state is present, byzantium fork is assumed. func (r *Receipt) EncodeRLP(w io.Writer) error { return rlp.Encode(w, &receiptRLP{r.statusEncoding(), r.CumulativeGasUsed, r.Bloom, r.Logs}) } diff --git a/core/types/transaction.go b/core/types/transaction.go index 947fc85d60..7f54860fc0 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -381,28 +381,32 @@ func (s *TxByPrice) Pop() interface{} { // transactions in a profit-maximising sorted order, while supporting removing // entire batches of transactions for non-executable accounts. type TransactionsByPriceAndNonce struct { - txs map[common.Address]Transactions // Per account nonce-sorted list of transactions - heads TxByPrice // Next transaction for each unique account (price heap) + txs map[common.Address]Transactions // Per account nonce-sorted list of transactions + heads TxByPrice // Next transaction for each unique account (price heap) + signer Signer // Signer for the set of transactions } // NewTransactionsByPriceAndNonce creates a transaction set that can retrieve // price sorted transactions in a nonce-honouring way. // // Note, the input map is reowned so the caller should not interact any more with -// if after providng it to the constructor. -func NewTransactionsByPriceAndNonce(txs map[common.Address]Transactions) *TransactionsByPriceAndNonce { +// if after providing it to the constructor. +func NewTransactionsByPriceAndNonce(signer Signer, txs map[common.Address]Transactions) *TransactionsByPriceAndNonce { // Initialize a price based heap with the head transactions heads := make(TxByPrice, 0, len(txs)) - for acc, accTxs := range txs { + for _, accTxs := range txs { heads = append(heads, accTxs[0]) + // Ensure the sender address is from the signer + acc, _ := Sender(signer, accTxs[0]) txs[acc] = accTxs[1:] } heap.Init(&heads) // Assemble and return the transaction set return &TransactionsByPriceAndNonce{ - txs: txs, - heads: heads, + txs: txs, + heads: heads, + signer: signer, } } @@ -416,9 +420,7 @@ func (t *TransactionsByPriceAndNonce) Peek() *Transaction { // Shift replaces the current best head with the next one from the same account. func (t *TransactionsByPriceAndNonce) Shift() { - signer := deriveSigner(t.heads[0].data.V) - // derive signer but don't cache. - acc, _ := Sender(signer, t.heads[0]) // we only sort valid txs so this cannot fail + acc, _ := Sender(t.signer, t.heads[0]) if txs, ok := t.txs[acc]; ok && len(txs) > 0 { t.heads[0], t.txs[acc] = txs[0], txs[1:] heap.Fix(&t.heads, 0) diff --git a/core/types/transaction_test.go b/core/types/transaction_test.go index edbe171e6f..df9d7ffd18 100644 --- a/core/types/transaction_test.go +++ b/core/types/transaction_test.go @@ -143,7 +143,7 @@ func TestTransactionPriceNonceSort(t *testing.T) { } } // Sort the transactions and cross check the nonce ordering - txset := NewTransactionsByPriceAndNonce(groups) + txset := NewTransactionsByPriceAndNonce(signer, groups) txs := Transactions{} for { diff --git a/core/vm/contracts.go b/core/vm/contracts.go index 790d42bbe6..7344b6043c 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -46,9 +46,9 @@ var PrecompiledContractsHomestead = map[common.Address]PrecompiledContract{ common.BytesToAddress([]byte{4}): &dataCopy{}, } -// PrecompiledContractsMetropolis contains the default set of pre-compiled Ethereum -// contracts used in the Metropolis release. -var PrecompiledContractsMetropolis = map[common.Address]PrecompiledContract{ +// PrecompiledContractsByzantium contains the default set of pre-compiled Ethereum +// contracts used in the Byzantium release. +var PrecompiledContractsByzantium = map[common.Address]PrecompiledContract{ common.BytesToAddress([]byte{1}): &ecrecover{}, common.BytesToAddress([]byte{2}): &sha256hash{}, common.BytesToAddress([]byte{3}): &ripemd160hash{}, diff --git a/core/vm/contracts_test.go b/core/vm/contracts_test.go index 880d7324ae..513651835a 100644 --- a/core/vm/contracts_test.go +++ b/core/vm/contracts_test.go @@ -321,7 +321,7 @@ var bn256PairingTests = []precompiledTest{ } func testPrecompiled(addr string, test precompiledTest, t *testing.T) { - p := PrecompiledContractsMetropolis[common.HexToAddress(addr)] + p := PrecompiledContractsByzantium[common.HexToAddress(addr)] in := common.Hex2Bytes(test.input) contract := NewContract(AccountRef(common.HexToAddress("1337")), nil, new(big.Int), p.RequiredGas(in)) @@ -338,7 +338,7 @@ func benchmarkPrecompiled(addr string, test precompiledTest, bench *testing.B) { if test.noBenchmark { return } - p := PrecompiledContractsMetropolis[common.HexToAddress(addr)] + p := PrecompiledContractsByzantium[common.HexToAddress(addr)] in := common.Hex2Bytes(test.input) reqGas := p.RequiredGas(in) contract := NewContract(AccountRef(common.HexToAddress("1337")), diff --git a/core/vm/evm.go b/core/vm/evm.go index caf8b4507d..093c7d4c14 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -41,8 +41,8 @@ type ( func run(evm *EVM, snapshot int, contract *Contract, input []byte) ([]byte, error) { if contract.CodeAddr != nil { precompiles := PrecompiledContractsHomestead - if evm.ChainConfig().IsMetropolis(evm.BlockNumber) { - precompiles = PrecompiledContractsMetropolis + if evm.ChainConfig().IsByzantium(evm.BlockNumber) { + precompiles = PrecompiledContractsByzantium } if p := precompiles[*contract.CodeAddr]; p != nil { return RunPrecompiledContract(p, input, contract) @@ -151,8 +151,8 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas ) if !evm.StateDB.Exist(addr) { precompiles := PrecompiledContractsHomestead - if evm.ChainConfig().IsMetropolis(evm.BlockNumber) { - precompiles = PrecompiledContractsMetropolis + if evm.ChainConfig().IsByzantium(evm.BlockNumber) { + precompiles = PrecompiledContractsByzantium } if precompiles[addr] == nil && evm.ChainConfig().IsEIP158(evm.BlockNumber) && value.Sign() == 0 { return nil, gas, nil @@ -299,9 +299,6 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte // Create creates a new contract using code as deployment code. func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.Int) (ret []byte, contractAddr common.Address, leftOverGas uint64, err error) { - if evm.vmConfig.NoRecursion && evm.depth > 0 { - return nil, common.Address{}, gas, nil - } // Depth check execution. Fail if we're trying to execute above the // limit. @@ -334,9 +331,12 @@ func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.I contract := NewContract(caller, AccountRef(contractAddr), value, gas) contract.SetCallCode(&contractAddr, crypto.Keccak256Hash(code), code) + if evm.vmConfig.NoRecursion && evm.depth > 0 { + return nil, contractAddr, gas, nil + } ret, err = run(evm, snapshot, contract, nil) // check whether the max code size has been exceeded - maxCodeSizeExceeded := len(ret) > params.MaxCodeSize + maxCodeSizeExceeded := evm.ChainConfig().IsEIP158(evm.BlockNumber) && len(ret) > params.MaxCodeSize // if the contract creation ran successfully and no errors were returned // calculate the gas required to store the code. If the code could not // be stored due to not enough gas set an error and let it be handled diff --git a/core/vm/gas_table.go b/core/vm/gas_table.go index 3b0698ce9e..0d8e295a52 100644 --- a/core/vm/gas_table.go +++ b/core/vm/gas_table.go @@ -324,7 +324,7 @@ func gasCall(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, mem eip158 = evm.ChainConfig().IsEIP158(evm.BlockNumber) ) if eip158 { - if evm.StateDB.Empty(address) && transfersValue { + if transfersValue && evm.StateDB.Empty(address) { gas += params.CallNewAccountGas } } else if !evm.StateDB.Exist(address) { diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index d3e24a7a45..b0d796a44b 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -70,8 +70,8 @@ func NewInterpreter(evm *EVM, cfg Config) *Interpreter { // we'll set the default jump table. if !cfg.JumpTable[STOP].valid { switch { - case evm.ChainConfig().IsMetropolis(evm.BlockNumber): - cfg.JumpTable = metropolisInstructionSet + case evm.ChainConfig().IsByzantium(evm.BlockNumber): + cfg.JumpTable = byzantiumInstructionSet case evm.ChainConfig().IsHomestead(evm.BlockNumber): cfg.JumpTable = homesteadInstructionSet default: @@ -88,7 +88,7 @@ func NewInterpreter(evm *EVM, cfg Config) *Interpreter { } func (in *Interpreter) enforceRestrictions(op OpCode, operation operation, stack *Stack) error { - if in.evm.chainRules.IsMetropolis { + if in.evm.chainRules.IsByzantium { if in.readOnly { // If the interpreter is operating in readonly mode, make sure no // state-modifying operation is performed. The 3rd stack item diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go index f0a922912f..9ef192fdf9 100644 --- a/core/vm/jump_table.go +++ b/core/vm/jump_table.go @@ -51,14 +51,14 @@ type operation struct { } var ( - frontierInstructionSet = NewFrontierInstructionSet() - homesteadInstructionSet = NewHomesteadInstructionSet() - metropolisInstructionSet = NewMetropolisInstructionSet() + frontierInstructionSet = NewFrontierInstructionSet() + homesteadInstructionSet = NewHomesteadInstructionSet() + byzantiumInstructionSet = NewByzantiumInstructionSet() ) -// NewMetropolisInstructionSet returns the frontier, homestead and -// metropolis instructions. -func NewMetropolisInstructionSet() [256]operation { +// NewByzantiumInstructionSet returns the frontier, homestead and +// byzantium instructions. +func NewByzantiumInstructionSet() [256]operation { // instructions that can be executed during the homestead phase. instructionSet := NewHomesteadInstructionSet() instructionSet[STATICCALL] = operation{ diff --git a/eth/api.go b/eth/api.go index 9904c6f536..a5b6e7076c 100644 --- a/eth/api.go +++ b/eth/api.go @@ -241,7 +241,7 @@ func (api *PrivateAdminAPI) ExportChain(file string) (bool, error) { func hasAllBlocks(chain *core.BlockChain, bs []*types.Block) bool { for _, b := range bs { - if !chain.HasBlock(b.Hash()) { + if !chain.HasBlock(b.Hash(), b.NumberU64()) { return false } } diff --git a/eth/api_backend.go b/eth/api_backend.go index abf52326b6..91f392f94f 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" @@ -115,10 +116,6 @@ func (b *EthApiBackend) GetEVM(ctx context.Context, msg core.Message, state *sta return vm.NewEVM(context, state, b.eth.chainConfig, vmCfg), vmError, nil } -func (b *EthApiBackend) SubscribeRemovedTxEvent(ch chan<- core.RemovedTransactionEvent) event.Subscription { - return b.eth.BlockChain().SubscribeRemovedTxEvent(ch) -} - func (b *EthApiBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { return b.eth.BlockChain().SubscribeRemovedLogsEvent(ch) } @@ -143,10 +140,6 @@ func (b *EthApiBackend) SendTx(ctx context.Context, signedTx *types.Transaction) return b.eth.txPool.AddLocal(signedTx) } -func (b *EthApiBackend) RemoveTx(txHash common.Hash) { - b.eth.txPool.Remove(txHash) -} - func (b *EthApiBackend) GetPoolTransactions() (types.Transactions, error) { pending, err := b.eth.txPool.Pending() if err != nil { @@ -202,3 +195,14 @@ func (b *EthApiBackend) EventMux() *event.TypeMux { func (b *EthApiBackend) AccountManager() *accounts.Manager { return b.eth.AccountManager() } + +func (b *EthApiBackend) BloomStatus() (uint64, uint64) { + sections, _, _ := b.eth.bloomIndexer.Sections() + return params.BloomBitsBlocks, sections +} + +func (b *EthApiBackend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) { + for i := 0; i < bloomFilterThreads; i++ { + go session.Multiplex(bloomRetrievalBatch, bloomRetrievalWait, b.eth.bloomRequests) + } +} diff --git a/eth/backend.go b/eth/backend.go index 5f4f2097a7..99a1928fec 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/downloader" @@ -57,15 +58,19 @@ type LesServer interface { // Ethereum implements the Ethereum full node service. type Ethereum struct { + config *Config chainConfig *params.ChainConfig + // Channel for shutting down the service shutdownChan chan bool // Channel for shutting down the ethereum stopDbUpgrade func() error // stop chain db sequential key upgrade + // Handlers txPool *core.TxPool blockchain *core.BlockChain protocolManager *ProtocolManager lesServer LesServer + // DB interfaces chainDb ethdb.Database // Block chain database @@ -73,6 +78,9 @@ type Ethereum struct { engine consensus.Engine accountManager *accounts.Manager + bloomRequests chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests + bloomIndexer *core.ChainIndexer // Bloom indexer operating during block imports + ApiBackend *EthApiBackend miner *miner.Miner @@ -98,7 +106,6 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { if !config.SyncMode.IsValid() { return nil, fmt.Errorf("invalid sync mode %d", config.SyncMode) } - chainDb, err := CreateDB(ctx, config, "chaindata") if err != nil { return nil, err @@ -111,6 +118,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { log.Info("Initialised chain configuration", "config", chainConfig) eth := &Ethereum{ + config: config, chainDb: chainDb, chainConfig: chainConfig, eventMux: ctx.EventMux, @@ -121,11 +129,10 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { networkId: config.NetworkId, gasPrice: config.GasPrice, etherbase: config.Etherbase, + bloomRequests: make(chan chan *bloombits.Retrieval), + bloomIndexer: NewBloomIndexer(chainDb, params.BloomBitsBlocks), } - if err := addMipmapBloomBins(chainDb); err != nil { - return nil, err - } log.Info("Initialising Ethereum protocol", "versions", ProtocolVersions, "network", config.NetworkId) if !config.SkipBcVersionCheck { @@ -147,27 +154,16 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { eth.blockchain.SetHead(compat.RewindTo) core.WriteChainConfig(chainDb, genesisHash, chainConfig) } + eth.bloomIndexer.Start(eth.blockchain.CurrentHeader(), eth.blockchain.SubscribeChainEvent) if config.TxPool.Journal != "" { config.TxPool.Journal = ctx.ResolvePath(config.TxPool.Journal) } eth.txPool = core.NewTxPool(config.TxPool, eth.chainConfig, eth.blockchain) - maxPeers := config.MaxPeers - if config.LightServ > 0 { - // if we are running a light server, limit the number of ETH peers so that we reserve some space for incoming LES connections - // temporary solution until the new peer connectivity API is finished - halfPeers := maxPeers / 2 - maxPeers -= config.LightPeers - if maxPeers < halfPeers { - maxPeers = halfPeers - } - } - - if eth.protocolManager, err = NewProtocolManager(eth.chainConfig, config.SyncMode, config.NetworkId, maxPeers, eth.eventMux, eth.txPool, eth.engine, eth.blockchain, chainDb); err != nil { + if eth.protocolManager, err = NewProtocolManager(eth.chainConfig, config.SyncMode, config.NetworkId, eth.eventMux, eth.txPool, eth.engine, eth.blockchain, chainDb); err != nil { return nil, err } - eth.miner = miner.New(eth, eth.chainConfig, eth.EventMux(), eth.engine) eth.miner.SetExtra(makeExtraData(config.ExtraData)) @@ -366,17 +362,29 @@ func (s *Ethereum) Downloader() *downloader.Downloader { return s.protocolManage func (s *Ethereum) Protocols() []p2p.Protocol { if s.lesServer == nil { return s.protocolManager.SubProtocols - } else { - return append(s.protocolManager.SubProtocols, s.lesServer.Protocols()...) } + return append(s.protocolManager.SubProtocols, s.lesServer.Protocols()...) } // Start implements node.Service, starting all internal goroutines needed by the // Ethereum protocol implementation. func (s *Ethereum) Start(srvr *p2p.Server) error { + // Start the bloom bits servicing goroutines + s.startBloomHandlers() + + // Start the RPC service s.netRPCService = ethapi.NewPublicNetAPI(srvr, s.NetVersion()) - s.protocolManager.Start() + // Figure out a max peers count based on the server limits + maxPeers := srvr.MaxPeers + if s.config.LightServ > 0 { + maxPeers -= s.config.LightPeers + if maxPeers < srvr.MaxPeers/2 { + maxPeers = srvr.MaxPeers / 2 + } + } + // Start the networking layer and the light server if requested + s.protocolManager.Start(maxPeers) if s.lesServer != nil { s.lesServer.Start(srvr) } @@ -389,6 +397,7 @@ func (s *Ethereum) Stop() error { if s.stopDbUpgrade != nil { s.stopDbUpgrade() } + s.bloomIndexer.Close() s.blockchain.Stop() s.protocolManager.Stop() if s.lesServer != nil { diff --git a/eth/backend_test.go b/eth/backend_test.go deleted file mode 100644 index 1fd25e95aa..0000000000 --- a/eth/backend_test.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package eth - -import ( - "math/big" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/params" -) - -func TestMipmapUpgrade(t *testing.T) { - db, _ := ethdb.NewMemDatabase() - addr := common.BytesToAddress([]byte("jeff")) - genesis := new(core.Genesis).MustCommit(db) - - chain, receipts := core.GenerateChain(params.TestChainConfig, genesis, db, 10, func(i int, gen *core.BlockGen) { - switch i { - case 1: - receipt := types.NewReceipt(nil, false, new(big.Int)) - receipt.Logs = []*types.Log{{Address: addr}} - gen.AddUncheckedReceipt(receipt) - case 2: - receipt := types.NewReceipt(nil, false, new(big.Int)) - receipt.Logs = []*types.Log{{Address: addr}} - gen.AddUncheckedReceipt(receipt) - } - }) - for i, block := range chain { - core.WriteBlock(db, block) - if err := core.WriteCanonicalHash(db, block.Hash(), block.NumberU64()); err != nil { - t.Fatalf("failed to insert block number: %v", err) - } - if err := core.WriteHeadBlockHash(db, block.Hash()); err != nil { - t.Fatalf("failed to insert block number: %v", err) - } - if err := core.WriteBlockReceipts(db, block.Hash(), block.NumberU64(), receipts[i]); err != nil { - t.Fatal("error writing block receipts:", err) - } - } - - err := addMipmapBloomBins(db) - if err != nil { - t.Fatal(err) - } - - bloom := core.GetMipmapBloom(db, 1, core.MIPMapLevels[0]) - if (bloom == types.Bloom{}) { - t.Error("got empty bloom filter") - } - - data, _ := db.Get([]byte("setting-mipmap-version")) - if len(data) == 0 { - t.Error("setting-mipmap-version not written to database") - } -} diff --git a/eth/bloombits.go b/eth/bloombits.go new file mode 100644 index 0000000000..32f6c7b318 --- /dev/null +++ b/eth/bloombits.go @@ -0,0 +1,142 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package eth + +import ( + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/bitutil" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/bloombits" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/params" +) + +const ( + // bloomServiceThreads is the number of goroutines used globally by an Ethereum + // instance to service bloombits lookups for all running filters. + bloomServiceThreads = 16 + + // bloomFilterThreads is the number of goroutines used locally per filter to + // multiplex requests onto the global servicing goroutines. + bloomFilterThreads = 3 + + // bloomRetrievalBatch is the maximum number of bloom bit retrievals to service + // in a single batch. + bloomRetrievalBatch = 16 + + // bloomRetrievalWait is the maximum time to wait for enough bloom bit requests + // to accumulate request an entire batch (avoiding hysteresis). + bloomRetrievalWait = time.Duration(0) +) + +// startBloomHandlers starts a batch of goroutines to accept bloom bit database +// retrievals from possibly a range of filters and serving the data to satisfy. +func (eth *Ethereum) startBloomHandlers() { + for i := 0; i < bloomServiceThreads; i++ { + go func() { + for { + select { + case <-eth.shutdownChan: + return + + case request := <-eth.bloomRequests: + task := <-request + + task.Bitsets = make([][]byte, len(task.Sections)) + for i, section := range task.Sections { + head := core.GetCanonicalHash(eth.chainDb, (section+1)*params.BloomBitsBlocks-1) + blob, err := bitutil.DecompressBytes(core.GetBloomBits(eth.chainDb, task.Bit, section, head), int(params.BloomBitsBlocks)/8) + if err != nil { + panic(err) + } + task.Bitsets[i] = blob + } + request <- task + } + } + }() + } +} + +const ( + // bloomConfirms is the number of confirmation blocks before a bloom section is + // considered probably final and its rotated bits are calculated. + bloomConfirms = 256 + + // bloomThrottling is the time to wait between processing two consecutive index + // sections. It's useful during chain upgrades to prevent disk overload. + bloomThrottling = 100 * time.Millisecond +) + +// BloomIndexer implements a core.ChainIndexer, building up a rotated bloom bits index +// for the Ethereum header bloom filters, permitting blazing fast filtering. +type BloomIndexer struct { + size uint64 // section size to generate bloombits for + + db ethdb.Database // database instance to write index data and metadata into + gen *bloombits.Generator // generator to rotate the bloom bits crating the bloom index + + section uint64 // Section is the section number being processed currently + head common.Hash // Head is the hash of the last header processed +} + +// NewBloomIndexer returns a chain indexer that generates bloom bits data for the +// canonical chain for fast logs filtering. +func NewBloomIndexer(db ethdb.Database, size uint64) *core.ChainIndexer { + backend := &BloomIndexer{ + db: db, + size: size, + } + table := ethdb.NewTable(db, string(core.BloomBitsIndexPrefix)) + + return core.NewChainIndexer(db, table, backend, size, bloomConfirms, bloomThrottling, "bloombits") +} + +// Reset implements core.ChainIndexerBackend, starting a new bloombits index +// section. +func (b *BloomIndexer) Reset(section uint64) { + gen, err := bloombits.NewGenerator(uint(b.size)) + if err != nil { + panic(err) + } + b.gen, b.section, b.head = gen, section, common.Hash{} +} + +// Process implements core.ChainIndexerBackend, adding a new header's bloom into +// the index. +func (b *BloomIndexer) Process(header *types.Header) { + b.gen.AddBloom(uint(header.Number.Uint64()-b.section*b.size), header.Bloom) + b.head = header.Hash() +} + +// Commit implements core.ChainIndexerBackend, finalizing the bloom section and +// writing it out into the database. +func (b *BloomIndexer) Commit() error { + batch := b.db.NewBatch() + + for i := 0; i < types.BloomBitLength; i++ { + bits, err := b.gen.Bitset(uint(i)) + if err != nil { + return err + } + core.WriteBloomBits(batch, uint(i), b.section, b.head, bitutil.CompressBytes(bits)) + } + return batch.Write() +} diff --git a/eth/config.go b/eth/config.go index 4109cff8b7..7bcfd403ea 100644 --- a/eth/config.go +++ b/eth/config.go @@ -79,7 +79,6 @@ type Config struct { // Light client options LightServ int `toml:",omitempty"` // Maximum percentage of time allowed for serving LES requests LightPeers int `toml:",omitempty"` // Maximum number of LES client peers - MaxPeers int `toml:"-"` // Maximum number of global peers // Database options SkipBcVersionCheck bool `toml:"-"` diff --git a/eth/db_upgrade.go b/eth/db_upgrade.go index 90111b2b38..d41afa17c0 100644 --- a/eth/db_upgrade.go +++ b/eth/db_upgrade.go @@ -19,7 +19,6 @@ package eth import ( "bytes" - "fmt" "time" "github.com/ethereum/go-ethereum/common" @@ -134,46 +133,3 @@ func upgradeDeduplicateData(db ethdb.Database) func() error { return <-errc } } - -func addMipmapBloomBins(db ethdb.Database) (err error) { - const mipmapVersion uint = 2 - - // check if the version is set. We ignore data for now since there's - // only one version so we can easily ignore it for now - var data []byte - data, _ = db.Get([]byte("setting-mipmap-version")) - if len(data) > 0 { - var version uint - if err := rlp.DecodeBytes(data, &version); err == nil && version == mipmapVersion { - return nil - } - } - - defer func() { - if err == nil { - var val []byte - val, err = rlp.EncodeToBytes(mipmapVersion) - if err == nil { - err = db.Put([]byte("setting-mipmap-version"), val) - } - return - } - }() - latestHash := core.GetHeadBlockHash(db) - latestBlock := core.GetBlock(db, latestHash, core.GetBlockNumber(db, latestHash)) - if latestBlock == nil { // clean database - return - } - - tstart := time.Now() - log.Warn("Upgrading db log bloom bins") - for i := uint64(0); i <= latestBlock.NumberU64(); i++ { - hash := core.GetCanonicalHash(db, i) - if (hash == common.Hash{}) { - return fmt.Errorf("chain db corrupted. Could not find block %d.", i) - } - core.WriteMipmapBloom(db, i, core.GetBlockReceipts(db, hash, i)) - } - log.Info("Bloom-bin upgrade completed", "elapsed", common.PrettyDuration(time.Since(tstart))) - return nil -} diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 6ac58140a3..5782d4cf53 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -156,7 +156,7 @@ type Downloader struct { // LightChain encapsulates functions required to synchronise a light chain. type LightChain interface { // HasHeader verifies a header's presence in the local chain. - HasHeader(common.Hash) bool + HasHeader(h common.Hash, number uint64) bool // GetHeaderByHash retrieves a header from the local chain. GetHeaderByHash(common.Hash) *types.Header @@ -666,7 +666,7 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err continue } // Otherwise check if we already know the header or not - if (d.mode == FullSync && d.blockchain.HasBlockAndState(headers[i].Hash())) || (d.mode != FullSync && d.lightchain.HasHeader(headers[i].Hash())) { + if (d.mode == FullSync && d.blockchain.HasBlockAndState(headers[i].Hash())) || (d.mode != FullSync && d.lightchain.HasHeader(headers[i].Hash(), headers[i].Number.Uint64())) { number, hash = headers[i].Number.Uint64(), headers[i].Hash() // If every header is known, even future ones, the peer straight out lied about its head @@ -731,7 +731,7 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err arrived = true // Modify the search interval based on the response - if (d.mode == FullSync && !d.blockchain.HasBlockAndState(headers[0].Hash())) || (d.mode != FullSync && !d.lightchain.HasHeader(headers[0].Hash())) { + if (d.mode == FullSync && !d.blockchain.HasBlockAndState(headers[0].Hash())) || (d.mode != FullSync && !d.lightchain.HasHeader(headers[0].Hash(), headers[0].Number.Uint64())) { end = check break } @@ -1093,6 +1093,10 @@ func (d *Downloader) fetchParts(errCancel error, deliveryCh chan dataPack, deliv throttled = true break } + // Short circuit if there is no more available task. + if pending() == 0 { + break + } // Reserve a chunk of fetches for a peer. A nil can mean either that // no more headers are available, or that the peer is known not to // have them. @@ -1170,7 +1174,7 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error { // If we're already past the pivot point, this could be an attack, thread carefully if rollback[len(rollback)-1].Number.Uint64() > pivot { - // If we didn't ever fail, lock in te pivot header (must! not! change!) + // If we didn't ever fail, lock in the pivot header (must! not! change!) if atomic.LoadUint32(&d.fsPivotFails) == 0 { for _, header := range rollback { if header.Number.Uint64() == pivot { @@ -1256,7 +1260,7 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error { // Collect the yet unknown headers to mark them as uncertain unknown := make([]*types.Header, 0, len(headers)) for _, header := range chunk { - if !d.lightchain.HasHeader(header.Hash()) { + if !d.lightchain.HasHeader(header.Hash(), header.Number.Uint64()) { unknown = append(unknown, header) } } @@ -1392,7 +1396,6 @@ func (d *Downloader) processFastSyncContent(latest *types.Header) error { stateSync.Cancel() if err := d.commitPivotBlock(P); err != nil { return err - } } if err := d.importBlockResults(afterP); err != nil { diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index d66aafe94e..58f6e9a624 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -217,7 +217,7 @@ func (dl *downloadTester) sync(id string, td *big.Int, mode SyncMode) error { } // HasHeader checks if a header is present in the testers canonical chain. -func (dl *downloadTester) HasHeader(hash common.Hash) bool { +func (dl *downloadTester) HasHeader(hash common.Hash, number uint64) bool { return dl.GetHeaderByHash(hash) != nil } diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index d0dc9a8aa6..e638744ea5 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -349,9 +349,10 @@ func (p *peerConnection) Lacks(hash common.Hash) bool { // peerSet represents the collection of active peer participating in the chain // download procedure. type peerSet struct { - peers map[string]*peerConnection - newPeerFeed event.Feed - lock sync.RWMutex + peers map[string]*peerConnection + newPeerFeed event.Feed + peerDropFeed event.Feed + lock sync.RWMutex } // newPeerSet creates a new peer set top track the active download sources. @@ -361,10 +362,16 @@ func newPeerSet() *peerSet { } } +// SubscribeNewPeers subscribes to peer arrival events. func (ps *peerSet) SubscribeNewPeers(ch chan<- *peerConnection) event.Subscription { return ps.newPeerFeed.Subscribe(ch) } +// SubscribePeerDrops subscribes to peer departure events. +func (ps *peerSet) SubscribePeerDrops(ch chan<- *peerConnection) event.Subscription { + return ps.peerDropFeed.Subscribe(ch) +} + // Reset iterates over the current peer set, and resets each of the known peers // to prepare for a next batch of block retrieval. func (ps *peerSet) Reset() { @@ -419,12 +426,15 @@ func (ps *peerSet) Register(p *peerConnection) error { // actions to/from that particular entity. func (ps *peerSet) Unregister(id string) error { ps.lock.Lock() - defer ps.lock.Unlock() - - if _, ok := ps.peers[id]; !ok { + p, ok := ps.peers[id] + if !ok { + defer ps.lock.Unlock() return errNotRegistered } delete(ps.peers, id) + ps.lock.Unlock() + + ps.peerDropFeed.Send(p) return nil } diff --git a/eth/downloader/statesync.go b/eth/downloader/statesync.go index a5ce8c42df..a0b05c9be6 100644 --- a/eth/downloader/statesync.go +++ b/eth/downloader/statesync.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/crypto/sha3" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie" ) @@ -39,6 +40,7 @@ type stateReq struct { timer *time.Timer // Timer to fire when the RTT timeout expires peer *peerConnection // Peer that we're requesting from response [][]byte // Response data of the peer (nil for timeouts) + dropped bool // Flag whether the peer dropped off early } // timedOut returns if this request timed out. @@ -104,6 +106,11 @@ func (d *Downloader) runStateSync(s *stateSync) *stateSync { go s.run() defer s.Cancel() + // Listen for peer departure events to cancel assigned tasks + peerDrop := make(chan *peerConnection, 1024) + peerSub := s.d.peers.SubscribePeerDrops(peerDrop) + defer peerSub.Unsubscribe() + for { // Enable sending of the first buffered element if there is one. var ( @@ -142,6 +149,20 @@ func (d *Downloader) runStateSync(s *stateSync) *stateSync { finished = append(finished, req) delete(active, pack.PeerId()) + // Handle dropped peer connections: + case p := <-peerDrop: + // Skip if no request is currently pending + req := active[p.id] + if req == nil { + continue + } + // Finalize the request and queue up for processing + req.timer.Stop() + req.dropped = true + + finished = append(finished, req) + delete(active, p.id) + // Handle timed-out requests: case req := <-timeout: // If the peer is already requesting something else, ignore the stale timeout. @@ -166,6 +187,9 @@ func (d *Downloader) runStateSync(s *stateSync) *stateSync { log.Warn("Busy peer assigned new state fetch", "peer", old.peer.id) // Make sure the previous one doesn't get siletly lost + old.timer.Stop() + old.dropped = true + finished = append(finished, old) } // Start a timer to notify the sync loop if the peer stalled. @@ -187,10 +211,13 @@ func (d *Downloader) runStateSync(s *stateSync) *stateSync { type stateSync struct { d *Downloader // Downloader instance to access and manage current peerset - sched *state.StateSync // State trie sync scheduler defining the tasks + sched *trie.TrieSync // State trie sync scheduler defining the tasks keccak hash.Hash // Keccak256 hasher to verify deliveries with tasks map[common.Hash]*stateTask // Set of tasks currently queued for retrieval + numUncommitted int + bytesUncommitted int + deliver chan *stateReq // Delivery channel multiplexing peer responses cancel chan struct{} // Channel to signal a termination request cancelOnce sync.Once // Ensures cancel only ever gets called once @@ -252,9 +279,10 @@ func (s *stateSync) loop() error { // Keep assigning new tasks until the sync completes or aborts for s.sched.Pending() > 0 { - if err := s.assignTasks(); err != nil { + if err := s.commit(false); err != nil { return err } + s.assignTasks() // Tasks assigned, wait for something to happen select { case <-newPeer: @@ -264,9 +292,9 @@ func (s *stateSync) loop() error { return errCancelStateFetch case req := <-s.deliver: - // Response or timeout triggered, drop the peer if stalling - log.Trace("Received node data response", "peer", req.peer.id, "count", len(req.response), "timeout", req.timedOut()) - if len(req.items) <= 2 && req.timedOut() { + // Response, disconnect or timeout triggered, drop the peer if stalling + log.Trace("Received node data response", "peer", req.peer.id, "count", len(req.response), "dropped", req.dropped, "timeout", !req.dropped && req.timedOut()) + if len(req.items) <= 2 && !req.dropped && req.timedOut() { // 2 items are the minimum requested, if even that times out, we've no use of // this peer at the moment. log.Warn("Stalling state sync, dropping peer", "peer", req.peer.id) @@ -284,12 +312,28 @@ func (s *stateSync) loop() error { } } } + return s.commit(true) +} + +func (s *stateSync) commit(force bool) error { + if !force && s.bytesUncommitted < ethdb.IdealBatchSize { + return nil + } + start := time.Now() + b := s.d.stateDB.NewBatch() + s.sched.Commit(b) + if err := b.Write(); err != nil { + return fmt.Errorf("DB write error: %v", err) + } + s.updateStats(s.numUncommitted, 0, 0, time.Since(start)) + s.numUncommitted = 0 + s.bytesUncommitted = 0 return nil } // assignTasks attempts to assing new tasks to all idle peers, either from the // batch currently being retried, or fetching new data from the trie sync itself. -func (s *stateSync) assignTasks() error { +func (s *stateSync) assignTasks() { // Iterate over all idle peers and try to assign them state fetches peers, _ := s.d.peers.NodeDataIdlePeers() for _, p := range peers { @@ -301,7 +345,6 @@ func (s *stateSync) assignTasks() error { // If the peer was assigned tasks to fetch, send the network request if len(req.items) > 0 { req.peer.log.Trace("Requesting new batch of data", "type", "state", "count", len(req.items)) - select { case s.d.trackStateReq <- req: req.peer.FetchNodeData(req.items) @@ -309,7 +352,6 @@ func (s *stateSync) assignTasks() error { } } } - return nil } // fillTasks fills the given request object with a maximum of n state download @@ -347,11 +389,11 @@ func (s *stateSync) fillTasks(n int, req *stateReq) { // delivered. func (s *stateSync) process(req *stateReq) (bool, error) { // Collect processing stats and update progress if valid data was received - processed, written, duplicate, unexpected := 0, 0, 0, 0 + duplicate, unexpected := 0, 0 defer func(start time.Time) { - if processed+written+duplicate+unexpected > 0 { - s.updateStats(processed, written, duplicate, unexpected, time.Since(start)) + if duplicate > 0 || unexpected > 0 { + s.updateStats(0, duplicate, unexpected, time.Since(start)) } }(time.Now()) @@ -362,7 +404,9 @@ func (s *stateSync) process(req *stateReq) (bool, error) { prog, hash, err := s.processNodeData(blob) switch err { case nil: - processed++ + s.numUncommitted++ + s.bytesUncommitted += len(blob) + progress = progress || prog case trie.ErrNotRequested: unexpected++ case trie.ErrAlreadyProcessed: @@ -370,38 +414,20 @@ func (s *stateSync) process(req *stateReq) (bool, error) { default: return stale, fmt.Errorf("invalid state node %s: %v", hash.TerminalString(), err) } - if prog { - progress = true - } // If the node delivered a requested item, mark the delivery non-stale if _, ok := req.tasks[hash]; ok { delete(req.tasks, hash) stale = false } } - // If some data managed to hit the database, flush and reset failure counters - if progress { - // Flush any accumulated data out to disk - batch := s.d.stateDB.NewBatch() - - count, err := s.sched.Commit(batch) - if err != nil { - return stale, err - } - if err := batch.Write(); err != nil { - return stale, err - } - written = count - - // If we're inside the critical section, reset fail counter since we progressed - if atomic.LoadUint32(&s.d.fsPivotFails) > 1 { - log.Trace("Fast-sync progressed, resetting fail counter", "previous", atomic.LoadUint32(&s.d.fsPivotFails)) - atomic.StoreUint32(&s.d.fsPivotFails, 1) // Don't ever reset to 0, as that will unlock the pivot block - } + // If we're inside the critical section, reset fail counter since we progressed. + if progress && atomic.LoadUint32(&s.d.fsPivotFails) > 1 { + log.Trace("Fast-sync progressed, resetting fail counter", "previous", atomic.LoadUint32(&s.d.fsPivotFails)) + atomic.StoreUint32(&s.d.fsPivotFails, 1) // Don't ever reset to 0, as that will unlock the pivot block } + // Put unfulfilled tasks back into the retry queue npeers := s.d.peers.Len() - for hash, task := range req.tasks { // If the node did deliver something, missing items may be due to a protocol // limit or a previous timeout + delayed delivery. Both cases should permit @@ -425,25 +451,25 @@ func (s *stateSync) process(req *stateReq) (bool, error) { // error occurred. func (s *stateSync) processNodeData(blob []byte) (bool, common.Hash, error) { res := trie.SyncResult{Data: blob} - s.keccak.Reset() s.keccak.Write(blob) s.keccak.Sum(res.Hash[:0]) - committed, _, err := s.sched.Process([]trie.SyncResult{res}) return committed, res.Hash, err } // updateStats bumps the various state sync progress counters and displays a log // message for the user to see. -func (s *stateSync) updateStats(processed, written, duplicate, unexpected int, duration time.Duration) { +func (s *stateSync) updateStats(written, duplicate, unexpected int, duration time.Duration) { s.d.syncStatsLock.Lock() defer s.d.syncStatsLock.Unlock() s.d.syncStatsState.pending = uint64(s.sched.Pending()) - s.d.syncStatsState.processed += uint64(processed) + s.d.syncStatsState.processed += uint64(written) s.d.syncStatsState.duplicate += uint64(duplicate) s.d.syncStatsState.unexpected += uint64(unexpected) - log.Info("Imported new state entries", "count", processed, "flushed", written, "elapsed", common.PrettyDuration(duration), "processed", s.d.syncStatsState.processed, "pending", s.d.syncStatsState.pending, "retry", len(s.tasks), "duplicate", s.d.syncStatsState.duplicate, "unexpected", s.d.syncStatsState.unexpected) + if written > 0 || duplicate > 0 || unexpected > 0 { + log.Info("Imported new state entries", "count", written, "elapsed", common.PrettyDuration(duration), "processed", s.d.syncStatsState.processed, "pending", s.d.syncStatsState.pending, "retry", len(s.tasks), "duplicate", s.d.syncStatsState.duplicate, "unexpected", s.d.syncStatsState.unexpected) + } } diff --git a/eth/filters/api.go b/eth/filters/api.go index fff58a268e..6e1d48adb6 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -52,8 +52,8 @@ type filter struct { // information related to the Ethereum protocol such als blocks, transactions and logs. type PublicFilterAPI struct { backend Backend - useMipMap bool mux *event.TypeMux + quit chan struct{} chainDb ethdb.Database events *EventSystem filtersMu sync.Mutex @@ -63,14 +63,12 @@ type PublicFilterAPI struct { // NewPublicFilterAPI returns a new PublicFilterAPI instance. func NewPublicFilterAPI(backend Backend, lightMode bool) *PublicFilterAPI { api := &PublicFilterAPI{ - backend: backend, - useMipMap: !lightMode, - mux: backend.EventMux(), - chainDb: backend.ChainDb(), - events: NewEventSystem(backend.EventMux(), backend, lightMode), - filters: make(map[rpc.ID]*filter), + backend: backend, + mux: backend.EventMux(), + chainDb: backend.ChainDb(), + events: NewEventSystem(backend.EventMux(), backend, lightMode), + filters: make(map[rpc.ID]*filter), } - go api.timeoutLoop() return api @@ -325,20 +323,20 @@ func (api *PublicFilterAPI) NewFilter(crit FilterCriteria) (rpc.ID, error) { // // https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getlogs func (api *PublicFilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([]*types.Log, error) { + // Convert the RPC block numbers into internal representations if crit.FromBlock == nil { crit.FromBlock = big.NewInt(rpc.LatestBlockNumber.Int64()) } if crit.ToBlock == nil { crit.ToBlock = big.NewInt(rpc.LatestBlockNumber.Int64()) } + // Create and run the filter to get all the logs + filter := New(api.backend, crit.FromBlock.Int64(), crit.ToBlock.Int64(), crit.Addresses, crit.Topics) - filter := New(api.backend, api.useMipMap) - filter.SetBeginBlock(crit.FromBlock.Int64()) - filter.SetEndBlock(crit.ToBlock.Int64()) - filter.SetAddresses(crit.Addresses) - filter.SetTopics(crit.Topics) - - logs, err := filter.Find(ctx) + logs, err := filter.Logs(ctx) + if err != nil { + return nil, err + } return returnLogs(logs), err } @@ -372,21 +370,18 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*ty return nil, fmt.Errorf("filter not found") } - filter := New(api.backend, api.useMipMap) + begin := rpc.LatestBlockNumber.Int64() if f.crit.FromBlock != nil { - filter.SetBeginBlock(f.crit.FromBlock.Int64()) - } else { - filter.SetBeginBlock(rpc.LatestBlockNumber.Int64()) + begin = f.crit.FromBlock.Int64() } + end := rpc.LatestBlockNumber.Int64() if f.crit.ToBlock != nil { - filter.SetEndBlock(f.crit.ToBlock.Int64()) - } else { - filter.SetEndBlock(rpc.LatestBlockNumber.Int64()) + end = f.crit.ToBlock.Int64() } - filter.SetAddresses(f.crit.Addresses) - filter.SetTopics(f.crit.Topics) + // Create and run the filter to get all the logs + filter := New(api.backend, begin, end, f.crit.Addresses, f.crit.Topics) - logs, err := filter.Find(ctx) + logs, err := filter.Logs(ctx) if err != nil { return nil, err } @@ -394,7 +389,7 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*ty } // GetFilterChanges returns the logs for the filter with the given id since -// last time is was called. This can be used for polling. +// last time it was called. This can be used for polling. // // For pending transaction and block filters the result is []common.Hash. // (pending)Log filters return []Log. diff --git a/eth/filters/bench_test.go b/eth/filters/bench_test.go new file mode 100644 index 0000000000..abbf4593e9 --- /dev/null +++ b/eth/filters/bench_test.go @@ -0,0 +1,201 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package filters + +import ( + "bytes" + "context" + "fmt" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/bitutil" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/bloombits" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/node" +) + +func BenchmarkBloomBits512(b *testing.B) { + benchmarkBloomBits(b, 512) +} + +func BenchmarkBloomBits1k(b *testing.B) { + benchmarkBloomBits(b, 1024) +} + +func BenchmarkBloomBits2k(b *testing.B) { + benchmarkBloomBits(b, 2048) +} + +func BenchmarkBloomBits4k(b *testing.B) { + benchmarkBloomBits(b, 4096) +} + +func BenchmarkBloomBits8k(b *testing.B) { + benchmarkBloomBits(b, 8192) +} + +func BenchmarkBloomBits16k(b *testing.B) { + benchmarkBloomBits(b, 16384) +} + +func BenchmarkBloomBits32k(b *testing.B) { + benchmarkBloomBits(b, 32768) +} + +const benchFilterCnt = 2000 + +func benchmarkBloomBits(b *testing.B, sectionSize uint64) { + benchDataDir := node.DefaultDataDir() + "/geth/chaindata" + fmt.Println("Running bloombits benchmark section size:", sectionSize) + + db, err := ethdb.NewLDBDatabase(benchDataDir, 128, 1024) + if err != nil { + b.Fatalf("error opening database at %v: %v", benchDataDir, err) + } + head := core.GetHeadBlockHash(db) + if head == (common.Hash{}) { + b.Fatalf("chain data not found at %v", benchDataDir) + } + + clearBloomBits(db) + fmt.Println("Generating bloombits data...") + headNum := core.GetBlockNumber(db, head) + if headNum < sectionSize+512 { + b.Fatalf("not enough blocks for running a benchmark") + } + + start := time.Now() + cnt := (headNum - 512) / sectionSize + var dataSize, compSize uint64 + for sectionIdx := uint64(0); sectionIdx < cnt; sectionIdx++ { + bc, err := bloombits.NewGenerator(uint(sectionSize)) + if err != nil { + b.Fatalf("failed to create generator: %v", err) + } + var header *types.Header + for i := sectionIdx * sectionSize; i < (sectionIdx+1)*sectionSize; i++ { + hash := core.GetCanonicalHash(db, i) + header = core.GetHeader(db, hash, i) + if header == nil { + b.Fatalf("Error creating bloomBits data") + } + bc.AddBloom(uint(i-sectionIdx*sectionSize), header.Bloom) + } + sectionHead := core.GetCanonicalHash(db, (sectionIdx+1)*sectionSize-1) + for i := 0; i < types.BloomBitLength; i++ { + data, err := bc.Bitset(uint(i)) + if err != nil { + b.Fatalf("failed to retrieve bitset: %v", err) + } + comp := bitutil.CompressBytes(data) + dataSize += uint64(len(data)) + compSize += uint64(len(comp)) + core.WriteBloomBits(db, uint(i), sectionIdx, sectionHead, comp) + } + //if sectionIdx%50 == 0 { + // fmt.Println(" section", sectionIdx, "/", cnt) + //} + } + + d := time.Since(start) + fmt.Println("Finished generating bloombits data") + fmt.Println(" ", d, "total ", d/time.Duration(cnt*sectionSize), "per block") + fmt.Println(" data size:", dataSize, " compressed size:", compSize, " compression ratio:", float64(compSize)/float64(dataSize)) + + fmt.Println("Running filter benchmarks...") + start = time.Now() + mux := new(event.TypeMux) + var backend *testBackend + + for i := 0; i < benchFilterCnt; i++ { + if i%20 == 0 { + db.Close() + db, _ = ethdb.NewLDBDatabase(benchDataDir, 128, 1024) + backend = &testBackend{mux, db, cnt, new(event.Feed), new(event.Feed), new(event.Feed), new(event.Feed)} + } + var addr common.Address + addr[0] = byte(i) + addr[1] = byte(i / 256) + filter := New(backend, 0, int64(cnt*sectionSize-1), []common.Address{addr}, nil) + if _, err := filter.Logs(context.Background()); err != nil { + b.Error("filter.Find error:", err) + } + } + d = time.Since(start) + fmt.Println("Finished running filter benchmarks") + fmt.Println(" ", d, "total ", d/time.Duration(benchFilterCnt), "per address", d*time.Duration(1000000)/time.Duration(benchFilterCnt*cnt*sectionSize), "per million blocks") + db.Close() +} + +func forEachKey(db ethdb.Database, startPrefix, endPrefix []byte, fn func(key []byte)) { + it := db.(*ethdb.LDBDatabase).NewIterator() + it.Seek(startPrefix) + for it.Valid() { + key := it.Key() + cmpLen := len(key) + if len(endPrefix) < cmpLen { + cmpLen = len(endPrefix) + } + if bytes.Compare(key[:cmpLen], endPrefix) == 1 { + break + } + fn(common.CopyBytes(key)) + it.Next() + } + it.Release() +} + +var bloomBitsPrefix = []byte("bloomBits-") + +func clearBloomBits(db ethdb.Database) { + fmt.Println("Clearing bloombits data...") + forEachKey(db, bloomBitsPrefix, bloomBitsPrefix, func(key []byte) { + db.Delete(key) + }) +} + +func BenchmarkNoBloomBits(b *testing.B) { + benchDataDir := node.DefaultDataDir() + "/geth/chaindata" + fmt.Println("Running benchmark without bloombits") + db, err := ethdb.NewLDBDatabase(benchDataDir, 128, 1024) + if err != nil { + b.Fatalf("error opening database at %v: %v", benchDataDir, err) + } + head := core.GetHeadBlockHash(db) + if head == (common.Hash{}) { + b.Fatalf("chain data not found at %v", benchDataDir) + } + headNum := core.GetBlockNumber(db, head) + + clearBloomBits(db) + + fmt.Println("Running filter benchmarks...") + start := time.Now() + mux := new(event.TypeMux) + backend := &testBackend{mux, db, 0, new(event.Feed), new(event.Feed), new(event.Feed), new(event.Feed)} + filter := New(backend, 0, int64(headNum), []common.Address{common.Address{}}, nil) + filter.Logs(context.Background()) + d := time.Since(start) + fmt.Println("Finished running filter benchmarks") + fmt.Println(" ", d, "total ", d*time.Duration(1000000)/time.Duration(headNum+1), "per million blocks") + db.Close() +} diff --git a/eth/filters/filter.go b/eth/filters/filter.go index f848bc6af9..4f6c30058e 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -18,11 +18,12 @@ package filters import ( "context" - "math" "math/big" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -34,167 +35,179 @@ type Backend interface { EventMux() *event.TypeMux HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) + SubscribeTxPreEvent(chan<- core.TxPreEvent) event.Subscription SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription + + BloomStatus() (uint64, uint64) + ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) } // Filter can be used to retrieve and filter logs. type Filter struct { - backend Backend - useMipMap bool + backend Backend db ethdb.Database begin, end int64 addresses []common.Address topics [][]common.Hash + + matcher *bloombits.Matcher } // New creates a new filter which uses a bloom filter on blocks to figure out whether // a particular block is interesting or not. -// MipMaps allow past blocks to be searched much more efficiently, but are not available -// to light clients. -func New(backend Backend, useMipMap bool) *Filter { +func New(backend Backend, begin, end int64, addresses []common.Address, topics [][]common.Hash) *Filter { + // Flatten the address and topic filter clauses into a single filter system + var filters [][][]byte + if len(addresses) > 0 { + filter := make([][]byte, len(addresses)) + for i, address := range addresses { + filter[i] = address.Bytes() + } + filters = append(filters, filter) + } + for _, topicList := range topics { + filter := make([][]byte, len(topicList)) + for i, topic := range topicList { + filter[i] = topic.Bytes() + } + filters = append(filters, filter) + } + // Assemble and return the filter + size, _ := backend.BloomStatus() + return &Filter{ backend: backend, - useMipMap: useMipMap, + begin: begin, + end: end, + addresses: addresses, + topics: topics, db: backend.ChainDb(), + matcher: bloombits.NewMatcher(size, filters), } } -// SetBeginBlock sets the earliest block for filtering. -// -1 = latest block (i.e., the current block) -// hash = particular hash from-to -func (f *Filter) SetBeginBlock(begin int64) { - f.begin = begin -} - -// SetEndBlock sets the latest block for filtering. -func (f *Filter) SetEndBlock(end int64) { - f.end = end -} - -// SetAddresses matches only logs that are generated from addresses that are included -// in the given addresses. -func (f *Filter) SetAddresses(addr []common.Address) { - f.addresses = addr -} - -// SetTopics matches only logs that have topics matching the given topics. -func (f *Filter) SetTopics(topics [][]common.Hash) { - f.topics = topics -} - -// FindOnce searches the blockchain for matching log entries, returning -// all matching entries from the first block that contains matches, -// updating the start point of the filter accordingly. If no results are -// found, a nil slice is returned. -func (f *Filter) FindOnce(ctx context.Context) ([]*types.Log, error) { - head, _ := f.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber) - if head == nil { +// Logs searches the blockchain for matching log entries, returning all from the +// first block that contains matches, updating the start of the filter accordingly. +func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { + // Figure out the limits of the filter range + header, _ := f.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber) + if header == nil { return nil, nil } - headBlockNumber := head.Number.Uint64() + head := header.Number.Uint64() - var beginBlockNo uint64 = uint64(f.begin) if f.begin == -1 { - beginBlockNo = headBlockNumber + f.begin = int64(head) } - var endBlockNo uint64 = uint64(f.end) + end := uint64(f.end) if f.end == -1 { - endBlockNo = headBlockNumber + end = head } - - // if no addresses are present we can't make use of fast search which - // uses the mipmap bloom filters to check for fast inclusion and uses - // higher range probability in order to ensure at least a false positive - if !f.useMipMap || len(f.addresses) == 0 { - logs, blockNumber, err := f.getLogs(ctx, beginBlockNo, endBlockNo) - f.begin = int64(blockNumber + 1) - return logs, err + // Gather all indexed logs, and finish with non indexed ones + var ( + logs []*types.Log + err error + ) + size, sections := f.backend.BloomStatus() + if indexed := sections * size; indexed > uint64(f.begin) { + if indexed > end { + logs, err = f.indexedLogs(ctx, end) + } else { + logs, err = f.indexedLogs(ctx, indexed-1) + } + if err != nil { + return logs, err + } } + rest, err := f.unindexedLogs(ctx, end) + logs = append(logs, rest...) + return logs, err +} - logs, blockNumber := f.mipFind(beginBlockNo, endBlockNo, 0) - f.begin = int64(blockNumber + 1) +// indexedLogs returns the logs matching the filter criteria based on the bloom +// bits indexed available locally or via the network. +func (f *Filter) indexedLogs(ctx context.Context, end uint64) ([]*types.Log, error) { + // Create a matcher session and request servicing from the backend + matches := make(chan uint64, 64) + + session, err := f.matcher.Start(uint64(f.begin), end, matches) + if err != nil { + return nil, err + } + defer session.Close(time.Second) + + f.backend.ServiceFilter(ctx, session) + + // Iterate over the matches until exhausted or context closed + var logs []*types.Log + + for { + select { + case number, ok := <-matches: + // Abort if all matches have been fulfilled + if !ok { + f.begin = int64(end) + 1 + return logs, nil + } + // Retrieve the suggested block and pull any truly matching logs + header, err := f.backend.HeaderByNumber(ctx, rpc.BlockNumber(number)) + if header == nil || err != nil { + return logs, err + } + found, err := f.checkMatches(ctx, header) + if err != nil { + return logs, err + } + logs = append(logs, found...) + + case <-ctx.Done(): + return logs, ctx.Err() + } + } +} + +// indexedLogs returns the logs matching the filter criteria based on raw block +// iteration and bloom matching. +func (f *Filter) unindexedLogs(ctx context.Context, end uint64) ([]*types.Log, error) { + var logs []*types.Log + + for ; f.begin <= int64(end); f.begin++ { + header, err := f.backend.HeaderByNumber(ctx, rpc.BlockNumber(f.begin)) + if header == nil || err != nil { + return logs, err + } + if bloomFilter(header.Bloom, f.addresses, f.topics) { + found, err := f.checkMatches(ctx, header) + if err != nil { + return logs, err + } + logs = append(logs, found...) + } + } return logs, nil } -// Run filters logs with the current parameters set -func (f *Filter) Find(ctx context.Context) (logs []*types.Log, err error) { - for { - newLogs, err := f.FindOnce(ctx) - if len(newLogs) == 0 || err != nil { - return logs, err - } - logs = append(logs, newLogs...) +// checkMatches checks if the receipts belonging to the given header contain any log events that +// match the filter criteria. This function is called when the bloom filter signals a potential match. +func (f *Filter) checkMatches(ctx context.Context, header *types.Header) (logs []*types.Log, err error) { + // Get the logs of the block + receipts, err := f.backend.GetReceipts(ctx, header.Hash()) + if err != nil { + return nil, err } -} - -func (f *Filter) mipFind(start, end uint64, depth int) (logs []*types.Log, blockNumber uint64) { - level := core.MIPMapLevels[depth] - // normalise numerator so we can work in level specific batches and - // work with the proper range checks - for num := start / level * level; num <= end; num += level { - // find addresses in bloom filters - bloom := core.GetMipmapBloom(f.db, num, level) - // Don't bother checking the first time through the loop - we're probably picking - // up where a previous run left off. - first := true - for _, addr := range f.addresses { - if first || bloom.TestBytes(addr[:]) { - first = false - // range check normalised values and make sure that - // we're resolving the correct range instead of the - // normalised values. - start := uint64(math.Max(float64(num), float64(start))) - end := uint64(math.Min(float64(num+level-1), float64(end))) - if depth+1 == len(core.MIPMapLevels) { - l, blockNumber, _ := f.getLogs(context.Background(), start, end) - if len(l) > 0 { - return l, blockNumber - } - } else { - l, blockNumber := f.mipFind(start, end, depth+1) - if len(l) > 0 { - return l, blockNumber - } - } - } - } + var unfiltered []*types.Log + for _, receipt := range receipts { + unfiltered = append(unfiltered, ([]*types.Log)(receipt.Logs)...) } - - return nil, end -} - -func (f *Filter) getLogs(ctx context.Context, start, end uint64) (logs []*types.Log, blockNumber uint64, err error) { - for i := start; i <= end; i++ { - blockNumber := rpc.BlockNumber(i) - header, err := f.backend.HeaderByNumber(ctx, blockNumber) - if header == nil || err != nil { - return logs, end, err - } - - // Use bloom filtering to see if this block is interesting given the - // current parameters - if f.bloomFilter(header.Bloom) { - // Get the logs of the block - receipts, err := f.backend.GetReceipts(ctx, header.Hash()) - if err != nil { - return nil, end, err - } - var unfiltered []*types.Log - for _, receipt := range receipts { - unfiltered = append(unfiltered, ([]*types.Log)(receipt.Logs)...) - } - logs = filterLogs(unfiltered, nil, nil, f.addresses, f.topics) - if len(logs) > 0 { - return logs, uint64(blockNumber), nil - } - } + logs = filterLogs(unfiltered, nil, nil, f.addresses, f.topics) + if len(logs) > 0 { + return logs, nil } - - return logs, end, nil + return nil, nil } func includes(addresses []common.Address, a common.Address) bool { @@ -251,10 +264,6 @@ Logs: return ret } -func (f *Filter) bloomFilter(bloom types.Bloom) bool { - return bloomFilter(bloom, f.addresses, f.topics) -} - func bloomFilter(bloom types.Bloom, addresses []common.Address, topics [][]common.Hash) bool { if len(addresses) > 0 { var included bool diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index fcc888b8cf..664ce07a5f 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -20,12 +20,14 @@ import ( "context" "fmt" "math/big" + "math/rand" "reflect" "testing" "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -36,6 +38,7 @@ import ( type testBackend struct { mux *event.TypeMux db ethdb.Database + sections uint64 txFeed *event.Feed rmLogsFeed *event.Feed logsFeed *event.Feed @@ -84,6 +87,37 @@ func (b *testBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subsc return b.chainFeed.Subscribe(ch) } +func (b *testBackend) BloomStatus() (uint64, uint64) { + return params.BloomBitsBlocks, b.sections +} + +func (b *testBackend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) { + requests := make(chan chan *bloombits.Retrieval) + + go session.Multiplex(16, 0, requests) + go func() { + for { + // Wait for a service request or a shutdown + select { + case <-ctx.Done(): + return + + case request := <-requests: + task := <-request + + task.Bitsets = make([][]byte, len(task.Sections)) + for i, section := range task.Sections { + if rand.Int()%4 != 0 { // Handle occasional missing deliveries + head := core.GetCanonicalHash(b.db, (section+1)*params.BloomBitsBlocks-1) + task.Bitsets[i] = core.GetBloomBits(b.db, task.Bit, section, head) + } + } + request <- task + } + } + }() +} + // TestBlockSubscription tests if a block subscription returns block hashes for posted chain events. // It creates multiple subscriptions: // - one at the start and should receive all posted chain events and a second (blockHashes) @@ -99,7 +133,7 @@ func TestBlockSubscription(t *testing.T) { rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) chainFeed = new(event.Feed) - backend = &testBackend{mux, db, txFeed, rmLogsFeed, logsFeed, chainFeed} + backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} api = NewPublicFilterAPI(backend, false) genesis = new(core.Genesis).MustCommit(db) chain, _ = core.GenerateChain(params.TestChainConfig, genesis, db, 10, func(i int, gen *core.BlockGen) {}) @@ -156,7 +190,7 @@ func TestPendingTxFilter(t *testing.T) { rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) chainFeed = new(event.Feed) - backend = &testBackend{mux, db, txFeed, rmLogsFeed, logsFeed, chainFeed} + backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} api = NewPublicFilterAPI(backend, false) transactions = []*types.Transaction{ @@ -219,7 +253,7 @@ func TestLogFilterCreation(t *testing.T) { rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) chainFeed = new(event.Feed) - backend = &testBackend{mux, db, txFeed, rmLogsFeed, logsFeed, chainFeed} + backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} api = NewPublicFilterAPI(backend, false) testCases = []struct { @@ -268,7 +302,7 @@ func TestInvalidLogFilterCreation(t *testing.T) { rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) chainFeed = new(event.Feed) - backend = &testBackend{mux, db, txFeed, rmLogsFeed, logsFeed, chainFeed} + backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} api = NewPublicFilterAPI(backend, false) ) @@ -298,7 +332,7 @@ func TestLogFilter(t *testing.T) { rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) chainFeed = new(event.Feed) - backend = &testBackend{mux, db, txFeed, rmLogsFeed, logsFeed, chainFeed} + backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} api = NewPublicFilterAPI(backend, false) firstAddr = common.HexToAddress("0x1111111111111111111111111111111111111111") @@ -415,7 +449,7 @@ func TestPendingLogsSubscription(t *testing.T) { rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) chainFeed = new(event.Feed) - backend = &testBackend{mux, db, txFeed, rmLogsFeed, logsFeed, chainFeed} + backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} api = NewPublicFilterAPI(backend, false) firstAddr = common.HexToAddress("0x1111111111111111111111111111111111111111") diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index cf508a218d..11235e95a7 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -41,8 +41,8 @@ func makeReceipt(addr common.Address) *types.Receipt { return receipt } -func BenchmarkMipmaps(b *testing.B) { - dir, err := ioutil.TempDir("", "mipmap") +func BenchmarkFilters(b *testing.B) { + dir, err := ioutil.TempDir("", "filtertest") if err != nil { b.Fatal(err) } @@ -55,7 +55,7 @@ func BenchmarkMipmaps(b *testing.B) { rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) chainFeed = new(event.Feed) - backend = &testBackend{mux, db, txFeed, rmLogsFeed, logsFeed, chainFeed} + backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) addr2 = common.BytesToAddress([]byte("jeff")) @@ -66,27 +66,21 @@ func BenchmarkMipmaps(b *testing.B) { genesis := core.GenesisBlockForTesting(db, addr1, big.NewInt(1000000)) chain, receipts := core.GenerateChain(params.TestChainConfig, genesis, db, 100010, func(i int, gen *core.BlockGen) { - var receipts types.Receipts switch i { case 2403: receipt := makeReceipt(addr1) - receipts = types.Receipts{receipt} gen.AddUncheckedReceipt(receipt) case 1034: receipt := makeReceipt(addr2) - receipts = types.Receipts{receipt} gen.AddUncheckedReceipt(receipt) case 34: receipt := makeReceipt(addr3) - receipts = types.Receipts{receipt} gen.AddUncheckedReceipt(receipt) case 99999: receipt := makeReceipt(addr4) - receipts = types.Receipts{receipt} gen.AddUncheckedReceipt(receipt) } - core.WriteMipmapBloom(db, uint64(i+1), receipts) }) for i, block := range chain { core.WriteBlock(db, block) @@ -102,13 +96,10 @@ func BenchmarkMipmaps(b *testing.B) { } b.ResetTimer() - filter := New(backend, true) - filter.SetAddresses([]common.Address{addr1, addr2, addr3, addr4}) - filter.SetBeginBlock(0) - filter.SetEndBlock(-1) + filter := New(backend, 0, -1, []common.Address{addr1, addr2, addr3, addr4}, nil) for i := 0; i < b.N; i++ { - logs, _ := filter.Find(context.Background()) + logs, _ := filter.Logs(context.Background()) if len(logs) != 4 { b.Fatal("expected 4 logs, got", len(logs)) } @@ -116,7 +107,7 @@ func BenchmarkMipmaps(b *testing.B) { } func TestFilters(t *testing.T) { - dir, err := ioutil.TempDir("", "mipmap") + dir, err := ioutil.TempDir("", "filtertest") if err != nil { t.Fatal(err) } @@ -129,7 +120,7 @@ func TestFilters(t *testing.T) { rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) chainFeed = new(event.Feed) - backend = &testBackend{mux, db, txFeed, rmLogsFeed, logsFeed, chainFeed} + backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr = crypto.PubkeyToAddress(key1.PublicKey) @@ -142,7 +133,6 @@ func TestFilters(t *testing.T) { genesis := core.GenesisBlockForTesting(db, addr, big.NewInt(1000000)) chain, receipts := core.GenerateChain(params.TestChainConfig, genesis, db, 1000, func(i int, gen *core.BlockGen) { - var receipts types.Receipts switch i { case 1: receipt := types.NewReceipt(nil, false, new(big.Int)) @@ -153,7 +143,6 @@ func TestFilters(t *testing.T) { }, } gen.AddUncheckedReceipt(receipt) - receipts = types.Receipts{receipt} case 2: receipt := types.NewReceipt(nil, false, new(big.Int)) receipt.Logs = []*types.Log{ @@ -163,7 +152,6 @@ func TestFilters(t *testing.T) { }, } gen.AddUncheckedReceipt(receipt) - receipts = types.Receipts{receipt} case 998: receipt := types.NewReceipt(nil, false, new(big.Int)) receipt.Logs = []*types.Log{ @@ -173,7 +161,6 @@ func TestFilters(t *testing.T) { }, } gen.AddUncheckedReceipt(receipt) - receipts = types.Receipts{receipt} case 999: receipt := types.NewReceipt(nil, false, new(big.Int)) receipt.Logs = []*types.Log{ @@ -183,12 +170,7 @@ func TestFilters(t *testing.T) { }, } gen.AddUncheckedReceipt(receipt) - receipts = types.Receipts{receipt} } - // i is used as block number for the writes but since the i - // starts at 0 and block 0 (genesis) is already present increment - // by one - core.WriteMipmapBloom(db, uint64(i+1), receipts) }) for i, block := range chain { core.WriteBlock(db, block) @@ -203,23 +185,15 @@ func TestFilters(t *testing.T) { } } - filter := New(backend, true) - filter.SetAddresses([]common.Address{addr}) - filter.SetTopics([][]common.Hash{{hash1, hash2, hash3, hash4}}) - filter.SetBeginBlock(0) - filter.SetEndBlock(-1) + filter := New(backend, 0, -1, []common.Address{addr}, [][]common.Hash{{hash1, hash2, hash3, hash4}}) - logs, _ := filter.Find(context.Background()) + logs, _ := filter.Logs(context.Background()) if len(logs) != 4 { t.Error("expected 4 log, got", len(logs)) } - filter = New(backend, true) - filter.SetAddresses([]common.Address{addr}) - filter.SetTopics([][]common.Hash{{hash3}}) - filter.SetBeginBlock(900) - filter.SetEndBlock(999) - logs, _ = filter.Find(context.Background()) + filter = New(backend, 900, 999, []common.Address{addr}, [][]common.Hash{{hash3}}) + logs, _ = filter.Logs(context.Background()) if len(logs) != 1 { t.Error("expected 1 log, got", len(logs)) } @@ -227,12 +201,8 @@ func TestFilters(t *testing.T) { t.Errorf("expected log[0].Topics[0] to be %x, got %x", hash3, logs[0].Topics[0]) } - filter = New(backend, true) - filter.SetAddresses([]common.Address{addr}) - filter.SetTopics([][]common.Hash{{hash3}}) - filter.SetBeginBlock(990) - filter.SetEndBlock(-1) - logs, _ = filter.Find(context.Background()) + filter = New(backend, 990, -1, []common.Address{addr}, [][]common.Hash{{hash3}}) + logs, _ = filter.Logs(context.Background()) if len(logs) != 1 { t.Error("expected 1 log, got", len(logs)) } @@ -240,44 +210,32 @@ func TestFilters(t *testing.T) { t.Errorf("expected log[0].Topics[0] to be %x, got %x", hash3, logs[0].Topics[0]) } - filter = New(backend, true) - filter.SetTopics([][]common.Hash{{hash1, hash2}}) - filter.SetBeginBlock(1) - filter.SetEndBlock(10) + filter = New(backend, 1, 10, nil, [][]common.Hash{{hash1, hash2}}) - logs, _ = filter.Find(context.Background()) + logs, _ = filter.Logs(context.Background()) if len(logs) != 2 { t.Error("expected 2 log, got", len(logs)) } failHash := common.BytesToHash([]byte("fail")) - filter = New(backend, true) - filter.SetTopics([][]common.Hash{{failHash}}) - filter.SetBeginBlock(0) - filter.SetEndBlock(-1) + filter = New(backend, 0, -1, nil, [][]common.Hash{{failHash}}) - logs, _ = filter.Find(context.Background()) + logs, _ = filter.Logs(context.Background()) if len(logs) != 0 { t.Error("expected 0 log, got", len(logs)) } failAddr := common.BytesToAddress([]byte("failmenow")) - filter = New(backend, true) - filter.SetAddresses([]common.Address{failAddr}) - filter.SetBeginBlock(0) - filter.SetEndBlock(-1) + filter = New(backend, 0, -1, []common.Address{failAddr}, nil) - logs, _ = filter.Find(context.Background()) + logs, _ = filter.Logs(context.Background()) if len(logs) != 0 { t.Error("expected 0 log, got", len(logs)) } - filter = New(backend, true) - filter.SetTopics([][]common.Hash{{failHash}, {hash1}}) - filter.SetBeginBlock(0) - filter.SetEndBlock(-1) + filter = New(backend, 0, -1, nil, [][]common.Hash{{failHash}, {hash1}}) - logs, _ = filter.Find(context.Background()) + logs, _ = filter.Logs(context.Background()) if len(logs) != 0 { t.Error("expected 0 log, got", len(logs)) } diff --git a/eth/gen_config.go b/eth/gen_config.go index 4774794197..4a4cd7b9c5 100644 --- a/eth/gen_config.go +++ b/eth/gen_config.go @@ -47,7 +47,6 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.SyncMode = c.SyncMode enc.LightServ = c.LightServ enc.LightPeers = c.LightPeers - enc.MaxPeers = c.MaxPeers enc.SkipBcVersionCheck = c.SkipBcVersionCheck enc.DatabaseHandles = c.DatabaseHandles enc.DatabaseCache = c.DatabaseCache @@ -119,9 +118,6 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.LightPeers != nil { c.LightPeers = *dec.LightPeers } - if dec.MaxPeers != nil { - c.MaxPeers = *dec.MaxPeers - } if dec.SkipBcVersionCheck != nil { c.SkipBcVersionCheck = *dec.SkipBcVersionCheck } diff --git a/eth/handler.go b/eth/handler.go index 9d230a4ad6..cee719ddb6 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -99,7 +99,7 @@ type ProtocolManager struct { // NewProtocolManager returns a new ethereum sub protocol manager. The Ethereum sub protocol manages peers capable // with the ethereum network. -func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, networkId uint64, maxPeers int, mux *event.TypeMux, txpool txPool, engine consensus.Engine, blockchain *core.BlockChain, chaindb ethdb.Database) (*ProtocolManager, error) { +func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, networkId uint64, mux *event.TypeMux, txpool txPool, engine consensus.Engine, blockchain *core.BlockChain, chaindb ethdb.Database) (*ProtocolManager, error) { // Create the protocol manager with the base fields manager := &ProtocolManager{ networkId: networkId, @@ -108,7 +108,6 @@ func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, ne blockchain: blockchain, chaindb: chaindb, chainconfig: config, - maxPeers: maxPeers, peers: newPeerSet(), newPeerCh: make(chan *peer), noMorePeers: make(chan struct{}), @@ -203,11 +202,14 @@ func (pm *ProtocolManager) removePeer(id string) { } } -func (pm *ProtocolManager) Start() { +func (pm *ProtocolManager) Start(maxPeers int) { + pm.maxPeers = maxPeers + // broadcast transactions pm.txCh = make(chan core.TxPreEvent, txChanSize) pm.txSub = pm.txpool.SubscribeTxPreEvent(pm.txCh) go pm.txBroadcastLoop() + // broadcast mined blocks pm.minedBlockSub = pm.eventMux.Subscribe(core.NewMinedBlockEvent{}) go pm.minedBroadcastLoop() @@ -607,7 +609,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { // Schedule all the unknown hashes for retrieval unknown := make(newBlockHashesData, 0, len(announces)) for _, block := range announces { - if !pm.blockchain.HasBlock(block.Hash) { + if !pm.blockchain.HasBlock(block.Hash, block.Number) { unknown = append(unknown, block) } } @@ -697,7 +699,7 @@ func (pm *ProtocolManager) BroadcastBlock(block *types.Block, propagate bool) { return } // Otherwise if the block is indeed in out own chain, announce it - if pm.blockchain.HasBlock(hash) { + if pm.blockchain.HasBlock(hash, block.NumberU64()) { for _, peer := range peers { peer.SendNewBlockHashes([]common.Hash{hash}, []uint64{block.NumberU64()}) } diff --git a/eth/handler_test.go b/eth/handler_test.go index aba2774444..6752cd2a84 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -476,11 +476,11 @@ func testDAOChallenge(t *testing.T, localForked, remoteForked bool, timeout bool genesis = gspec.MustCommit(db) blockchain, _ = core.NewBlockChain(db, config, pow, vm.Config{}) ) - pm, err := NewProtocolManager(config, downloader.FullSync, DefaultConfig.NetworkId, 1000, evmux, new(testTxPool), pow, blockchain, db) + pm, err := NewProtocolManager(config, downloader.FullSync, DefaultConfig.NetworkId, evmux, new(testTxPool), pow, blockchain, db) if err != nil { t.Fatalf("failed to start test protocol manager: %v", err) } - pm.Start() + pm.Start(1000) defer pm.Stop() // Connect a new peer and check that we receive the DAO challenge diff --git a/eth/helper_test.go b/eth/helper_test.go index f1dab95283..b665531352 100644 --- a/eth/helper_test.go +++ b/eth/helper_test.go @@ -66,11 +66,11 @@ func newTestProtocolManager(mode downloader.SyncMode, blocks int, generator func panic(err) } - pm, err := NewProtocolManager(gspec.Config, mode, DefaultConfig.NetworkId, 1000, evmux, &testTxPool{added: newtx}, engine, blockchain, db) + pm, err := NewProtocolManager(gspec.Config, mode, DefaultConfig.NetworkId, evmux, &testTxPool{added: newtx}, engine, blockchain, db) if err != nil { return nil, err } - pm.Start() + pm.Start(1000) return pm, nil } diff --git a/eth/sync.go b/eth/sync.go index 7442f912c7..a8ae646170 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -188,7 +188,17 @@ func (pm *ProtocolManager) synchronise(peer *peer) { atomic.StoreUint32(&pm.fastSync, 1) mode = downloader.FastSync } - if err := pm.downloader.Synchronise(peer.id, pHead, pTd, mode); err != nil { + // Run the sync cycle, and disable fast sync if we've went past the pivot block + err := pm.downloader.Synchronise(peer.id, pHead, pTd, mode) + + if atomic.LoadUint32(&pm.fastSync) == 1 { + // Disable fast sync if we indeed have something in our chain + if pm.blockchain.CurrentBlock().NumberU64() > 0 { + log.Info("Fast sync complete, auto disabling") + atomic.StoreUint32(&pm.fastSync, 0) + } + } + if err != nil { return } atomic.StoreUint32(&pm.acceptTxs, 1) // Mark initial sync done @@ -201,12 +211,4 @@ func (pm *ProtocolManager) synchronise(peer *peer) { // more reliably update peers or the local TD state. go pm.BroadcastBlock(head, false) } - // If fast sync was enabled, and we synced up, disable it - if atomic.LoadUint32(&pm.fastSync) == 1 { - // Disable fast sync if we indeed have something in our chain - if pm.blockchain.CurrentBlock().NumberU64() > 0 { - log.Info("Fast sync complete, auto disabling") - atomic.StoreUint32(&pm.fastSync, 0) - } - } } diff --git a/ethdb/database.go b/ethdb/database.go index 7d5fb0b9ef..93755dd7e3 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -109,6 +109,10 @@ func (db *LDBDatabase) Put(key []byte, value []byte) error { return db.db.Put(key, value, nil) } +func (db *LDBDatabase) Has(key []byte) (bool, error) { + return db.db.Has(key, nil) +} + // Get returns the given key if it's present. func (db *LDBDatabase) Get(key []byte) ([]byte, error) { // Measure the database get latency, if requested @@ -271,19 +275,19 @@ func (db *LDBDatabase) meter(refresh time.Duration) { } } -// TODO: remove this stuff and expose leveldb directly - func (db *LDBDatabase) NewBatch() Batch { return &ldbBatch{db: db.db, b: new(leveldb.Batch)} } type ldbBatch struct { - db *leveldb.DB - b *leveldb.Batch + db *leveldb.DB + b *leveldb.Batch + size int } func (b *ldbBatch) Put(key, value []byte) error { b.b.Put(key, value) + b.size += len(value) return nil } @@ -291,6 +295,10 @@ func (b *ldbBatch) Write() error { return b.db.Write(b.b, nil) } +func (b *ldbBatch) ValueSize() int { + return b.size +} + type table struct { db Database prefix string @@ -309,6 +317,10 @@ func (dt *table) Put(key []byte, value []byte) error { return dt.db.Put(append([]byte(dt.prefix), key...), value) } +func (dt *table) Has(key []byte) (bool, error) { + return dt.db.Has(append([]byte(dt.prefix), key...)) +} + func (dt *table) Get(key []byte) ([]byte, error) { return dt.db.Get(append([]byte(dt.prefix), key...)) } @@ -342,3 +354,7 @@ func (tb *tableBatch) Put(key, value []byte) error { func (tb *tableBatch) Write() error { return tb.batch.Write() } + +func (tb *tableBatch) ValueSize() int { + return tb.batch.ValueSize() +} diff --git a/ethdb/interface.go b/ethdb/interface.go index f4b787a52a..99a5b770d5 100644 --- a/ethdb/interface.go +++ b/ethdb/interface.go @@ -16,15 +16,29 @@ package ethdb -type Database interface { +// Code using batches should try to add this much data to the batch. +// The value was determined empirically. +const IdealBatchSize = 100 * 1024 + +// Putter wraps the database write operation supported by both batches and regular databases. +type Putter interface { Put(key []byte, value []byte) error +} + +// Database wraps all database operations. All methods are safe for concurrent use. +type Database interface { + Putter Get(key []byte) ([]byte, error) + Has(key []byte) (bool, error) Delete(key []byte) error Close() NewBatch() Batch } +// Batch is a write-only database that commits changes to its host database +// when Write is called. Batch cannot be used concurrently. type Batch interface { - Put(key, value []byte) error + Putter + ValueSize() int // amount of data in the batch Write() error } diff --git a/ethdb/memory_database.go b/ethdb/memory_database.go index 11b0937248..699bd0c9fd 100644 --- a/ethdb/memory_database.go +++ b/ethdb/memory_database.go @@ -45,6 +45,14 @@ func (db *MemDatabase) Put(key []byte, value []byte) error { return nil } +func (db *MemDatabase) Has(key []byte) (bool, error) { + db.lock.RLock() + defer db.lock.RUnlock() + + _, ok := db.db[string(key)] + return ok, nil +} + func (db *MemDatabase) Get(key []byte) ([]byte, error) { db.lock.RLock() defer db.lock.RUnlock() @@ -93,21 +101,16 @@ type kv struct{ k, v []byte } type memBatch struct { db *MemDatabase writes []kv - lock sync.RWMutex + size int } func (b *memBatch) Put(key, value []byte) error { - b.lock.Lock() - defer b.lock.Unlock() - b.writes = append(b.writes, kv{common.CopyBytes(key), common.CopyBytes(value)}) + b.size += len(value) return nil } func (b *memBatch) Write() error { - b.lock.RLock() - defer b.lock.RUnlock() - b.db.lock.Lock() defer b.db.lock.Unlock() @@ -116,3 +119,7 @@ func (b *memBatch) Write() error { } return nil } + +func (b *memBatch) ValueSize() int { + return b.size +} diff --git a/interfaces.go b/interfaces.go index 744f07b95a..67f236ef75 100644 --- a/interfaces.go +++ b/interfaces.go @@ -102,7 +102,7 @@ type SyncProgress struct { CurrentBlock uint64 // Current block number where sync is at HighestBlock uint64 // Highest alleged block number in the chain PulledStates uint64 // Number of state trie entries already downloaded - KnownStates uint64 // Total number os state trie entries known about + KnownStates uint64 // Total number of state trie entries known about } // ChainSyncReader wraps access to the node's current sync status. If there's no @@ -129,7 +129,7 @@ type ContractCaller interface { CallContract(ctx context.Context, call CallMsg, blockNumber *big.Int) ([]byte, error) } -// FilterQuery contains options for contact log filtering. +// FilterQuery contains options for contract log filtering. type FilterQuery struct { FromBlock *big.Int // beginning of the queried range, nil means genesis block ToBlock *big.Int // end of the range, nil means latest block diff --git a/internal/debug/api.go b/internal/debug/api.go index 8b7693f6a4..3547b05641 100644 --- a/internal/debug/api.go +++ b/internal/debug/api.go @@ -176,6 +176,17 @@ func (*HandlerT) Stacks() string { return string(buf) } +// FreeOSMemory returns unused memory to the OS. +func (*HandlerT) FreeOSMemory() { + debug.FreeOSMemory() +} + +// SetGCPercent sets the garbage collection target percentage. It returns the previous +// setting. A negative value disables GC. +func (*HandlerT) SetGCPercent(v int) int { + return debug.SetGCPercent(v) +} + func writeProfile(name, file string) error { p := pprof.Lookup(name) log.Info("Writing profile records", "count", p.Count(), "type", name, "dump", file) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 30710aaabd..0775749e70 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1265,7 +1265,6 @@ func (s *PublicTransactionPoolAPI) Resend(ctx context.Context, sendArgs SendTxAr if err != nil { return common.Hash{}, err } - s.b.RemoveTx(p.Hash()) if err = s.b.SendTx(ctx, signedTx); err != nil { return common.Hash{}, err } diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index 0a3fe4af09..d32ae83f37 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -59,7 +59,6 @@ type Backend interface { // TxPool API SendTx(ctx context.Context, signedTx *types.Transaction) error - RemoveTx(txHash common.Hash) GetPoolTransactions() (types.Transactions, error) GetPoolTransaction(txHash common.Hash) *types.Transaction GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index bf1db88196..215eae7010 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -34,58 +34,56 @@ var Modules = map[string]string{ const Chequebook_JS = ` web3._extend({ - property: 'chequebook', - methods: - [ - new web3._extend.Method({ - name: 'deposit', - call: 'chequebook_deposit', - params: 1, - inputFormatter: [null] - }), - new web3._extend.Property({ + property: 'chequebook', + methods: [ + new web3._extend.Method({ + name: 'deposit', + call: 'chequebook_deposit', + params: 1, + inputFormatter: [null] + }), + new web3._extend.Property({ name: 'balance', getter: 'chequebook_balance', - outputFormatter: web3._extend.utils.toDecimal + outputFormatter: web3._extend.utils.toDecimal }), - new web3._extend.Method({ - name: 'cash', - call: 'chequebook_cash', - params: 1, - inputFormatter: [null] - }), - new web3._extend.Method({ - name: 'issue', - call: 'chequebook_issue', - params: 2, - inputFormatter: [null, null] - }), - ] + new web3._extend.Method({ + name: 'cash', + call: 'chequebook_cash', + params: 1, + inputFormatter: [null] + }), + new web3._extend.Method({ + name: 'issue', + call: 'chequebook_issue', + params: 2, + inputFormatter: [null, null] + }), + ] }); ` const Clique_JS = ` web3._extend({ - property: 'clique', - methods: - [ + property: 'clique', + methods: [ new web3._extend.Method({ name: 'getSnapshot', call: 'clique_getSnapshot', params: 1, - inputFormatter: [null] + inputFormatter: [null] }), new web3._extend.Method({ name: 'getSnapshotAtHash', call: 'clique_getSnapshotAtHash', params: 1 }), - new web3._extend.Method({ - name: 'getSigners', - call: 'clique_getSigners', - params: 1, - inputFormatter: [null] - }), + new web3._extend.Method({ + name: 'getSigners', + call: 'clique_getSigners', + params: 1, + inputFormatter: [null] + }), new web3._extend.Method({ name: 'getSignersAtHash', call: 'clique_getSignersAtHash', @@ -100,10 +98,9 @@ web3._extend({ name: 'discard', call: 'clique_discard', params: 1 - }) - ], - properties: - [ + }), + ], + properties: [ new web3._extend.Property({ name: 'proposals', getter: 'clique_proposals' @@ -115,8 +112,7 @@ web3._extend({ const Admin_JS = ` web3._extend({ property: 'admin', - methods: - [ + methods: [ new web3._extend.Method({ name: 'addPeer', call: 'admin_addPeer', @@ -162,10 +158,9 @@ web3._extend({ new web3._extend.Method({ name: 'stopWS', call: 'admin_stopWS' - }) + }), ], - properties: - [ + properties: [ new web3._extend.Property({ name: 'nodeInfo', getter: 'admin_nodeInfo' @@ -177,7 +172,7 @@ web3._extend({ new web3._extend.Property({ name: 'datadir', getter: 'admin_datadir' - }) + }), ] }); ` @@ -185,8 +180,7 @@ web3._extend({ const Debug_JS = ` web3._extend({ property: 'debug', - methods: - [ + methods: [ new web3._extend.Method({ name: 'printBlock', call: 'debug_printBlock', @@ -268,6 +262,16 @@ web3._extend({ params: 0, outputFormatter: console.log }), + new web3._extend.Method({ + name: 'freeOSMemory', + call: 'debug_freeOSMemory', + params: 0, + }), + new web3._extend.Method({ + name: 'setGCPercent', + call: 'debug_setGCPercent', + params: 1, + }), new web3._extend.Method({ name: 'memStats', call: 'debug_memStats', @@ -358,8 +362,7 @@ web3._extend({ const Eth_JS = ` web3._extend({ property: 'eth', - methods: - [ + methods: [ new web3._extend.Method({ name: 'sign', call: 'eth_sign', @@ -396,10 +399,9 @@ web3._extend({ }, params: 2, inputFormatter: [web3._extend.formatters.inputBlockNumberFormatter, web3._extend.utils.toHex] - }) + }), ], - properties: - [ + properties: [ new web3._extend.Property({ name: 'pendingTransactions', getter: 'eth_pendingTransactions', @@ -411,7 +413,7 @@ web3._extend({ } return formatted; } - }) + }), ] }); ` @@ -419,8 +421,7 @@ web3._extend({ const Miner_JS = ` web3._extend({ property: 'miner', - methods: - [ + methods: [ new web3._extend.Method({ name: 'start', call: 'miner_start', @@ -451,7 +452,7 @@ web3._extend({ new web3._extend.Method({ name: 'getHashrate', call: 'miner_getHashrate' - }) + }), ], properties: [] }); @@ -461,12 +462,11 @@ const Net_JS = ` web3._extend({ property: 'net', methods: [], - properties: - [ + properties: [ new web3._extend.Property({ name: 'version', getter: 'net_version' - }) + }), ] }); ` @@ -474,8 +474,7 @@ web3._extend({ const Personal_JS = ` web3._extend({ property: 'personal', - methods: - [ + methods: [ new web3._extend.Method({ name: 'importRawKey', call: 'personal_importRawKey', @@ -501,14 +500,13 @@ web3._extend({ name: 'deriveAccount', call: 'personal_deriveAccount', params: 3 - }) + }), ], - properties: - [ + properties: [ new web3._extend.Property({ name: 'listWallets', getter: 'personal_listWallets' - }) + }), ] }) ` @@ -517,12 +515,11 @@ const RPC_JS = ` web3._extend({ property: 'rpc', methods: [], - properties: - [ + properties: [ new web3._extend.Property({ name: 'modules', getter: 'rpc_modules' - }) + }), ] }); ` @@ -633,7 +630,7 @@ web3._extend({ name: 'newMessageFilter', call: 'shh_newMessageFilter', params: 1 - }) + }), ], properties: [ @@ -669,7 +666,7 @@ web3._extend({ name: 'listmounts', call: 'swarmfs_listmounts', params: 0 - }) + }), ] }); ` @@ -696,7 +693,7 @@ web3._extend({ status.queued = web3._extend.utils.toDecimal(status.queued); return status; } - }) + }), ] }); ` diff --git a/les/api_backend.go b/les/api_backend.go index 1323e88644..0d2d31b673 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" @@ -171,3 +172,10 @@ func (b *LesApiBackend) EventMux() *event.TypeMux { func (b *LesApiBackend) AccountManager() *accounts.Manager { return b.eth.accountManager } + +func (b *LesApiBackend) BloomStatus() (uint64, uint64) { + return params.BloomBitsBlocks, 0 +} + +func (b *LesApiBackend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) { +} diff --git a/les/handler.go b/les/handler.go index 1a75cd369d..df7eb6af51 100644 --- a/les/handler.go +++ b/les/handler.go @@ -70,7 +70,7 @@ func errResp(code errCode, format string, v ...interface{}) error { } type BlockChain interface { - HasHeader(hash common.Hash) bool + HasHeader(hash common.Hash, number uint64) bool GetHeader(hash common.Hash, number uint64) *types.Header GetHeaderByHash(hash common.Hash) *types.Header CurrentHeader() *types.Header diff --git a/light/lightchain.go b/light/lightchain.go index df194ecadf..4c877a771a 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -97,9 +97,14 @@ func NewLightChain(odr OdrBackend, config *params.ChainConfig, engine consensus. } if bc.genesisBlock.Hash() == params.MainnetGenesisHash { // add trusted CHT - WriteTrustedCht(bc.chainDb, TrustedCht{Number: 805, Root: common.HexToHash("85e4286fe0a730390245c49de8476977afdae0eb5530b277f62a52b12313d50f")}) + WriteTrustedCht(bc.chainDb, TrustedCht{Number: 1040, Root: common.HexToHash("bb4fb4076cbe6923c8a8ce8f158452bbe19564959313466989fda095a60884ca")}) log.Info("Added trusted CHT for mainnet") } + if bc.genesisBlock.Hash() == params.TestnetGenesisHash { + // add trusted CHT + WriteTrustedCht(bc.chainDb, TrustedCht{Number: 400, Root: common.HexToHash("2a4befa19e4675d939c3dc22dca8c6ae9fcd642be1f04b06bd6e4203cc304660")}) + log.Info("Added trusted CHT for ropsten testnet") + } if err := bc.loadLastState(); err != nil { return nil, err @@ -252,8 +257,8 @@ func (self *LightChain) GetBodyRLP(ctx context.Context, hash common.Hash) (rlp.R // HasBlock checks if a block is fully present in the database or not, caching // it if present. -func (bc *LightChain) HasBlock(hash common.Hash) bool { - blk, _ := bc.GetBlockByHash(NoOdr, hash) +func (bc *LightChain) HasBlock(hash common.Hash, number uint64) bool { + blk, _ := bc.GetBlock(NoOdr, hash, number) return blk != nil } @@ -418,8 +423,8 @@ func (self *LightChain) GetHeaderByHash(hash common.Hash) *types.Header { // HasHeader checks if a block header is present in the database or not, caching // it if present. -func (bc *LightChain) HasHeader(hash common.Hash) bool { - return bc.hc.HasHeader(hash) +func (bc *LightChain) HasHeader(hash common.Hash, number uint64) bool { + return bc.hc.HasHeader(hash, number) } // GetBlockHashesFromHash retrieves a number of block hashes starting at a given diff --git a/miner/worker.go b/miner/worker.go index 24e03be60a..bf24970f5c 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -71,7 +71,6 @@ type Work struct { family *set.Set // family set (used for checking uncle invalidity) uncles *set.Set // uncle set tcount int // tx count in cycle - failedTxs types.Transactions Block *types.Block // the new block @@ -126,8 +125,6 @@ type worker struct { // atomic status counters mining int32 atWork int32 - - fullValidation bool } func newWorker(config *params.ChainConfig, engine consensus.Engine, coinbase common.Address, eth Backend, mux *event.TypeMux) *worker { @@ -147,7 +144,6 @@ func newWorker(config *params.ChainConfig, engine consensus.Engine, coinbase com coinbase: coinbase, agents: make(map[Agent]struct{}), unconfirmed: newUnconfirmedBlocks(eth.BlockChain(), miningLogAtDepth), - fullValidation: false, } // Subscribe TxPreEvent for tx pool worker.txSub = eth.TxPool().SubscribeTxPreEvent(worker.txCh) @@ -269,7 +265,7 @@ func (self *worker) update() { self.currentMu.Lock() acc, _ := types.Sender(self.current.signer, ev.Tx) txs := map[common.Address]types.Transactions{acc: {ev.Tx}} - txset := types.NewTransactionsByPriceAndNonce(txs) + txset := types.NewTransactionsByPriceAndNonce(self.current.signer, txs) self.current.commitTransactions(self.mux, txset, self.chain, self.coinbase) self.currentMu.Unlock() @@ -298,59 +294,38 @@ func (self *worker) wait() { block := result.Block work := result.Work - if self.fullValidation { - if _, err := self.chain.InsertChain(types.Blocks{block}); err != nil { - log.Error("Mined invalid block", "err", err) - continue + // Update the block hash in all logs since it is now available and not when the + // receipt/log of individual transactions were created. + for _, r := range work.receipts { + for _, l := range r.Logs { + l.BlockHash = block.Hash() } - go self.mux.Post(core.NewMinedBlockEvent{Block: block}) - } else { - work.state.CommitTo(self.chainDb, self.config.IsEIP158(block.Number())) - stat, err := self.chain.WriteBlock(block) - if err != nil { - log.Error("Failed writing block to chain", "err", err) - continue - } - // update block hash since it is now available and not when the receipt/log of individual transactions were created - for _, r := range work.receipts { - for _, l := range r.Logs { - l.BlockHash = block.Hash() - } - } - for _, log := range work.state.Logs() { - log.BlockHash = block.Hash() - } - - // check if canon block and write transactions - if stat == core.CanonStatTy { - // This puts transactions in a extra db for rpc - core.WriteTxLookupEntries(self.chainDb, block) - // Write map map bloom filters - core.WriteMipmapBloom(self.chainDb, block.NumberU64(), work.receipts) - // implicit by posting ChainHeadEvent - mustCommitNewWork = false - } - - // broadcast before waiting for validation - go func(block *types.Block, logs []*types.Log, receipts []*types.Receipt) { - self.mux.Post(core.NewMinedBlockEvent{Block: block}) - var ( - events []interface{} - coalescedLogs []*types.Log - ) - events = append(events, core.ChainEvent{Block: block, Hash: block.Hash(), Logs: logs}) - - if stat == core.CanonStatTy { - events = append(events, core.ChainHeadEvent{Block: block}) - coalescedLogs = logs - } - // post blockchain events - self.chain.PostChainEvents(events, coalescedLogs) - if err := core.WriteBlockReceipts(self.chainDb, block.Hash(), block.NumberU64(), receipts); err != nil { - log.Warn("Failed writing block receipts", "err", err) - } - }(block, work.state.Logs(), work.receipts) } + for _, log := range work.state.Logs() { + log.BlockHash = block.Hash() + } + stat, err := self.chain.WriteBlockAndState(block, work.receipts, work.state) + if err != nil { + log.Error("Failed writing block to chain", "err", err) + continue + } + // check if canon block and write transactions + if stat == core.CanonStatTy { + // implicit by posting ChainHeadEvent + mustCommitNewWork = false + } + // Broadcast the block and announce chain insertion event + self.mux.Post(core.NewMinedBlockEvent{Block: block}) + var ( + events []interface{} + logs = work.state.Logs() + ) + events = append(events, core.ChainEvent{Block: block, Hash: block.Hash(), Logs: logs}) + if stat == core.CanonStatTy { + events = append(events, core.ChainHeadEvent{Block: block}) + } + self.chain.PostChainEvents(events, logs) + // Insert the block into the set of pending ones to wait for confirmations self.unconfirmed.Insert(block.NumberU64(), block.Hash()) @@ -474,11 +449,9 @@ func (self *worker) commitNewWork() { log.Error("Failed to fetch pending transactions", "err", err) return } - txs := types.NewTransactionsByPriceAndNonce(pending) + txs := types.NewTransactionsByPriceAndNonce(self.current.signer, pending) work.commitTransactions(self.mux, txs, self.chain, self.coinbase) - self.eth.TxPool().RemoveBatch(work.failedTxs) - // compute uncles for the new block. var ( uncles []*types.Header @@ -563,6 +536,16 @@ func (env *Work) commitTransactions(mux *event.TypeMux, txs *types.TransactionsB log.Trace("Gas limit exceeded for current block", "sender", from) txs.Pop() + case core.ErrNonceTooLow: + // New head notification data race between the transaction pool and miner, shift + log.Trace("Skipping transaction with low nonce", "sender", from, "nonce", tx.Nonce()) + txs.Shift() + + case core.ErrNonceTooHigh: + // Reorg notification data race between the transaction pool and miner, skip account = + log.Trace("Skipping account with hight nonce", "sender", from, "nonce", tx.Nonce()) + txs.Pop() + case nil: // Everything ok, collect the logs and shift in the next transaction from the same account coalescedLogs = append(coalescedLogs, logs...) @@ -570,10 +553,10 @@ func (env *Work) commitTransactions(mux *event.TypeMux, txs *types.TransactionsB txs.Shift() default: - // Pop the current failed transaction without shifting in the next from the account - log.Trace("Transaction failed, will be removed", "hash", tx.Hash(), "err", err) - env.failedTxs = append(env.failedTxs, tx) - txs.Pop() + // Strange error, discard the transaction and get the next in line (note, the + // nonce-too-high clause will prevent us from executing in vain). + log.Debug("Transaction failed, account skipped", "hash", tx.Hash(), "err", err) + txs.Shift() } } diff --git a/p2p/peer.go b/p2p/peer.go index a9c20189a4..fb4b39e950 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -190,7 +190,7 @@ loop: } func (p *Peer) pingLoop() { - ping := time.NewTicker(pingInterval) + ping := time.NewTimer(pingInterval) defer p.wg.Done() defer ping.Stop() for { @@ -200,6 +200,7 @@ func (p *Peer) pingLoop() { p.protoErr <- err return } + ping.Reset(pingInterval) case <-p.closed: return } diff --git a/params/bootnodes.go b/params/bootnodes.go index 6a145a7a8f..82ab6d48c2 100644 --- a/params/bootnodes.go +++ b/params/bootnodes.go @@ -19,7 +19,6 @@ package params // MainnetBootnodes are the enode URLs of the P2P bootstrap nodes running on // the main Ethereum network. var MainnetBootnodes = []string{ - // Ethereum Foundation Go Bootnodes "enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@52.16.188.185:30303", // IE "enode://3f1d12044546b76342d59d4a05532c14b85aa669704bfe1f864fe079415aa2c02d743e03218e57a33fb94523adb54032871a6c51b2cc5514cb7c7e35b3ed0a99@13.93.211.84:30303", // US-WEST @@ -29,7 +28,6 @@ var MainnetBootnodes = []string{ // Ethereum Foundation Cpp Bootnodes "enode://979b7fa28feeb35a4741660a16076f1943202cb72b6af70d327f053e248bab9ba81760f39d0701ef1d8f89cc1fbd2cacba0710a12cd5314d5e0c9021aa3637f9@5.1.83.226:30303", // DE - } // TestnetBootnodes are the enode URLs of the P2P bootstrap nodes running on the @@ -43,12 +41,14 @@ var TestnetBootnodes = []string{ // Rinkeby test network. var RinkebyBootnodes = []string{ "enode://a24ac7c5484ef4ed0c5eb2d36620ba4e4aa13b8c84684e1b4aab0cebea2ae45cb4d375b77eab56516d34bfbd3c1a833fc51296ff084b770b94fb9028c4d25ccf@52.169.42.101:30303", // IE + "enode://343149e4feefa15d882d9fe4ac7d88f885bd05ebb735e547f12e12080a9fa07c8014ca6fd7f373123488102fe5e34111f8509cf0b7de3f5b44339c9f25e87cb8@52.3.158.184:30303", // INFURA } // RinkebyV5Bootnodes are the enode URLs of the P2P bootstrap nodes running on the // Rinkeby test network for the experimental RLPx v5 topic-discovery network. var RinkebyV5Bootnodes = []string{ "enode://a24ac7c5484ef4ed0c5eb2d36620ba4e4aa13b8c84684e1b4aab0cebea2ae45cb4d375b77eab56516d34bfbd3c1a833fc51296ff084b770b94fb9028c4d25ccf@52.169.42.101:30303?discport=30304", // IE + "enode://343149e4feefa15d882d9fe4ac7d88f885bd05ebb735e547f12e12080a9fa07c8014ca6fd7f373123488102fe5e34111f8509cf0b7de3f5b44339c9f25e87cb8@52.3.158.184:30303?discport=30304", // INFURA } // DiscoveryV5Bootnodes are the enode URLs of the P2P bootstrap nodes for the diff --git a/params/config.go b/params/config.go index f4bb6172ba..5c6f0b6bf7 100644 --- a/params/config.go +++ b/params/config.go @@ -32,45 +32,45 @@ var ( var ( // MainnetChainConfig is the chain parameters to run a node on the main network. MainnetChainConfig = &ChainConfig{ - ChainId: big.NewInt(1), - HomesteadBlock: big.NewInt(1150000), - DAOForkBlock: big.NewInt(1920000), - DAOForkSupport: true, - EIP150Block: big.NewInt(2463000), - EIP150Hash: common.HexToHash("0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0"), - EIP155Block: big.NewInt(2675000), - EIP158Block: big.NewInt(2675000), - MetropolisBlock: big.NewInt(math.MaxInt64), // Don't enable yet + ChainId: big.NewInt(1), + HomesteadBlock: big.NewInt(1150000), + DAOForkBlock: big.NewInt(1920000), + DAOForkSupport: true, + EIP150Block: big.NewInt(2463000), + EIP150Hash: common.HexToHash("0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0"), + EIP155Block: big.NewInt(2675000), + EIP158Block: big.NewInt(2675000), + ByzantiumBlock: big.NewInt(math.MaxInt64), // Don't enable yet Ethash: new(EthashConfig), } // TestnetChainConfig contains the chain parameters to run a node on the Ropsten test network. TestnetChainConfig = &ChainConfig{ - ChainId: big.NewInt(3), - HomesteadBlock: big.NewInt(0), - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: big.NewInt(0), - EIP150Hash: common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d"), - EIP155Block: big.NewInt(10), - EIP158Block: big.NewInt(10), - MetropolisBlock: big.NewInt(math.MaxInt64), // Don't enable yet + ChainId: big.NewInt(3), + HomesteadBlock: big.NewInt(0), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(0), + EIP150Hash: common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d"), + EIP155Block: big.NewInt(10), + EIP158Block: big.NewInt(10), + ByzantiumBlock: big.NewInt(1700000), Ethash: new(EthashConfig), } // RinkebyChainConfig contains the chain parameters to run a node on the Rinkeby test network. RinkebyChainConfig = &ChainConfig{ - ChainId: big.NewInt(4), - HomesteadBlock: big.NewInt(1), - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: big.NewInt(2), - EIP150Hash: common.HexToHash("0x9b095b36c15eaf13044373aef8ee0bd3a382a5abb92e402afa44b8249c3a90e9"), - EIP155Block: big.NewInt(3), - EIP158Block: big.NewInt(3), - MetropolisBlock: big.NewInt(math.MaxInt64), // Don't enable yet + ChainId: big.NewInt(4), + HomesteadBlock: big.NewInt(1), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(2), + EIP150Hash: common.HexToHash("0x9b095b36c15eaf13044373aef8ee0bd3a382a5abb92e402afa44b8249c3a90e9"), + EIP155Block: big.NewInt(3), + EIP158Block: big.NewInt(3), + ByzantiumBlock: big.NewInt(math.MaxInt64), // Don't enable yet Clique: &CliqueConfig{ Period: 15, @@ -86,8 +86,8 @@ var ( // means that all fields must be set at all times. This forces // anyone adding flags to the config to also have to set these // fields. - AllProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(math.MaxInt64) /*disabled*/, new(EthashConfig), nil} - TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil} + AllProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), new(EthashConfig), nil} + TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), new(EthashConfig), nil} TestRules = TestChainConfig.Rules(new(big.Int)) ) @@ -110,7 +110,7 @@ type ChainConfig struct { EIP155Block *big.Int `json:"eip155Block,omitempty"` // EIP155 HF block EIP158Block *big.Int `json:"eip158Block,omitempty"` // EIP158 HF block - MetropolisBlock *big.Int `json:"metropolisBlock,omitempty"` // Metropolis switch block (nil = no fork, 0 = alraedy on homestead) + ByzantiumBlock *big.Int `json:"byzantiumBlock,omitempty"` // Byzantium switch block (nil = no fork, 0 = alraedy on homestead) // Various consensus engines Ethash *EthashConfig `json:"ethash,omitempty"` @@ -147,7 +147,7 @@ func (c *ChainConfig) String() string { default: engine = "unknown" } - return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Metropolis: %v Engine: %v}", + return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Engine: %v}", c.ChainId, c.HomesteadBlock, c.DAOForkBlock, @@ -155,7 +155,7 @@ func (c *ChainConfig) String() string { c.EIP150Block, c.EIP155Block, c.EIP158Block, - c.MetropolisBlock, + c.ByzantiumBlock, engine, ) } @@ -182,8 +182,8 @@ func (c *ChainConfig) IsEIP158(num *big.Int) bool { return isForked(c.EIP158Block, num) } -func (c *ChainConfig) IsMetropolis(num *big.Int) bool { - return isForked(c.MetropolisBlock, num) +func (c *ChainConfig) IsByzantium(num *big.Int) bool { + return isForked(c.ByzantiumBlock, num) } // GasTable returns the gas table corresponding to the current phase (homestead or homestead reprice). @@ -243,8 +243,8 @@ func (c *ChainConfig) checkCompatible(newcfg *ChainConfig, head *big.Int) *Confi if c.IsEIP158(head) && !configNumEqual(c.ChainId, newcfg.ChainId) { return newCompatError("EIP158 chain ID", c.EIP158Block, newcfg.EIP158Block) } - if isForkIncompatible(c.MetropolisBlock, newcfg.MetropolisBlock, head) { - return newCompatError("Metropolis fork block", c.MetropolisBlock, newcfg.MetropolisBlock) + if isForkIncompatible(c.ByzantiumBlock, newcfg.ByzantiumBlock, head) { + return newCompatError("Byzantium fork block", c.ByzantiumBlock, newcfg.ByzantiumBlock) } return nil } @@ -312,7 +312,7 @@ func (err *ConfigCompatError) Error() string { type Rules struct { ChainId *big.Int IsHomestead, IsEIP150, IsEIP155, IsEIP158 bool - IsMetropolis bool + IsByzantium bool } func (c *ChainConfig) Rules(num *big.Int) Rules { @@ -320,5 +320,5 @@ func (c *ChainConfig) Rules(num *big.Int) Rules { if chainId == nil { chainId = new(big.Int) } - return Rules{ChainId: new(big.Int).Set(chainId), IsHomestead: c.IsHomestead(num), IsEIP150: c.IsEIP150(num), IsEIP155: c.IsEIP155(num), IsEIP158: c.IsEIP158(num), IsMetropolis: c.IsMetropolis(num)} + return Rules{ChainId: new(big.Int).Set(chainId), IsHomestead: c.IsHomestead(num), IsEIP150: c.IsEIP150(num), IsEIP155: c.IsEIP155(num), IsEIP158: c.IsEIP158(num), IsByzantium: c.IsByzantium(num)} } diff --git a/params/network_params.go b/params/network_params.go new file mode 100644 index 0000000000..536a46d3df --- /dev/null +++ b/params/network_params.go @@ -0,0 +1,26 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package params + +// These are network parameters that need to be constant between clients, but +// aren't necesarilly consensus related. + +const ( + // BloomBitsBlocks is the number of blocks a single bloom bit section vector + // contains. + BloomBitsBlocks uint64 = 4096 +) diff --git a/params/protocol_params.go b/params/protocol_params.go index 9c84c7d348..c56faf56f8 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -64,9 +64,9 @@ const ( Ripemd160PerWordGas uint64 = 120 // Per-word price for a RIPEMD160 operation IdentityBaseGas uint64 = 15 // Base price for a data copy operation IdentityPerWordGas uint64 = 3 // Per-work price for a data copy operation - ModExpQuadCoeffDiv uint64 = 100 // Divisor for the quadratic particle of the big int modular exponentiation + ModExpQuadCoeffDiv uint64 = 20 // Divisor for the quadratic particle of the big int modular exponentiation Bn256AddGas uint64 = 500 // Gas needed for an elliptic curve addition - Bn256ScalarMulGas uint64 = 2000 // Gas needed for an elliptic curve scalar multiplication + Bn256ScalarMulGas uint64 = 40000 // Gas needed for an elliptic curve scalar multiplication Bn256PairingBaseGas uint64 = 100000 // Base price for an elliptic curve pairing check Bn256PairingPerPointGas uint64 = 80000 // Per-point price for an elliptic curve pairing check ) diff --git a/params/version.go b/params/version.go index 0896662606..80329c91d9 100644 --- a/params/version.go +++ b/params/version.go @@ -23,7 +23,7 @@ import ( const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 7 // Minor version component of the current release - VersionPatch = 0 // Patch version component of the current release + VersionPatch = 1 // Patch version component of the current release VersionMeta = "unstable" // Version metadata to append to the version string ) diff --git a/swarm/api/api.go b/swarm/api/api.go index 7d185ab3ca..a5941fb5cd 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -132,6 +132,7 @@ func (self *Api) Put(content, contentType string) (storage.Key, error) { func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionReader, mimeType string, status int, err error) { trie, err := loadManifest(self.dpa, key, nil) if err != nil { + status = http.StatusNotFound log.Warn(fmt.Sprintf("loadManifestTrie error: %v", err)) return } diff --git a/swarm/api/http/error.go b/swarm/api/http/error.go new file mode 100644 index 0000000000..ebb5e3ebe2 --- /dev/null +++ b/swarm/api/http/error.go @@ -0,0 +1,116 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +/* +Show nicely (but simple) formatted HTML error pages (or respond with JSON +if the appropriate `Accept` header is set)) for the http package. +*/ +package http + +import ( + "encoding/json" + "fmt" + "html/template" + "net/http" + "time" + + "github.com/ethereum/go-ethereum/log" +) + +//templateMap holds a mapping of an HTTP error code to a template +var templateMap map[int]*template.Template + +//parameters needed for formatting the correct HTML page +type ErrorParams struct { + Msg string + Code int + Timestamp string + template *template.Template + Details template.HTML +} + +//we init the error handling right on boot time, so lookup and http response is fast +func init() { + initErrHandling() +} + +func initErrHandling() { + //pages are saved as strings - get these strings + genErrPage := GetGenericErrorPage() + notFoundPage := GetNotFoundErrorPage() + //map the codes to the available pages + tnames := map[int]string{ + 0: genErrPage, //default + 400: genErrPage, + 404: notFoundPage, + 500: genErrPage, + } + templateMap = make(map[int]*template.Template) + for code, tname := range tnames { + //assign formatted HTML to the code + templateMap[code] = template.Must(template.New(fmt.Sprintf("%d", code)).Parse(tname)) + } +} + +//ShowError is used to show an HTML error page to a client. +//If there is an `Accept` header of `application/json`, JSON will be returned instead +//The function just takes a string message which will be displayed in the error page. +//The code is used to evaluate which template will be displayed +//(and return the correct HTTP status code) +func ShowError(w http.ResponseWriter, r *http.Request, msg string, code int) { + if code == http.StatusInternalServerError { + log.Error(msg) + } + respond(w, r, &ErrorParams{ + Code: code, + Msg: msg, + Timestamp: time.Now().Format(time.RFC1123), + template: getTemplate(code), + }) +} + +//evaluate if client accepts html or json response +func respond(w http.ResponseWriter, r *http.Request, params *ErrorParams) { + w.WriteHeader(params.Code) + if r.Header.Get("Accept") == "application/json" { + respondJson(w, params) + } else { + respondHtml(w, params) + } +} + +//return a HTML page +func respondHtml(w http.ResponseWriter, params *ErrorParams) { + err := params.template.Execute(w, params) + if err != nil { + log.Error(err.Error()) + } +} + +//return JSON +func respondJson(w http.ResponseWriter, params *ErrorParams) { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(params) +} + +//get the HTML template for a given code +func getTemplate(code int) *template.Template { + if val, tmpl := templateMap[code]; tmpl { + return val + } else { + return templateMap[0] + } +} diff --git a/swarm/api/http/error_templates.go b/swarm/api/http/error_templates.go new file mode 100644 index 0000000000..29bd3bfbb1 --- /dev/null +++ b/swarm/api/http/error_templates.go @@ -0,0 +1,376 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +/* +We use html templates to handle simple but as informative as possible error pages. + +To eliminate circular dependency in case of an error, we don't store error pages on swarm. +We can't save the error pages as html files on disk, or when deploying compiled binaries +they won't be found. + +For this reason we resort to save the HTML error pages as strings, which then can be +parsed by Go's html/template package +*/ +package http + +//This returns the HTML for generic errors +func GetGenericErrorPage() string { + page := ` + + + + + + + + + + + + Swarm::HTTP Error Page + + + + +
+ +
+
+ +
+
+

There was a problem serving the requested page

+
+
+
{{.Timestamp}}
+
+
+ + +
+ + + + + + + + + + + + + + + + + + + + +
+ Hmmmmm....Swarm was not able to serve your request! +
+ Error message: +
+ {{.Msg}} +
+ Error code: +
+ {{.Code}} +
+
+ + +
+

+ Swarm: Serverless Hosting Incentivised Peer-To-Peer Storage And Content Distribution
+ Swarm +

+
+ + +
+ + + +` + return page +} + +//This returns the HTML for a 404 Not Found error +func GetNotFoundErrorPage() string { + page := ` + + + + + + + + + + + + Swarm::404 HTTP Not Found + + + + +
+ +
+
+ +
+
+

Resource Not Found

+
+
+
{{.Timestamp}}
+
+
+ + +
+ + + + + + + + + + + + + + + + + + + + +
+ Unfortunately, the resource you were trying to access could not be found on swarm. +
+
+ {{.Msg}} +
+ Error code: +
+ {{.Code}} +
+
+ + +
+

+ Swarm: Serverless Hosting Incentivised Peer-To-Peer Storage And Content Distribution
+ Swarm +

+
+ + +
+ + + +` + return page +} diff --git a/swarm/api/http/error_test.go b/swarm/api/http/error_test.go new file mode 100644 index 0000000000..465fbf4cad --- /dev/null +++ b/swarm/api/http/error_test.go @@ -0,0 +1,143 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package http_test + +import ( + "encoding/json" + "golang.org/x/net/html" + "io/ioutil" + "net/http" + "strings" + "testing" + + "github.com/ethereum/go-ethereum/swarm/testutil" +) + +func TestError(t *testing.T) { + + srv := testutil.NewTestSwarmServer(t) + defer srv.Close() + + var resp *http.Response + var respbody []byte + + url := srv.URL + "/this_should_fail_as_no_bzz_protocol_present" + resp, err := http.Get(url) + + if err != nil { + t.Fatalf("Request failed: %v", err) + } + defer resp.Body.Close() + respbody, err = ioutil.ReadAll(resp.Body) + + if resp.StatusCode != 400 && !strings.Contains(string(respbody), "Invalid URI "/this_should_fail_as_no_bzz_protocol_present": unknown scheme") { + t.Fatalf("Response body does not match, expected: %v, to contain: %v; received code %d, expected code: %d", string(respbody), "Invalid bzz URI: unknown scheme", 400, resp.StatusCode) + } + + _, err = html.Parse(strings.NewReader(string(respbody))) + if err != nil { + t.Fatalf("HTML validation failed for error page returned!") + } +} + +func Test404Page(t *testing.T) { + srv := testutil.NewTestSwarmServer(t) + defer srv.Close() + + var resp *http.Response + var respbody []byte + + url := srv.URL + "/bzz:/1234567890123456789012345678901234567890123456789012345678901234" + resp, err := http.Get(url) + + if err != nil { + t.Fatalf("Request failed: %v", err) + } + defer resp.Body.Close() + respbody, err = ioutil.ReadAll(resp.Body) + + if resp.StatusCode != 404 || !strings.Contains(string(respbody), "404") { + t.Fatalf("Invalid Status Code received, expected 404, got %d", resp.StatusCode) + } + + _, err = html.Parse(strings.NewReader(string(respbody))) + if err != nil { + t.Fatalf("HTML validation failed for error page returned!") + } +} + +func Test500Page(t *testing.T) { + srv := testutil.NewTestSwarmServer(t) + defer srv.Close() + + var resp *http.Response + var respbody []byte + + url := srv.URL + "/bzz:/thisShouldFailWith500Code" + resp, err := http.Get(url) + + if err != nil { + t.Fatalf("Request failed: %v", err) + } + defer resp.Body.Close() + respbody, err = ioutil.ReadAll(resp.Body) + + if resp.StatusCode != 500 || !strings.Contains(string(respbody), "500") { + t.Fatalf("Invalid Status Code received, expected 500, got %d", resp.StatusCode) + } + + _, err = html.Parse(strings.NewReader(string(respbody))) + if err != nil { + t.Fatalf("HTML validation failed for error page returned!") + } +} + +func TestJsonResponse(t *testing.T) { + srv := testutil.NewTestSwarmServer(t) + defer srv.Close() + + var resp *http.Response + var respbody []byte + + url := srv.URL + "/bzz:/thisShouldFailWith500Code/" + req, err := http.NewRequest("GET", url, nil) + if err != nil { + t.Fatalf("Request failed: %v", err) + } + req.Header.Set("Accept", "application/json") + resp, err = http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("Request failed: %v", err) + } + + defer resp.Body.Close() + respbody, err = ioutil.ReadAll(resp.Body) + + if resp.StatusCode != 500 { + t.Fatalf("Invalid Status Code received, expected 500, got %d", resp.StatusCode) + } + + if !isJSON(string(respbody)) { + t.Fatalf("Expected repsonse to be JSON, received invalid JSON: %s", string(respbody)) + } + +} + +func isJSON(s string) bool { + var js map[string]interface{} + return json.Unmarshal([]byte(s), &js) == nil +} diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 0b4ec7e18e..a637b87351 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -332,7 +332,7 @@ func (s *Server) HandleGetRaw(w http.ResponseWriter, r *Request) { return api.SkipManifest }) if entry == nil { - http.NotFound(w, &r.Request) + s.NotFound(w, r, fmt.Errorf("Manifest entry could not be loaded")) return } key = storage.Key(common.Hex2Bytes(entry.Hash)) @@ -341,8 +341,7 @@ func (s *Server) HandleGetRaw(w http.ResponseWriter, r *Request) { // check the root chunk exists by retrieving the file's size reader := s.api.Retrieve(key) if _, err := reader.Size(nil); err != nil { - s.logDebug("key not found %s: %s", key, err) - http.NotFound(w, &r.Request) + s.NotFound(w, r, fmt.Errorf("Root chunk not found %s: %s", key, err)) return } @@ -534,16 +533,20 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { return } - reader, contentType, _, err := s.api.Get(key, r.uri.Path) + reader, contentType, status, err := s.api.Get(key, r.uri.Path) if err != nil { - s.Error(w, r, err) + switch status { + case http.StatusNotFound: + s.NotFound(w, r, err) + default: + s.Error(w, r, err) + } return } // check the root chunk exists by retrieving the file's size if _, err := reader.Size(nil); err != nil { - s.logDebug("file not found %s: %s", r.uri, err) - http.NotFound(w, &r.Request) + s.NotFound(w, r, fmt.Errorf("File not found %s: %s", r.uri, err)) return } @@ -556,14 +559,14 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.logDebug("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, r.URL.Host, r.URL.Path, r.Referer(), r.Header.Get("Accept")) uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/")) + req := &Request{Request: *r, uri: uri} if err != nil { s.logError("Invalid URI %q: %s", r.URL.Path, err) - http.Error(w, fmt.Sprintf("Invalid bzz URI: %s", err), http.StatusBadRequest) + s.BadRequest(w, req, fmt.Sprintf("Invalid URI %q: %s", r.URL.Path, err)) return } s.logDebug("%s request received for %s", r.Method, uri) - req := &Request{Request: *r, uri: uri} switch r.Method { case "POST": if uri.Raw() { @@ -579,7 +582,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { // strictly a traditional PUT request which replaces content // at a URI, and POST is more ubiquitous) if uri.Raw() { - http.Error(w, fmt.Sprintf("No PUT to %s allowed.", uri), http.StatusBadRequest) + ShowError(w, r, fmt.Sprintf("No PUT to %s allowed.", uri), http.StatusBadRequest) return } else { s.HandlePostFiles(w, req) @@ -587,7 +590,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { case "DELETE": if uri.Raw() { - http.Error(w, fmt.Sprintf("No DELETE to %s allowed.", uri), http.StatusBadRequest) + ShowError(w, r, fmt.Sprintf("No DELETE to %s allowed.", uri), http.StatusBadRequest) return } s.HandleDelete(w, req) @@ -611,7 +614,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.HandleGetFile(w, req) default: - http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed) + ShowError(w, r, fmt.Sprintf("Method "+r.Method+" is not supported.", uri), http.StatusMethodNotAllowed) } } @@ -643,11 +646,13 @@ func (s *Server) logError(format string, v ...interface{}) { } func (s *Server) BadRequest(w http.ResponseWriter, r *Request, reason string) { - s.logDebug("bad request %s %s: %s", r.Method, r.uri, reason) - http.Error(w, reason, http.StatusBadRequest) + ShowError(w, &r.Request, fmt.Sprintf("Bad request %s %s: %s", r.Method, r.uri, reason), http.StatusBadRequest) } func (s *Server) Error(w http.ResponseWriter, r *Request, err error) { - s.logError("error serving %s %s: %s", r.Method, r.uri, err) - http.Error(w, err.Error(), http.StatusInternalServerError) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, err), http.StatusInternalServerError) +} + +func (s *Server) NotFound(w http.ResponseWriter, r *Request, err error) { + ShowError(w, &r.Request, fmt.Sprintf("NOT FOUND error serving %s %s: %s", r.Method, r.uri, err), http.StatusNotFound) } diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index d3374594b6..ffeaf6e0d8 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -22,6 +22,7 @@ import ( "fmt" "io/ioutil" "net/http" + "strings" "sync" "testing" @@ -110,9 +111,9 @@ func TestBzzrGetPath(t *testing.T) { } nonhashresponses := []string{ - "error resolving name: no DNS to resolve name: \"name\"\n", - "error resolving nonhash: immutable address not a content hash: \"nonhash\"\n", - "error resolving nonhash: no DNS to resolve name: \"nonhash\"\n", + "error resolving name: no DNS to resolve name: "name"", + "error resolving nonhash: immutable address not a content hash: "nonhash"", + "error resolving nonhash: no DNS to resolve name: "nonhash"", } for i, url := range nonhashtests { @@ -129,7 +130,7 @@ func TestBzzrGetPath(t *testing.T) { if err != nil { t.Fatalf("ReadAll failed: %v", err) } - if string(respbody) != nonhashresponses[i] { + if !strings.Contains(string(respbody), nonhashresponses[i]) { t.Fatalf("Non-Hash response body does not match, expected: %v, got: %v", nonhashresponses[i], string(respbody)) } } diff --git a/swarm/fuse/swarmfs_test.go b/swarm/fuse/swarmfs_test.go index 69f3cc6157..93f1d4c2fd 100644 --- a/swarm/fuse/swarmfs_test.go +++ b/swarm/fuse/swarmfs_test.go @@ -140,7 +140,7 @@ func compareGeneratedFileWithFileInMount(t *testing.T, files map[string]fileInfo if err != nil { t.Fatalf("Could not readfile %v : %v", fname, err) } - if bytes.Compare(fileContents, finfo.contents) != 0 { + if !bytes.Equal(fileContents, finfo.contents) { t.Fatalf("File %v contents mismatch: %v , %v", fname, fileContents, finfo.contents) } diff --git a/swarm/storage/chunker.go b/swarm/storage/chunker.go index 563793e98f..ca85e43337 100644 --- a/swarm/storage/chunker.go +++ b/swarm/storage/chunker.go @@ -51,7 +51,8 @@ data_{i} := size(subtree_{i}) || key_{j} || key_{j+1} .... || key_{j+n-1} */ const ( - defaultHash = "SHA3" // http://golang.org/pkg/hash/#Hash + defaultHash = "SHA3" + // defaultHash = "BMTSHA3" // http://golang.org/pkg/hash/#Hash // defaultHash = "SHA256" // http://golang.org/pkg/hash/#Hash defaultBranches int64 = 128 // hashSize int64 = hasherfunc.New().Size() // hasher knows about its own length in bytes diff --git a/swarm/storage/types.go b/swarm/storage/types.go index cc5ded931d..a9de23c93a 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -24,6 +24,7 @@ import ( "io" "sync" + // "github.com/ethereum/go-ethereum/bmt" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto/sha3" ) diff --git a/tests/block_test.go b/tests/block_test.go index 56e1e1e8da..669d3ca08a 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -32,8 +32,6 @@ func TestBlockchain(t *testing.T) { bt.skipLoad(`^bcTotalDifficultyTest/(lotsOfLeafs|lotsOfBranches|sideChainWithMoreTransactions)`) // Constantinople is not implemented yet. bt.skipLoad(`(?i)(constantinople)`) - // Expected failures: - bt.fails(`^TransitionTests/bcHomesteadToDao/DaoTransactions(|_UncleExtradata|_EmptyTransactionAndForkBlocksAhead)\.json`, "issue in test") // Still failing tests bt.skipLoad(`^bcWalletTest.*_Byzantium$`) diff --git a/tests/init.go b/tests/init.go index 0c3fe61d13..a2c633ad67 100644 --- a/tests/init.go +++ b/tests/init.go @@ -45,13 +45,13 @@ var Forks = map[string]*params.ChainConfig{ EIP158Block: big.NewInt(0), }, "Byzantium": ¶ms.ChainConfig{ - ChainId: big.NewInt(1), - HomesteadBlock: big.NewInt(0), - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - DAOForkBlock: big.NewInt(0), - MetropolisBlock: big.NewInt(0), + ChainId: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + DAOForkBlock: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), }, "FrontierToHomesteadAt5": ¶ms.ChainConfig{ ChainId: big.NewInt(1), @@ -69,12 +69,12 @@ var Forks = map[string]*params.ChainConfig{ DAOForkSupport: true, }, "EIP158ToByzantiumAt5": ¶ms.ChainConfig{ - ChainId: big.NewInt(1), - HomesteadBlock: big.NewInt(0), - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - MetropolisBlock: big.NewInt(5), + ChainId: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(5), }, } diff --git a/tests/state_test.go b/tests/state_test.go index 3d7b29012e..9a7430fbeb 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -35,12 +35,17 @@ func TestState(t *testing.T) { st.skipLoad(`^stTransactionTest/OverflowGasRequire\.json`) // gasLimit > 256 bits st.skipLoad(`^stTransactionTest/zeroSigTransa[^/]*\.json`) // EIP-86 is not supported yet // Expected failures: - st.fails(`^stCodeSizeLimit/codesizeOOGInvalidSize\.json/(Frontier|Homestead|EIP150)`, - "code size limit implementation is not conditional on fork") st.fails(`^stRevertTest/RevertPrecompiledTouch\.json/EIP158`, "bug in test") st.fails(`^stRevertTest/RevertPrefoundEmptyOOG\.json/EIP158`, "bug in test") st.fails(`^stRevertTest/RevertPrecompiledTouch\.json/Byzantium`, "bug in test") st.fails(`^stRevertTest/RevertPrefoundEmptyOOG\.json/Byzantium`, "bug in test") + st.fails( `^stRandom/randomStatetest645\.json/EIP150/.*`, "known bug #15119") + st.fails( `^stRandom/randomStatetest645\.json/Frontier/.*`, "known bug #15119") + st.fails( `^stRandom/randomStatetest645\.json/Homestead/.*`, "known bug #15119") + st.fails( `^stRandom/randomStatetest644\.json/EIP150/.*`, "known bug #15119") + st.fails( `^stRandom/randomStatetest644\.json/Frontier/.*`, "known bug #15119") + st.fails( `^stRandom/randomStatetest644\.json/Homestead/.*`, "known bug #15119") + st.walk(t, stateTestDir, func(t *testing.T, name string, test *StateTest) { for _, subtest := range test.Subtests() { @@ -52,7 +57,8 @@ func TestState(t *testing.T) { t.Skip("constantinople not supported yet") } withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { - return st.checkFailure(t, name, test.Run(subtest, vmconfig)) + _, err := test.Run(subtest, vmconfig) + return st.checkFailure(t, name, err) }) }) } @@ -60,7 +66,8 @@ func TestState(t *testing.T) { } // Transactions with gasLimit above this value will not get a VM trace on failure. -const traceErrorLimit = 400000 +//const traceErrorLimit = 400000 +const traceErrorLimit = 0 func withTrace(t *testing.T, gasLimit uint64, test func(vm.Config) error) { err := test(vm.Config{}) diff --git a/tests/state_test_util.go b/tests/state_test_util.go index ecaa6c6684..64bf09cb42 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -120,10 +120,10 @@ func (t *StateTest) Subtests() []StateSubtest { } // Run executes a specific subtest. -func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config) error { +func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config) (*state.StateDB, error) { config, ok := Forks[subtest.Fork] if !ok { - return UnsupportedForkError{subtest.Fork} + return nil, UnsupportedForkError{subtest.Fork} } block, _ := t.genesis(config).ToBlock() db, _ := ethdb.NewMemDatabase() @@ -132,7 +132,7 @@ func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config) error { post := t.json.Post[subtest.Fork][subtest.Index] msg, err := t.json.Tx.toMessage(post) if err != nil { - return err + return nil, err } context := core.NewEVMContext(msg, block.Header(), nil, &t.json.Env.Coinbase) context.GetHash = vmTestBlockHash @@ -145,13 +145,13 @@ func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config) error { statedb.RevertToSnapshot(snapshot) } if logs := rlpHash(statedb.Logs()); logs != common.Hash(post.Logs) { - return fmt.Errorf("post state logs hash mismatch: got %x, want %x", logs, post.Logs) + return statedb, fmt.Errorf("post state logs hash mismatch: got %x, want %x", logs, post.Logs) } root, _ := statedb.CommitTo(db, config.IsEIP158(block.Number())) if root != common.Hash(post.Root) { - return fmt.Errorf("post state root mismatch: got %x, want %x", root, post.Root) + return statedb, fmt.Errorf("post state root mismatch: got %x, want %x", root, post.Root) } - return nil + return statedb, nil } func (t *StateTest) gasLimit(subtest StateSubtest) uint64 { diff --git a/tests/testdata b/tests/testdata index 1d30b47956..ca41e90635 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit 1d30b4795664f64b1b157971754e14a10cfd9115 +Subproject commit ca41e906351209481bce3a1b35501f25a79023c5 diff --git a/tests/transaction_test.go b/tests/transaction_test.go index 72d43c0ece..c743996c2a 100644 --- a/tests/transaction_test.go +++ b/tests/transaction_test.go @@ -37,12 +37,12 @@ func TestTransaction(t *testing.T) { EIP158Block: big.NewInt(0), ChainId: big.NewInt(1), }) - txt.config(`^Metropolis/`, params.ChainConfig{ - HomesteadBlock: big.NewInt(0), - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - MetropolisBlock: big.NewInt(0), + txt.config(`^Byzantium/`, params.ChainConfig{ + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), }) txt.walk(t, transactionTestDir, func(t *testing.T, name string, test *TransactionTest) { diff --git a/trie/sync.go b/trie/sync.go index 1e4f8d87c1..fea10051f4 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -138,7 +138,7 @@ func (s *TrieSync) AddRawEntry(hash common.Hash, depth int, parent common.Hash) if _, ok := s.membatch.batch[hash]; ok { return } - if blob, _ := s.database.Get(hash.Bytes()); blob != nil { + if ok, _ := s.database.Has(hash.Bytes()); ok { return } // Assemble the new sub-trie sync request @@ -296,8 +296,7 @@ func (s *TrieSync) children(req *request, object node) ([]*request, error) { if _, ok := s.membatch.batch[hash]; ok { continue } - blob, _ := s.database.Get(node) - if local, err := decodeNode(node[:], blob, 0); local != nil && err == nil { + if ok, _ := s.database.Has(node); ok { continue } // Locally unknown node, schedule for retrieval diff --git a/trie/trie.go b/trie/trie.go index a3151b1ce7..7f69a3d1de 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -66,6 +66,7 @@ type Database interface { // DatabaseReader wraps the Get method of a backing store for the trie. type DatabaseReader interface { Get(key []byte) (value []byte, err error) + Has(key []byte) (bool, error) } // DatabaseWriter wraps the Put method of a backing store for the trie. diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db.go b/vendor/github.com/syndtr/goleveldb/leveldb/db.go index a02cb2c500..b0cdcb3d0a 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/db.go @@ -844,7 +844,7 @@ func (db *DB) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) { // Has returns true if the DB does contains the given key. // -// It is safe to modify the contents of the argument after Get returns. +// It is safe to modify the contents of the argument after Has returns. func (db *DB) Has(key []byte, ro *opt.ReadOptions) (ret bool, err error) { err = db.ok() if err != nil { diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go b/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go index 2d0ad0753c..b6563e87e4 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go @@ -289,7 +289,7 @@ func (db *DB) memCompaction() { close(resumeC) resumeC = nil case <-db.closeC: - return + db.compactionExitTransact() } var ( @@ -338,7 +338,7 @@ func (db *DB) memCompaction() { case <-resumeC: close(resumeC) case <-db.closeC: - return + db.compactionExitTransact() } } diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go b/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go index 85b02d24bf..65e1c54bb4 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go @@ -7,6 +7,7 @@ package leveldb import ( + "errors" "sync/atomic" "time" @@ -15,6 +16,10 @@ import ( "github.com/syndtr/goleveldb/leveldb/storage" ) +var ( + errHasFrozenMem = errors.New("has frozen mem") +) + type memDB struct { db *DB *memdb.DB @@ -126,7 +131,7 @@ func (db *DB) newMem(n int) (mem *memDB, err error) { defer db.memMu.Unlock() if db.frozenMem != nil { - panic("still has frozen mem") + return nil, errHasFrozenMem } if db.journal == nil { diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db_write.go b/vendor/github.com/syndtr/goleveldb/leveldb/db_write.go index cc428b695c..5b6cb487dc 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db_write.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/db_write.go @@ -32,15 +32,24 @@ func (db *DB) writeJournal(batches []*Batch, seq uint64, sync bool) error { } func (db *DB) rotateMem(n int, wait bool) (mem *memDB, err error) { + retryLimit := 3 +retry: // Wait for pending memdb compaction. err = db.compTriggerWait(db.mcompCmdC) if err != nil { return } + retryLimit-- // Create new memdb and journal. mem, err = db.newMem(n) if err != nil { + if err == errHasFrozenMem { + if retryLimit <= 0 { + panic("BUG: still has frozen memdb") + } + goto retry + } return } diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/doc.go b/vendor/github.com/syndtr/goleveldb/leveldb/doc.go index 53f13bb24c..be768e5739 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/doc.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/doc.go @@ -8,6 +8,8 @@ // // Create or open a database: // +// // The returned DB instance is safe for concurrent use. Which mean that all +// // DB's methods may be called concurrently from multiple goroutine. // db, err := leveldb.OpenFile("path/to/db", nil) // ... // defer db.Close() diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage.go b/vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage.go index e53434cab7..1189decac6 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage.go @@ -234,14 +234,30 @@ func (fs *fileStorage) SetMeta(fd FileDesc) (err error) { return } _, err = fmt.Fprintln(w, fsGenName(fd)) - // Close the file first. - if cerr := w.Close(); cerr != nil { - fs.log(fmt.Sprintf("close CURRENT.%d: %v", fd.Num, cerr)) + if err != nil { + fs.log(fmt.Sprintf("write CURRENT.%d: %v", fd.Num, err)) + return + } + if err = w.Sync(); err != nil { + fs.log(fmt.Sprintf("flush CURRENT.%d: %v", fd.Num, err)) + return + } + if err = w.Close(); err != nil { + fs.log(fmt.Sprintf("close CURRENT.%d: %v", fd.Num, err)) + return } if err != nil { return } - return rename(path, filepath.Join(fs.path, "CURRENT")) + if err = rename(path, filepath.Join(fs.path, "CURRENT")); err != nil { + fs.log(fmt.Sprintf("rename CURRENT.%d: %v", fd.Num, err)) + return + } + // Sync root directory. + if err = syncDir(fs.path); err != nil { + fs.log(fmt.Sprintf("syncDir: %v", err)) + } + return } func (fs *fileStorage) GetMeta() (fd FileDesc, err error) { diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/table/reader.go b/vendor/github.com/syndtr/goleveldb/leveldb/table/reader.go index c5be420b3f..16cfbaa006 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/table/reader.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/table/reader.go @@ -581,6 +581,7 @@ func (r *Reader) readRawBlock(bh blockHandle, verifyChecksum bool) ([]byte, erro case blockTypeSnappyCompression: decLen, err := snappy.DecodedLen(data[:bh.length]) if err != nil { + r.bpool.Put(data) return nil, r.newErrCorruptedBH(bh, err.Error()) } decData := r.bpool.Get(decLen) diff --git a/vendor/vendor.json b/vendor/vendor.json index 56c95e341f..a9de0ec728 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -334,76 +334,76 @@ "revisionTime": "2016-06-18T19:32:21Z" }, { - "checksumSHA1": "QvOrAO5S37PL/6XZVWIVGyAbn60=", + "checksumSHA1": "yHbyLpI/Meh0DGrmi8x6FrDxxUY=", "path": "github.com/syndtr/goleveldb/leveldb", - "revision": "23851d93a2292dcc56e71a18ec9e0624d84a0f65", - "revisionTime": "2016-12-27T11:05:19Z" + "revision": "b89cc31ef7977104127d34c1bd31ebd1a9db2199", + "revisionTime": "2017-07-25T06:48:36Z" }, { "checksumSHA1": "EKIow7XkgNdWvR/982ffIZxKG8Y=", "path": "github.com/syndtr/goleveldb/leveldb/cache", - "revision": "23851d93a2292dcc56e71a18ec9e0624d84a0f65", - "revisionTime": "2016-12-27T11:05:19Z" + "revision": "b89cc31ef7977104127d34c1bd31ebd1a9db2199", + "revisionTime": "2017-07-25T06:48:36Z" }, { "checksumSHA1": "5KPgnvCPlR0ysDAqo6jApzRQ3tw=", "path": "github.com/syndtr/goleveldb/leveldb/comparer", - "revision": "23851d93a2292dcc56e71a18ec9e0624d84a0f65", - "revisionTime": "2016-12-27T11:05:19Z" + "revision": "b89cc31ef7977104127d34c1bd31ebd1a9db2199", + "revisionTime": "2017-07-25T06:48:36Z" }, { "checksumSHA1": "1DRAxdlWzS4U0xKN/yQ/fdNN7f0=", "path": "github.com/syndtr/goleveldb/leveldb/errors", - "revision": "23851d93a2292dcc56e71a18ec9e0624d84a0f65", - "revisionTime": "2016-12-27T11:05:19Z" + "revision": "b89cc31ef7977104127d34c1bd31ebd1a9db2199", + "revisionTime": "2017-07-25T06:48:36Z" }, { "checksumSHA1": "eqKeD6DS7eNCtxVYZEHHRKkyZrw=", "path": "github.com/syndtr/goleveldb/leveldb/filter", - "revision": "23851d93a2292dcc56e71a18ec9e0624d84a0f65", - "revisionTime": "2016-12-27T11:05:19Z" + "revision": "b89cc31ef7977104127d34c1bd31ebd1a9db2199", + "revisionTime": "2017-07-25T06:48:36Z" }, { "checksumSHA1": "8dXuAVIsbtaMiGGuHjzGR6Ny/5c=", "path": "github.com/syndtr/goleveldb/leveldb/iterator", - "revision": "23851d93a2292dcc56e71a18ec9e0624d84a0f65", - "revisionTime": "2016-12-27T11:05:19Z" + "revision": "b89cc31ef7977104127d34c1bd31ebd1a9db2199", + "revisionTime": "2017-07-25T06:48:36Z" }, { "checksumSHA1": "gJY7bRpELtO0PJpZXgPQ2BYFJ88=", "path": "github.com/syndtr/goleveldb/leveldb/journal", - "revision": "23851d93a2292dcc56e71a18ec9e0624d84a0f65", - "revisionTime": "2016-12-27T11:05:19Z" + "revision": "b89cc31ef7977104127d34c1bd31ebd1a9db2199", + "revisionTime": "2017-07-25T06:48:36Z" }, { "checksumSHA1": "j+uaQ6DwJ50dkIdfMQu1TXdlQcY=", "path": "github.com/syndtr/goleveldb/leveldb/memdb", - "revision": "23851d93a2292dcc56e71a18ec9e0624d84a0f65", - "revisionTime": "2016-12-27T11:05:19Z" + "revision": "b89cc31ef7977104127d34c1bd31ebd1a9db2199", + "revisionTime": "2017-07-25T06:48:36Z" }, { "checksumSHA1": "UmQeotV+m8/FduKEfLOhjdp18rs=", "path": "github.com/syndtr/goleveldb/leveldb/opt", - "revision": "23851d93a2292dcc56e71a18ec9e0624d84a0f65", - "revisionTime": "2016-12-27T11:05:19Z" + "revision": "b89cc31ef7977104127d34c1bd31ebd1a9db2199", + "revisionTime": "2017-07-25T06:48:36Z" }, { - "checksumSHA1": "/Wvv9HeJTN9UUjdjwUlz7X4ioIo=", + "checksumSHA1": "tQ2AqXXAEy9icbZI9dLVdZGvWMw=", "path": "github.com/syndtr/goleveldb/leveldb/storage", - "revision": "23851d93a2292dcc56e71a18ec9e0624d84a0f65", - "revisionTime": "2016-12-27T11:05:19Z" + "revision": "b89cc31ef7977104127d34c1bd31ebd1a9db2199", + "revisionTime": "2017-07-25T06:48:36Z" }, { - "checksumSHA1": "JTJA+u8zk7EXy1UUmpFPNGvtO2A=", + "checksumSHA1": "gWFPMz8OQeul0t54RM66yMTX49g=", "path": "github.com/syndtr/goleveldb/leveldb/table", - "revision": "23851d93a2292dcc56e71a18ec9e0624d84a0f65", - "revisionTime": "2016-12-27T11:05:19Z" + "revision": "b89cc31ef7977104127d34c1bd31ebd1a9db2199", + "revisionTime": "2017-07-25T06:48:36Z" }, { "checksumSHA1": "4zil8Gwg8VPkDn1YzlgCvtukJFU=", "path": "github.com/syndtr/goleveldb/leveldb/util", - "revision": "23851d93a2292dcc56e71a18ec9e0624d84a0f65", - "revisionTime": "2016-12-27T11:05:19Z" + "revision": "b89cc31ef7977104127d34c1bd31ebd1a9db2199", + "revisionTime": "2017-07-25T06:48:36Z" }, { "checksumSHA1": "TT1rac6kpQp2vz24m5yDGUNQ/QQ=",