diff --git a/.travis.yml b/.travis.yml index 2ba6f18f26..ce6500c46a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,6 @@ matrix: script: - unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703 - brew update - - brew install caskroom/cask/brew-cask - brew cask install osxfuse - go run build/ci.go install - go run build/ci.go test -coverage $TEST_PACKAGES @@ -153,10 +152,10 @@ matrix: - export GOPATH=$HOME/go script: # Build the Android archive and upload it to Maven Central and Azure - - curl https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip -o android-ndk-r15c.zip - - unzip -q android-ndk-r15c.zip && rm android-ndk-r15c.zip - - mv android-ndk-r15c $HOME - - export ANDROID_NDK=$HOME/android-ndk-r15c + - curl https://dl.google.com/android/repository/android-ndk-r16b-linux-x86_64.zip -o android-ndk-r16b.zip + - unzip -q android-ndk-r16b.zip && rm android-ndk-r16b.zip + - mv android-ndk-r16b $HOME + - export ANDROID_NDK=$HOME/android-ndk-r16b - mkdir -p $GOPATH/src/github.com/ethereum - ln -s `pwd` $GOPATH/src/github.com/ethereum diff --git a/VERSION b/VERSION index 88d3ee7900..1790d35988 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.8.7 +1.8.8 diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 621de58685..fef06d2a8c 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" @@ -65,7 +66,7 @@ type SimulatedBackend struct { // NewSimulatedBackend creates a new binding backend using a simulated blockchain // for testing purposes. func NewSimulatedBackend(alloc core.GenesisAlloc) *SimulatedBackend { - database, _ := ethdb.NewMemDatabase() + database := ethdb.NewMemDatabase() genesis := core.Genesis{Config: params.AllEthashProtocolChanges, Alloc: alloc} genesis.MustCommit(database) blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}) @@ -159,7 +160,7 @@ func (b *SimulatedBackend) StorageAt(ctx context.Context, contract common.Addres // TransactionReceipt returns the receipt of a transaction. func (b *SimulatedBackend) TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) { - receipt, _, _, _ := core.GetReceipt(b.database, txHash) + receipt, _, _, _ := rawdb.ReadReceipt(b.database, txHash) return receipt, nil } @@ -430,11 +431,19 @@ func (fb *filterBackend) HeaderByNumber(ctx context.Context, block rpc.BlockNumb } func (fb *filterBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { - return core.GetBlockReceipts(fb.db, hash, core.GetBlockNumber(fb.db, hash)), nil + number := rawdb.ReadHeaderNumber(fb.db, hash) + if number == nil { + return nil, nil + } + return rawdb.ReadReceipts(fb.db, hash, *number), nil } func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { - receipts := core.GetBlockReceipts(fb.db, hash, core.GetBlockNumber(fb.db, hash)) + number := rawdb.ReadHeaderNumber(fb.db, hash) + if number == nil { + return nil, nil + } + receipts := rawdb.ReadReceipts(fb.db, hash, *number) if receipts == nil { return nil, nil } diff --git a/accounts/abi/event.go b/accounts/abi/event.go index 595f169f3a..a3f6be9732 100644 --- a/accounts/abi/event.go +++ b/accounts/abi/event.go @@ -33,15 +33,15 @@ type Event struct { Inputs Arguments } -func (event Event) String() string { - inputs := make([]string, len(event.Inputs)) - for i, input := range event.Inputs { +func (e Event) String() string { + inputs := make([]string, len(e.Inputs)) + for i, input := range e.Inputs { inputs[i] = fmt.Sprintf("%v %v", input.Name, input.Type) if input.Indexed { inputs[i] = fmt.Sprintf("%v indexed %v", input.Name, input.Type) } } - return fmt.Sprintf("event %v(%v)", event.Name, strings.Join(inputs, ", ")) + return fmt.Sprintf("e %v(%v)", e.Name, strings.Join(inputs, ", ")) } // Id returns the canonical representation of the event's signature used by the diff --git a/accounts/keystore/keystore_passphrase.go b/accounts/keystore/keystore_passphrase.go index eaec39f7df..da632fe345 100644 --- a/accounts/keystore/keystore_passphrase.go +++ b/accounts/keystore/keystore_passphrase.go @@ -108,9 +108,8 @@ func (ks keyStorePassphrase) StoreKey(filename string, key *Key, auth string) er func (ks keyStorePassphrase) JoinPath(filename string) string { if filepath.IsAbs(filename) { return filename - } else { - return filepath.Join(ks.keysDirPath, filename) } + return filepath.Join(ks.keysDirPath, filename) } // EncryptKey encrypts a key using the specified scrypt parameters into a json diff --git a/accounts/keystore/keystore_plain.go b/accounts/keystore/keystore_plain.go index b490ca72b8..f62a133ce1 100644 --- a/accounts/keystore/keystore_plain.go +++ b/accounts/keystore/keystore_plain.go @@ -56,7 +56,6 @@ func (ks keyStorePlain) StoreKey(filename string, key *Key, auth string) error { func (ks keyStorePlain) JoinPath(filename string) string { if filepath.IsAbs(filename) { return filename - } else { - return filepath.Join(ks.keysDirPath, filename) } + return filepath.Join(ks.keysDirPath, filename) } diff --git a/bmt/bmt.go b/bmt/bmt.go index 3408758675..c290223452 100644 --- a/bmt/bmt.go +++ b/bmt/bmt.go @@ -150,29 +150,29 @@ func NewTreePool(hasher BaseHasher, segmentCount, capacity int) *TreePool { } // Drain drains the pool until it has no more than n resources -func (self *TreePool) Drain(n int) { - self.lock.Lock() - defer self.lock.Unlock() - for len(self.c) > n { - <-self.c - self.count-- +func (p *TreePool) Drain(n int) { + p.lock.Lock() + defer p.lock.Unlock() + for len(p.c) > n { + <-p.c + p.count-- } } // Reserve is blocking until it returns an available Tree // it reuses free Trees or creates a new one if size is not reached -func (self *TreePool) Reserve() *Tree { - self.lock.Lock() - defer self.lock.Unlock() +func (p *TreePool) Reserve() *Tree { + p.lock.Lock() + defer p.lock.Unlock() var t *Tree - if self.count == self.Capacity { - return <-self.c + if p.count == p.Capacity { + return <-p.c } select { - case t = <-self.c: + case t = <-p.c: default: - t = NewTree(self.hasher, self.SegmentSize, self.SegmentCount) - self.count++ + t = NewTree(p.hasher, p.SegmentSize, p.SegmentCount) + p.count++ } return t } @@ -180,8 +180,8 @@ func (self *TreePool) Reserve() *Tree { // Release gives back a Tree to the pool. // This Tree is guaranteed to be in reusable state // does not need locking -func (self *TreePool) Release(t *Tree) { - self.c <- t // can never fail but... +func (p *TreePool) Release(t *Tree) { + p.c <- t // can never fail but... } // Tree is a reusable control structure representing a BMT @@ -193,17 +193,17 @@ type Tree struct { } // Draw draws the BMT (badly) -func (self *Tree) Draw(hash []byte, d int) string { +func (t *Tree) Draw(hash []byte, d int) string { var left, right []string var anc []*Node - for i, n := range self.leaves { + for i, n := range t.leaves { left = append(left, fmt.Sprintf("%v", hashstr(n.left))) if i%2 == 0 { anc = append(anc, n.parent) } right = append(right, fmt.Sprintf("%v", hashstr(n.right))) } - anc = self.leaves + anc = t.leaves var hashes [][]string for l := 0; len(anc) > 0; l++ { var nodes []*Node @@ -277,42 +277,42 @@ func NewTree(hasher BaseHasher, segmentSize, segmentCount int) *Tree { // methods needed by hash.Hash // Size returns the size -func (self *Hasher) Size() int { - return self.size +func (h *Hasher) Size() int { + return h.size } // BlockSize returns the block size -func (self *Hasher) BlockSize() int { - return self.blocksize +func (h *Hasher) BlockSize() int { + return h.blocksize } // Sum returns the hash of the buffer // hash.Hash interface Sum method appends the byte slice to the underlying // data before it calculates and returns the hash of the chunk -func (self *Hasher) Sum(b []byte) (r []byte) { - t := self.bmt - i := self.cur +func (h *Hasher) Sum(b []byte) (r []byte) { + t := h.bmt + i := h.cur n := t.leaves[i] j := i // must run strictly before all nodes calculate // datanodes are guaranteed to have a parent - if len(self.segment) > self.size && i > 0 && n.parent != nil { + if len(h.segment) > h.size && i > 0 && n.parent != nil { n = n.parent } else { i *= 2 } - d := self.finalise(n, i) - self.writeSegment(j, self.segment, d) - c := <-self.result - self.releaseTree() + d := h.finalise(n, i) + h.writeSegment(j, h.segment, d) + c := <-h.result + h.releaseTree() // sha3(length + BMT(pure_chunk)) - if self.blockLength == nil { + if h.blockLength == nil { return c } - res := self.pool.hasher() + res := h.pool.hasher() res.Reset() - res.Write(self.blockLength) + res.Write(h.blockLength) res.Write(c) return res.Sum(nil) } @@ -321,8 +321,8 @@ func (self *Hasher) Sum(b []byte) (r []byte) { // Hash waits for the hasher result and returns it // caller must call this on a BMT Hasher being written to -func (self *Hasher) Hash() []byte { - return <-self.result +func (h *Hasher) Hash() []byte { + return <-h.result } // Hasher implements the io.Writer interface @@ -330,16 +330,16 @@ func (self *Hasher) Hash() []byte { // Write fills the buffer to hash // with every full segment complete launches a hasher go routine // that shoots up the BMT -func (self *Hasher) Write(b []byte) (int, error) { +func (h *Hasher) Write(b []byte) (int, error) { l := len(b) if l <= 0 { return 0, nil } - s := self.segment - i := self.cur - count := (self.count + 1) / 2 - need := self.count*self.size - self.cur*2*self.size - size := self.size + s := h.segment + i := h.cur + count := (h.count + 1) / 2 + need := h.count*h.size - h.cur*2*h.size + size := h.size if need > size { size *= 2 } @@ -356,7 +356,7 @@ func (self *Hasher) Write(b []byte) (int, error) { // read full segments and the last possibly partial segment for need > 0 && i < count-1 { // push all finished chunks we read - self.writeSegment(i, s, self.depth) + h.writeSegment(i, s, h.depth) need -= size if need < 0 { size += need @@ -365,8 +365,8 @@ func (self *Hasher) Write(b []byte) (int, error) { rest += size i++ } - self.segment = s - self.cur = i + h.segment = s + h.cur = i // otherwise, we can assume len(s) == 0, so all buffer is read and chunk is not yet full return l, nil } @@ -376,8 +376,8 @@ func (self *Hasher) Write(b []byte) (int, error) { // ReadFrom reads from io.Reader and appends to the data to hash using Write // it reads so that chunk to hash is maximum length or reader reaches EOF // caller must Reset the hasher prior to call -func (self *Hasher) ReadFrom(r io.Reader) (m int64, err error) { - bufsize := self.size*self.count - self.size*self.cur - len(self.segment) +func (h *Hasher) ReadFrom(r io.Reader) (m int64, err error) { + bufsize := h.size*h.count - h.size*h.cur - len(h.segment) buf := make([]byte, bufsize) var read int for { @@ -385,7 +385,7 @@ func (self *Hasher) ReadFrom(r io.Reader) (m int64, err error) { n, err = r.Read(buf) read += n if err == io.EOF || read == len(buf) { - hash := self.Sum(buf[:n]) + hash := h.Sum(buf[:n]) if read == len(buf) { err = NewEOC(hash) } @@ -394,7 +394,7 @@ func (self *Hasher) ReadFrom(r io.Reader) (m int64, err error) { if err != nil { break } - n, err = self.Write(buf[:n]) + n, err = h.Write(buf[:n]) if err != nil { break } @@ -403,9 +403,9 @@ func (self *Hasher) ReadFrom(r io.Reader) (m int64, err error) { } // Reset needs to be called before writing to the hasher -func (self *Hasher) Reset() { - self.getTree() - self.blockLength = nil +func (h *Hasher) Reset() { + h.getTree() + h.blockLength = nil } // Hasher implements the SwarmHash interface @@ -413,52 +413,52 @@ func (self *Hasher) Reset() { // ResetWithLength needs to be called before writing to the hasher // the argument is supposed to be the byte slice binary representation of // the length of the data subsumed under the hash -func (self *Hasher) ResetWithLength(l []byte) { - self.Reset() - self.blockLength = l +func (h *Hasher) ResetWithLength(l []byte) { + h.Reset() + h.blockLength = l } // Release gives back the Tree to the pool whereby it unlocks // it resets tree, segment and index -func (self *Hasher) releaseTree() { - if self.bmt != nil { - n := self.bmt.leaves[self.cur] +func (h *Hasher) releaseTree() { + if h.bmt != nil { + n := h.bmt.leaves[h.cur] for ; n != nil; n = n.parent { n.unbalanced = false if n.parent != nil { n.root = false } } - self.pool.Release(self.bmt) - self.bmt = nil + h.pool.Release(h.bmt) + h.bmt = nil } - self.cur = 0 - self.segment = nil + h.cur = 0 + h.segment = nil } -func (self *Hasher) writeSegment(i int, s []byte, d int) { - h := self.pool.hasher() - n := self.bmt.leaves[i] +func (h *Hasher) writeSegment(i int, s []byte, d int) { + hash := h.pool.hasher() + n := h.bmt.leaves[i] - if len(s) > self.size && n.parent != nil { + if len(s) > h.size && n.parent != nil { go func() { - h.Reset() - h.Write(s) - s = h.Sum(nil) + hash.Reset() + hash.Write(s) + s = hash.Sum(nil) if n.root { - self.result <- s + h.result <- s return } - self.run(n.parent, h, d, n.index, s) + h.run(n.parent, hash, d, n.index, s) }() return } - go self.run(n, h, d, i*2, s) + go h.run(n, hash, d, i*2, s) } -func (self *Hasher) run(n *Node, h hash.Hash, d int, i int, s []byte) { +func (h *Hasher) run(n *Node, hash hash.Hash, d int, i int, s []byte) { isLeft := i%2 == 0 for { if isLeft { @@ -470,18 +470,18 @@ func (self *Hasher) run(n *Node, h hash.Hash, d int, i int, s []byte) { return } if !n.unbalanced || !isLeft || i == 0 && d == 0 { - h.Reset() - h.Write(n.left) - h.Write(n.right) - s = h.Sum(nil) + hash.Reset() + hash.Write(n.left) + hash.Write(n.right) + s = hash.Sum(nil) } else { s = append(n.left, n.right...) } - self.hash = s + h.hash = s if n.root { - self.result <- s + h.result <- s return } @@ -492,20 +492,20 @@ func (self *Hasher) run(n *Node, h hash.Hash, d int, i int, s []byte) { } // getTree obtains a BMT resource by reserving one from the pool -func (self *Hasher) getTree() *Tree { - if self.bmt != nil { - return self.bmt +func (h *Hasher) getTree() *Tree { + if h.bmt != nil { + return h.bmt } - t := self.pool.Reserve() - self.bmt = t + t := h.pool.Reserve() + h.bmt = t return t } // atomic bool toggle implementing a concurrent reusable 2-state object // atomic addint with %2 implements atomic bool toggle // it returns true if the toggler just put it in the active/waiting state -func (self *Node) toggle() bool { - return atomic.AddInt32(&self.state, 1)%2 == 1 +func (n *Node) toggle() bool { + return atomic.AddInt32(&n.state, 1)%2 == 1 } func hashstr(b []byte) string { @@ -525,7 +525,7 @@ func depth(n int) (d int) { // finalise is following the zigzags on the tree belonging // to the final datasegment -func (self *Hasher) finalise(n *Node, i int) (d int) { +func (h *Hasher) finalise(n *Node, i int) (d int) { isLeft := i%2 == 0 for { // when the final segment's path is going via left segments @@ -550,8 +550,8 @@ type EOC struct { } // Error returns the error string -func (self *EOC) Error() string { - return fmt.Sprintf("hasher limit reached, chunk hash: %x", self.Hash) +func (e *EOC) Error() string { + return fmt.Sprintf("hasher limit reached, chunk hash: %x", e.Hash) } // NewEOC creates new end of chunk error with the hash diff --git a/build/ci.go b/build/ci.go index 5fce3c8e60..79dcc146c3 100644 --- a/build/ci.go +++ b/build/ci.go @@ -731,7 +731,7 @@ func doAndroidArchive(cmdline []string) { // Build the Android archive and Maven resources build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind")) build.MustRun(gomobileTool("init", "--ndk", os.Getenv("ANDROID_NDK"))) - build.MustRun(gomobileTool("bind", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/ethereum/go-ethereum/mobile")) + build.MustRun(gomobileTool("bind", "-ldflags", "-s -w", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/ethereum/go-ethereum/mobile")) if *local { // If we're building locally, copy bundle to build dir and skip Maven @@ -755,14 +755,18 @@ func doAndroidArchive(cmdline []string) { os.Rename(archive, meta.Package+".aar") if *signer != "" && *deploy != "" { // Import the signing key into the local GPG instance - if b64key := os.Getenv(*signer); b64key != "" { - key, err := base64.StdEncoding.DecodeString(b64key) - if err != nil { - log.Fatalf("invalid base64 %s", *signer) - } - gpg := exec.Command("gpg", "--import") - gpg.Stdin = bytes.NewReader(key) - build.MustRun(gpg) + b64key := os.Getenv(*signer) + key, err := base64.StdEncoding.DecodeString(b64key) + if err != nil { + log.Fatalf("invalid base64 %s", *signer) + } + gpg := exec.Command("gpg", "--import") + gpg.Stdin = bytes.NewReader(key) + build.MustRun(gpg) + + keyID, err := build.PGPKeyID(string(key)) + if err != nil { + log.Fatal(err) } // Upload the artifacts to Sonatype and/or Maven Central repo := *deploy + "/service/local/staging/deploy/maven2" @@ -771,6 +775,7 @@ func doAndroidArchive(cmdline []string) { } build.MustRunCommand("mvn", "gpg:sign-and-deploy-file", "-e", "-X", "-settings=build/mvn.settings", "-Durl="+repo, "-DrepositoryId=ossrh", + "-Dgpg.keyname="+keyID, "-DpomFile="+meta.Package+".pom", "-Dfile="+meta.Package+".aar") } } @@ -852,7 +857,7 @@ func doXCodeFramework(cmdline []string) { // Build the iOS XCode framework build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind")) build.MustRun(gomobileTool("init")) - bind := gomobileTool("bind", "--target", "ios", "--tags", "ios", "-v", "github.com/ethereum/go-ethereum/mobile") + bind := gomobileTool("bind", "-ldflags", "-s -w", "--target", "ios", "--tags", "ios", "-v", "github.com/ethereum/go-ethereum/mobile") if *local { // If we're building locally, use the build folder and stop afterwards diff --git a/cmd/clef/README.md b/cmd/clef/README.md index 93799a7610..027c22c98f 100644 --- a/cmd/clef/README.md +++ b/cmd/clef/README.md @@ -12,6 +12,11 @@ synchronised with the chain or a particular Ethereum node that has no built-in ( Clef can run as a daemon on the same machine, or off a usb-stick like [usb armory](https://inversepath.com/usbarmory), or a separate VM in a [QubesOS](https://www.qubes-os.org/) type os setup. +Check out + +* the [tutorial](tutorial.md) for some concrete examples on how the signer works. +* the [setup docs](docs/setup.md) for some information on how to configure it to work on QubesOS or USBArmory. + ## Command line flags Clef accepts the following command line options: @@ -49,7 +54,6 @@ Example: signer -keystore /my/keystore -chainid 4 ``` -Check out the [tutorial](tutorial.md) for some concrete examples on how the signer works. ## Security model @@ -862,3 +866,12 @@ A UI should conform to the following rules. along with the UI. +### UI Implementations + +There are a couple of implementation for a UI. We'll try to keep this list up to date. + +| Name | Repo | UI type| No external resources| Blocky support| Verifies permissions | Hash information | No secondary storage | Statically linked| Can modify parameters| +| ---- | ---- | -------| ---- | ---- | ---- |---- | ---- | ---- | ---- | +| QtSigner| https://github.com/holiman/qtsigner/| Python3/QT-based| :+1:| :+1:| :+1:| :+1:| :+1:| :x: | :+1: (partially)| +| GtkSigner| https://github.com/holiman/gtksigner| Python3/GTK-based| :+1:| :x:| :x:| :+1:| :+1:| :x: | :x: | +| Frame | https://github.com/floating/frame/commits/go-signer| Electron-based| :x:| :x:| :x:| :x:| ?| :x: | :x: | diff --git a/cmd/clef/docs/qubes/clef_qubes_http.png b/cmd/clef/docs/qubes/clef_qubes_http.png new file mode 100644 index 0000000000..a641e1987f Binary files /dev/null and b/cmd/clef/docs/qubes/clef_qubes_http.png differ diff --git a/cmd/clef/docs/qubes/clef_qubes_qrexec.png b/cmd/clef/docs/qubes/clef_qubes_qrexec.png new file mode 100644 index 0000000000..f57fc8933e Binary files /dev/null and b/cmd/clef/docs/qubes/clef_qubes_qrexec.png differ diff --git a/cmd/clef/docs/qubes/qrexec-example.png b/cmd/clef/docs/qubes/qrexec-example.png new file mode 100644 index 0000000000..0d86fde19d Binary files /dev/null and b/cmd/clef/docs/qubes/qrexec-example.png differ diff --git a/cmd/clef/docs/qubes/qubes-client.py b/cmd/clef/docs/qubes/qubes-client.py new file mode 100644 index 0000000000..93a74b899b --- /dev/null +++ b/cmd/clef/docs/qubes/qubes-client.py @@ -0,0 +1,23 @@ +""" +This implements a dispatcher which listens to localhost:8550, and proxies +requests via qrexec to the service qubes.EthSign on a target domain +""" + +import http.server +import socketserver,subprocess + +PORT=8550 +TARGET_DOMAIN= 'debian-work' + +class Dispatcher(http.server.BaseHTTPRequestHandler): + def do_POST(self): + post_data = self.rfile.read(int(self.headers['Content-Length'])) + p = subprocess.Popen(['/usr/bin/qrexec-client-vm',TARGET_DOMAIN,'qubes.Clefsign'],stdin=subprocess.PIPE, stdout=subprocess.PIPE) + output = p.communicate(post_data)[0] + self.wfile.write(output) + + +with socketserver.TCPServer(("",PORT), Dispatcher) as httpd: + print("Serving at port", PORT) + httpd.serve_forever() + diff --git a/cmd/clef/docs/qubes/qubes.Clefsign b/cmd/clef/docs/qubes/qubes.Clefsign new file mode 100644 index 0000000000..9b5af7b4fe --- /dev/null +++ b/cmd/clef/docs/qubes/qubes.Clefsign @@ -0,0 +1,16 @@ +#!/bin/bash + +SIGNER_BIN="/home/user/tools/clef/clef" +SIGNER_CMD="/home/user/tools/gtksigner/gtkui.py -s $SIGNER_BIN" + +# Start clef if not already started +if [ ! -S /home/user/.clef/clef.ipc ]; then + $SIGNER_CMD & + sleep 1 +fi + +# Should be started by now +if [ -S /home/user/.clef/clef.ipc ]; then + # Post incoming request to HTTP channel + curl -H "Content-Type: application/json" -X POST -d @- http://localhost:8550 2>/dev/null +fi diff --git a/cmd/clef/docs/qubes/qubes_newaccount-1.png b/cmd/clef/docs/qubes/qubes_newaccount-1.png new file mode 100644 index 0000000000..598dbbee7a Binary files /dev/null and b/cmd/clef/docs/qubes/qubes_newaccount-1.png differ diff --git a/cmd/clef/docs/qubes/qubes_newaccount-2.png b/cmd/clef/docs/qubes/qubes_newaccount-2.png new file mode 100644 index 0000000000..cd762a1934 Binary files /dev/null and b/cmd/clef/docs/qubes/qubes_newaccount-2.png differ diff --git a/cmd/clef/docs/setup.md b/cmd/clef/docs/setup.md new file mode 100644 index 0000000000..33d2b0381f --- /dev/null +++ b/cmd/clef/docs/setup.md @@ -0,0 +1,198 @@ +# Setting up Clef + +This document describes how Clef can be used in a more secure manner than executing it from your everyday laptop, +in order to ensure that the keys remain safe in the event that your computer should get compromised. + +## Qubes OS + + +### Background + +The Qubes operating system is based around virtual machines (qubes), where a set of virtual machines are configured, typically for +different purposes such as: + +- personal + - Your personal email, browsing etc +- work + - Work email etc +- vault + - a VM without network access, where gpg-keys and/or keepass credentials are stored. + +A couple of dedicated virtual machines handle externalities: + +- sys-net provides networking to all other (network-enabled) machines +- sys-firewall handles firewall rules +- sys-usb handles USB devices, and can map usb-devices to certain qubes. + +The goal of this document is to describe how we can set up clef to provide secure transaction +signing from a `vault` vm, to another networked qube which runs Dapps. + +### Setup + +There are two ways that this can be achieved: integrated via Qubes or integrated via networking. + + +#### 1. Qubes Integrated + +Qubes provdes a facility for inter-qubes communication via `qrexec`. A qube can request to make a cross-qube RPC request +to another qube. The OS then asks the user if the call is permitted. + +![Example](qubes/qrexec-example.png) + +A policy-file can be created to allow such interaction. On the `target` domain, a service is invoked which can read the +`stdin` from the `client` qube. + +This is how [Split GPG](https://www.qubes-os.org/doc/split-gpg/) is implemented. We can set up Clef the same way: + +##### Server + +![Clef via qrexec](qubes/clef_qubes_qrexec.png) + +On the `target` qubes, we need to define the rpc service. + +[qubes.Clefsign](qubes/qubes.Clefsign): + +```bash +#!/bin/bash + +SIGNER_BIN="/home/user/tools/clef/clef" +SIGNER_CMD="/home/user/tools/gtksigner/gtkui.py -s $SIGNER_BIN" + +# Start clef if not already started +if [ ! -S /home/user/.clef/clef.ipc ]; then + $SIGNER_CMD & + sleep 1 +fi + +# Should be started by now +if [ -S /home/user/.clef/clef.ipc ]; then + # Post incoming request to HTTP channel + curl -H "Content-Type: application/json" -X POST -d @- http://localhost:8550 2>/dev/null +fi + +``` +This RPC service is not complete (see notes about HTTP headers below), but works as a proof-of-concept. +It will forward the data received on `stdin` (forwarded by the OS) to Clef's HTTP channel. + +It would have been possible to send data directly to the `/home/user/.clef/.clef.ipc` +socket via e.g `nc -U /home/user/.clef/clef.ipc`, but the reason for sending the request +data over `HTTP` instead of `IPC` is that we want the ability to forward `HTTP` headers. + +To enable the service: + +``` bash +sudo cp qubes.Clefsign /etc/qubes-rpc/ +sudo chmod +x /etc/qubes-rpc/ qubes.Clefsign +``` + +This setup uses [gtksigner](https://github.com/holiman/gtksigner), which is a very minimal GTK-based UI that works well +with minimal requirements. + +##### Client + + +On the `client` qube, we need to create a listener which will receive the request from the Dapp, and proxy it. + + +[qubes-client.py](qubes/client/qubes-client.py): + +```python + +""" +This implements a dispatcher which listens to localhost:8550, and proxies +requests via qrexec to the service qubes.EthSign on a target domain +""" + +import http.server +import socketserver,subprocess + +PORT=8550 +TARGET_DOMAIN= 'debian-work' + +class Dispatcher(http.server.BaseHTTPRequestHandler): + def do_POST(self): + post_data = self.rfile.read(int(self.headers['Content-Length'])) + p = subprocess.Popen(['/usr/bin/qrexec-client-vm',TARGET_DOMAIN,'qubes.Clefsign'],stdin=subprocess.PIPE, stdout=subprocess.PIPE) + output = p.communicate(post_data)[0] + self.wfile.write(output) + + +with socketserver.TCPServer(("",PORT), Dispatcher) as httpd: + print("Serving at port", PORT) + httpd.serve_forever() + + +``` + +#### Testing + +To test the flow, if we have set up `debian-work` as the `target`, we can do + +```bash +$ cat newaccnt.json +{ "id": 0, "jsonrpc": "2.0","method": "account_new","params": []} + +$ cat newaccnt.json| qrexec-client-vm debian-work qubes.Clefsign +``` + +This should pop up first a dialog to allow the IPC call: + +![one](qubes/qubes_newaccount-1.png) + +Followed by a GTK-dialog to approve the operation + +![two](qubes/qubes_newaccount-2.png) + +To test the full flow, we use the client wrapper. Start it on the `client` qube: +``` +[user@work qubes]$ python3 qubes-client.py +``` + +Make the request over http (`client` qube): +``` +[user@work clef]$ cat newaccnt.json | curl -X POST -d @- http://localhost:8550 +``` +And it should show the same popups again. + +##### Pros and cons + +The benefits of this setup are: + +- This is the qubes-os intended model for inter-qube communication, +- and thus benefits from qubes-os dialogs and policies for user approval + +However, it comes with a couple of drawbacks: + +- The `qubes-gpg-client` must forward the http request via RPC to the `target` qube. When doing so, the proxy + will either drop important headers, or replace them. + - The `Host` header is most likely `localhost` + - The `Origin` header must be forwarded + - Information about the remote ip must be added as a `X-Forwarded-For`. However, Clef cannot always trust an `XFF` header, + since malicious clients may lie about `XFF` in order to fool the http server into believing it comes from another address. +- Even with a policy in place to allow rpc-calls between `caller` and `target`, there will be several popups: + - One qubes-specific where the user specifies the `target` vm + - One clef-specific to approve the transaction + + +#### 2. Network integrated + +The second way to set up Clef on a qubes system is to allow networking, and have Clef listen to a port which is accessible +form other qubes. + +![Clef via http](qubes/clef_qubes_http.png) + + + + +## USBArmory + +The [USB armory](https://inversepath.com/usbarmory) is an open source hardware design with an 800 Mhz ARM processor. It is a pocket-size +computer. When inserted into a laptop, it identifies itself as a USB network interface, basically adding another network +to your computer. Over this new network interface, you can SSH into the device. + +Running Clef off a USB armory means that you can use the armory as a very versatile offline computer, which only +ever connects to a local network between your computer and the device itself. + +Needless to say, the while this model should be fairly secure against remote attacks, an attacker with physical access +to the USB Armory would trivially be able to extract the contents of the device filesystem. + diff --git a/cmd/clef/main.go b/cmd/clef/main.go index 9a1ae91acb..348bcb22f6 100644 --- a/cmd/clef/main.go +++ b/cmd/clef/main.go @@ -47,11 +47,11 @@ import ( "gopkg.in/urfave/cli.v1" ) -// ExternalApiVersion -- see extapi_changelog.md -const ExternalApiVersion = "2.0.0" +// ExternalAPIVersion -- see extapi_changelog.md +const ExternalAPIVersion = "2.0.0" -// InternalApiVersion -- see intapi_changelog.md -const InternalApiVersion = "2.0.0" +// InternalAPIVersion -- see intapi_changelog.md +const InternalAPIVersion = "2.0.0" const legalWarning = ` WARNING! @@ -398,10 +398,10 @@ func signer(c *cli.Context) error { } // register signer API with server var ( - extapiUrl = "n/a" - ipcApiUrl = "n/a" + extapiURL = "n/a" + ipcapiURL = "n/a" ) - rpcApi := []rpc.API{ + rpcAPI := []rpc.API{ { Namespace: "account", Public: true, @@ -415,12 +415,12 @@ func signer(c *cli.Context) error { // start http server httpEndpoint := fmt.Sprintf("%s:%d", c.String(utils.RPCListenAddrFlag.Name), c.Int(rpcPortFlag.Name)) - listener, _, err := rpc.StartHTTPEndpoint(httpEndpoint, rpcApi, []string{"account"}, cors, vhosts) + listener, _, err := rpc.StartHTTPEndpoint(httpEndpoint, rpcAPI, []string{"account"}, cors, vhosts) if err != nil { utils.Fatalf("Could not start RPC api: %v", err) } - extapiUrl = fmt.Sprintf("http://%s", httpEndpoint) - log.Info("HTTP endpoint opened", "url", extapiUrl) + extapiURL = fmt.Sprintf("http://%s", httpEndpoint) + log.Info("HTTP endpoint opened", "url", extapiURL) defer func() { listener.Close() @@ -430,19 +430,19 @@ func signer(c *cli.Context) error { } if !c.Bool(utils.IPCDisabledFlag.Name) { if c.IsSet(utils.IPCPathFlag.Name) { - ipcApiUrl = c.String(utils.IPCPathFlag.Name) + ipcapiURL = c.String(utils.IPCPathFlag.Name) } else { - ipcApiUrl = filepath.Join(configDir, "clef.ipc") + ipcapiURL = filepath.Join(configDir, "clef.ipc") } - listener, _, err := rpc.StartIPCEndpoint(ipcApiUrl, rpcApi) + listener, _, err := rpc.StartIPCEndpoint(ipcapiURL, rpcAPI) if err != nil { utils.Fatalf("Could not start IPC api: %v", err) } - log.Info("IPC endpoint opened", "url", ipcApiUrl) + log.Info("IPC endpoint opened", "url", ipcapiURL) defer func() { listener.Close() - log.Info("IPC endpoint closed", "url", ipcApiUrl) + log.Info("IPC endpoint closed", "url", ipcapiURL) }() } @@ -453,10 +453,10 @@ func signer(c *cli.Context) error { } ui.OnSignerStartup(core.StartupInfo{ Info: map[string]interface{}{ - "extapi_version": ExternalApiVersion, - "intapi_version": InternalApiVersion, - "extapi_http": extapiUrl, - "extapi_ipc": ipcApiUrl, + "extapi_version": ExternalAPIVersion, + "intapi_version": InternalAPIVersion, + "extapi_http": extapiURL, + "extapi_ipc": ipcapiURL, }, }) diff --git a/cmd/evm/json_logger.go b/cmd/evm/json_logger.go index 0e7a911896..f16424fbe6 100644 --- a/cmd/evm/json_logger.go +++ b/cmd/evm/json_logger.go @@ -32,6 +32,8 @@ type JSONLogger struct { cfg *vm.LogConfig } +// NewJSONLogger creates a new EVM tracer that prints execution steps as JSON objects +// into the provided stream. func NewJSONLogger(cfg *vm.LogConfig, writer io.Writer) *JSONLogger { return &JSONLogger{json.NewEncoder(writer), cfg} } diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index 2d9d31fb08..7138a9ddd4 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -21,12 +21,12 @@ import ( "encoding/json" "fmt" "io/ioutil" + "math/big" "os" + goruntime "runtime" "runtime/pprof" "time" - goruntime "runtime" - "github.com/ethereum/go-ethereum/cmd/evm/internal/compiler" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" @@ -86,6 +86,7 @@ func runCmd(ctx *cli.Context) error { chainConfig *params.ChainConfig sender = common.BytesToAddress([]byte("sender")) receiver = common.BytesToAddress([]byte("receiver")) + blockNumber uint64 ) if ctx.GlobalBool(MachineFlag.Name) { tracer = NewJSONLogger(logconfig, os.Stdout) @@ -97,13 +98,13 @@ func runCmd(ctx *cli.Context) error { } if ctx.GlobalString(GenesisFlag.Name) != "" { gen := readGenesis(ctx.GlobalString(GenesisFlag.Name)) - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() genesis := gen.ToBlock(db) statedb, _ = state.New(genesis.Root(), state.NewDatabase(db)) chainConfig = gen.Config + blockNumber = gen.Number } else { - db, _ := ethdb.NewMemDatabase() - statedb, _ = state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) } if ctx.GlobalString(SenderFlag.Name) != "" { sender = common.HexToAddress(ctx.GlobalString(SenderFlag.Name)) @@ -156,11 +157,12 @@ func runCmd(ctx *cli.Context) error { initialGas := ctx.GlobalUint64(GasFlag.Name) runtimeConfig := runtime.Config{ - Origin: sender, - State: statedb, - GasLimit: initialGas, - GasPrice: utils.GlobalBig(ctx, PriceFlag.Name), - Value: utils.GlobalBig(ctx, ValueFlag.Name), + Origin: sender, + State: statedb, + GasLimit: initialGas, + GasPrice: utils.GlobalBig(ctx, PriceFlag.Name), + Value: utils.GlobalBig(ctx, ValueFlag.Name), + BlockNumber: new(big.Int).SetUint64(blockNumber), EVMConfig: vm.Config{ Tracer: tracer, Debug: ctx.GlobalBool(DebugFlag.Name) || ctx.GlobalBool(MachineFlag.Name), diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index 071ea94ad0..6d5ff069f6 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -38,6 +38,8 @@ var stateTestCommand = cli.Command{ ArgsUsage: "", } +// StatetestResult contains the execution status after running a state test, any +// error that might have occurred and a dump of the final state if requested. type StatetestResult struct { Name string `json:"name"` Pass bool `json:"pass"` diff --git a/cmd/geth/accountcmd.go b/cmd/geth/accountcmd.go index 0db5c4ce0f..071be85397 100644 --- a/cmd/geth/accountcmd.go +++ b/cmd/geth/accountcmd.go @@ -340,7 +340,7 @@ func importWallet(ctx *cli.Context) error { if len(keyfile) == 0 { utils.Fatalf("keyfile must be given as argument") } - keyJson, err := ioutil.ReadFile(keyfile) + keyJSON, err := ioutil.ReadFile(keyfile) if err != nil { utils.Fatalf("Could not read wallet file: %v", err) } @@ -349,7 +349,7 @@ func importWallet(ctx *cli.Context) error { passphrase := getPassPhrase("", false, 0, utils.MakePasswordList(ctx)) ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore) - acct, err := ks.ImportPreSaleKey(keyJson, passphrase) + acct, err := ks.ImportPreSaleKey(keyJSON, passphrase) if err != nil { utils.Fatalf("%v", err) } diff --git a/cmd/geth/bugcmd.go b/cmd/geth/bugcmd.go index 51187ac90e..7e9a8ccc70 100644 --- a/cmd/geth/bugcmd.go +++ b/cmd/geth/bugcmd.go @@ -41,7 +41,7 @@ var bugCommand = cli.Command{ Category: "MISCELLANEOUS COMMANDS", } -const issueUrl = "https://github.com/ethereum/go-ethereum/issues/new" +const issueURL = "https://github.com/ethereum/go-ethereum/issues/new" // reportBug reports a bug by opening a new URL to the go-ethereum GH issue // tracker and setting default values as the issue body. @@ -58,8 +58,8 @@ func reportBug(ctx *cli.Context) error { fmt.Fprintln(&buff, header) // open a new GH issue - if !browser.Open(issueUrl + "?body=" + url.QueryEscape(buff.String())) { - fmt.Printf("Please file a new issue at %s using this template:\n\n%s", issueUrl, buff.String()) + if !browser.Open(issueURL + "?body=" + url.QueryEscape(buff.String())) { + fmt.Printf("Please file a new issue at %s using this template:\n\n%s", issueURL, buff.String()) } return nil } diff --git a/cmd/geth/dao_test.go b/cmd/geth/dao_test.go index a8dbc51630..52983ff2af 100644 --- a/cmd/geth/dao_test.go +++ b/cmd/geth/dao_test.go @@ -24,7 +24,7 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" ) @@ -131,8 +131,8 @@ func testDAOForkBlockNewChain(t *testing.T, test int, genesis string, expectBloc if genesis != "" { genesisHash = daoGenesisHash } - config, err := core.GetChainConfig(db, genesisHash) - if err != nil { + config := rawdb.ReadChainConfig(db, genesisHash) + if config == nil { t.Errorf("test %d: failed to retrieve chain config: %v", test, err) return // we want to return here, the other checks can't make it past this point (nil panic). } diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index c0af4c13e7..58d72f32ba 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" @@ -271,15 +272,13 @@ func ImportPreimages(db *ethdb.LDBDatabase, fn string) error { // Accumulate the preimages and flush when enough ws gathered preimages[crypto.Keccak256Hash(blob)] = common.CopyBytes(blob) if len(preimages) > 1024 { - if err := core.WritePreimages(db, 0, preimages); err != nil { - return err - } + rawdb.WritePreimages(db, 0, preimages) preimages = make(map[common.Hash][]byte) } } // Flush the last batch preimage data if len(preimages) > 0 { - return core.WritePreimages(db, 0, preimages) + rawdb.WritePreimages(db, 0, preimages) } return nil } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index ba8378ef2d..ef5f6a9f08 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -158,11 +158,11 @@ var ( } FastSyncFlag = cli.BoolFlag{ Name: "fast", - Usage: "Enable fast syncing through state downloads", + Usage: "Enable fast syncing through state downloads (replaced by --syncmode)", } LightModeFlag = cli.BoolFlag{ Name: "light", - Usage: "Enable light client mode", + Usage: "Enable light client mode (replaced by --syncmode)", } defaultSyncMode = eth.DefaultConfig.SyncMode SyncModeFlag = TextMarshalerFlag{ diff --git a/cmd/wnode/main.go b/cmd/wnode/main.go index 988c50ce3d..5031a088cb 100644 --- a/cmd/wnode/main.go +++ b/cmd/wnode/main.go @@ -271,7 +271,9 @@ func initialize() { if *mailServerMode { shh.RegisterServer(&mailServer) - mailServer.Init(shh, *argDBPath, msPassword, *argServerPoW) + if err := mailServer.Init(shh, *argDBPath, msPassword, *argServerPoW); err != nil { + utils.Fatalf("Failed to init MailServer: %s", err) + } } server = &p2p.Server{ diff --git a/common/bytes.go b/common/bytes.go index ba00e8a4b2..e2adc80059 100644 --- a/common/bytes.go +++ b/common/bytes.go @@ -87,15 +87,13 @@ func Hex2BytesFixed(str string, flen int) []byte { h, _ := hex.DecodeString(str) if len(h) == flen { return h - } else { - if len(h) > flen { - return h[len(h)-flen:] - } else { - hh := make([]byte, flen) - copy(hh[flen-len(h):flen], h[:]) - return hh - } } + if len(h) > flen { + return h[len(h)-flen:] + } + hh := make([]byte, flen) + copy(hh[flen-len(h):flen], h[:]) + return hh } func RightPadBytes(slice []byte, l int) []byte { diff --git a/consensus/clique/snapshot_test.go b/consensus/clique/snapshot_test.go index 8b51e6e094..29a8379832 100644 --- a/consensus/clique/snapshot_test.go +++ b/consensus/clique/snapshot_test.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" @@ -81,7 +82,7 @@ func (r *testerChainReader) GetBlock(common.Hash, uint64) *types.Block { panic func (r *testerChainReader) GetHeaderByHash(common.Hash) *types.Header { panic("not supported") } func (r *testerChainReader) GetHeaderByNumber(number uint64) *types.Header { if number == 0 { - return core.GetHeader(r.db, core.GetCanonicalHash(r.db, 0), 0) + return rawdb.ReadHeader(r.db, rawdb.ReadCanonicalHash(r.db, 0), 0) } panic("not supported") } @@ -351,7 +352,7 @@ func TestVoting(t *testing.T) { copy(genesis.ExtraData[extraVanity+j*common.AddressLength:], signer[:]) } // Create a pristine blockchain with the genesis injected - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() genesis.Commit(db) // Assemble a chain of headers from the cast votes diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go index 1b3dcee302..ac049f9c36 100644 --- a/consensus/ethash/ethash.go +++ b/consensus/ethash/ethash.go @@ -156,7 +156,7 @@ type lru struct { futureItem interface{} } -// newlru create a new least-recently-used cache for ither the verification caches +// newlru create a new least-recently-used cache for either the verification caches // or the mining datasets. func newlru(what string, maxItems int, new func(epoch uint64) interface{}) *lru { if maxItems <= 0 { diff --git a/core/bench_test.go b/core/bench_test.go index e23f0d19d1..748aebe407 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" @@ -148,7 +149,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) { // Create the database in memory or in a temporary directory. var db ethdb.Database if !disk { - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() } else { dir, err := ioutil.TempDir("", "eth-core-bench") if err != nil { @@ -234,13 +235,15 @@ func makeChainForBench(db ethdb.Database, full bool, count uint64) { ReceiptHash: types.EmptyRootHash, } hash = header.Hash() - WriteHeader(db, header) - WriteCanonicalHash(db, hash, n) - WriteTd(db, hash, n, big.NewInt(int64(n+1))) + + rawdb.WriteHeader(db, header) + rawdb.WriteCanonicalHash(db, hash, n) + rawdb.WriteTd(db, hash, n, big.NewInt(int64(n+1))) + if full || n == 0 { block := types.NewBlockWithHeader(header) - WriteBody(db, hash, n, block.Body()) - WriteBlockReceipts(db, hash, n, nil) + rawdb.WriteBody(db, hash, n, block.Body()) + rawdb.WriteReceipts(db, hash, n, nil) } } } @@ -292,11 +295,10 @@ func benchReadChain(b *testing.B, full bool, count uint64) { header := chain.GetHeaderByNumber(n) if full { hash := header.Hash() - GetBody(db, hash, n) - GetBlockReceipts(db, hash, n) + rawdb.ReadBody(db, hash, n) + rawdb.ReadReceipts(db, hash, n) } } - chain.Stop() db.Close() } diff --git a/core/block_validator_test.go b/core/block_validator_test.go index e334b3c3cd..2a171218e3 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -32,7 +32,7 @@ import ( func TestHeaderVerification(t *testing.T) { // Create a simple chain to verify var ( - testdb, _ = ethdb.NewMemDatabase() + testdb = ethdb.NewMemDatabase() gspec = &Genesis{Config: params.TestChainConfig} genesis = gspec.MustCommit(testdb) blocks, _ = GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), testdb, 8, nil) @@ -84,7 +84,7 @@ func TestHeaderConcurrentVerification32(t *testing.T) { testHeaderConcurrentVeri func testHeaderConcurrentVerification(t *testing.T, threads int) { // Create a simple chain to verify var ( - testdb, _ = ethdb.NewMemDatabase() + testdb = ethdb.NewMemDatabase() gspec = &Genesis{Config: params.TestChainConfig} genesis = gspec.MustCommit(testdb) blocks, _ = GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), testdb, 8, nil) @@ -156,7 +156,7 @@ func TestHeaderConcurrentAbortion32(t *testing.T) { testHeaderConcurrentAbortion func testHeaderConcurrentAbortion(t *testing.T, threads int) { // Create a simple chain to verify var ( - testdb, _ = ethdb.NewMemDatabase() + testdb = ethdb.NewMemDatabase() gspec = &Genesis{Config: params.TestChainConfig} genesis = gspec.MustCommit(testdb) blocks, _ = GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), testdb, 1024, nil) diff --git a/core/blockchain.go b/core/blockchain.go index a76663fb5a..3040f82944 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" @@ -203,7 +204,7 @@ func (bc *BlockChain) getProcInterrupt() bool { // assumes that the chain manager mutex is held. func (bc *BlockChain) loadLastState() error { // Restore the last known head block - head := GetHeadBlockHash(bc.db) + head := rawdb.ReadHeadBlockHash(bc.db) if head == (common.Hash{}) { // Corrupt or empty database, init from scratch log.Warn("Empty database, resetting chain") @@ -229,7 +230,7 @@ func (bc *BlockChain) loadLastState() error { // Restore the last known head header currentHeader := currentBlock.Header() - if head := GetHeadHeaderHash(bc.db); head != (common.Hash{}) { + if head := rawdb.ReadHeadHeaderHash(bc.db); head != (common.Hash{}) { if header := bc.GetHeaderByHash(head); header != nil { currentHeader = header } @@ -238,7 +239,7 @@ func (bc *BlockChain) loadLastState() error { // Restore the last known head fast block bc.currentFastBlock.Store(currentBlock) - if head := GetHeadFastBlockHash(bc.db); head != (common.Hash{}) { + if head := rawdb.ReadHeadFastBlockHash(bc.db); head != (common.Hash{}) { if block := bc.GetBlockByHash(head); block != nil { bc.currentFastBlock.Store(block) } @@ -270,7 +271,7 @@ func (bc *BlockChain) SetHead(head uint64) error { // Rewind the header chain, deleting all block bodies until then delFn := func(hash common.Hash, num uint64) { - DeleteBody(bc.db, hash, num) + rawdb.DeleteBody(bc.db, hash, num) } bc.hc.SetHead(head, delFn) currentHeader := bc.hc.CurrentHeader() @@ -304,12 +305,10 @@ func (bc *BlockChain) SetHead(head uint64) error { } currentBlock := bc.CurrentBlock() currentFastBlock := bc.CurrentFastBlock() - if err := WriteHeadBlockHash(bc.db, currentBlock.Hash()); err != nil { - log.Crit("Failed to reset head full block", "err", err) - } - if err := WriteHeadFastBlockHash(bc.db, currentFastBlock.Hash()); err != nil { - log.Crit("Failed to reset head fast block", "err", err) - } + + rawdb.WriteHeadBlockHash(bc.db, currentBlock.Hash()) + rawdb.WriteHeadFastBlockHash(bc.db, currentFastBlock.Hash()) + return bc.loadLastState() } @@ -407,9 +406,8 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error { if err := bc.hc.WriteTd(genesis.Hash(), genesis.NumberU64(), genesis.Difficulty()); err != nil { log.Crit("Failed to write genesis block TD", "err", err) } - if err := WriteBlock(bc.db, genesis); err != nil { - log.Crit("Failed to write genesis block", "err", err) - } + rawdb.WriteBlock(bc.db, genesis) + bc.genesisBlock = genesis bc.insert(bc.genesisBlock) bc.currentBlock.Store(bc.genesisBlock) @@ -475,24 +473,19 @@ func (bc *BlockChain) ExportN(w io.Writer, first uint64, last uint64) error { // Note, this function assumes that the `mu` mutex is held! func (bc *BlockChain) insert(block *types.Block) { // If the block is on a side chain or an unknown one, force other heads onto it too - updateHeads := GetCanonicalHash(bc.db, block.NumberU64()) != block.Hash() + updateHeads := rawdb.ReadCanonicalHash(bc.db, block.NumberU64()) != block.Hash() // Add the block to the canonical chain number scheme and mark as the head - if err := WriteCanonicalHash(bc.db, block.Hash(), block.NumberU64()); err != nil { - log.Crit("Failed to insert block number", "err", err) - } - if err := WriteHeadBlockHash(bc.db, block.Hash()); err != nil { - log.Crit("Failed to insert head block hash", "err", err) - } + rawdb.WriteCanonicalHash(bc.db, block.Hash(), block.NumberU64()) + rawdb.WriteHeadBlockHash(bc.db, block.Hash()) + bc.currentBlock.Store(block) // If the block is better than our head or is on a different chain, force update heads if updateHeads { bc.hc.SetCurrentHeader(block.Header()) + rawdb.WriteHeadFastBlockHash(bc.db, block.Hash()) - if err := WriteHeadFastBlockHash(bc.db, block.Hash()); err != nil { - log.Crit("Failed to insert head fast block hash", "err", err) - } bc.currentFastBlock.Store(block) } } @@ -510,7 +503,11 @@ func (bc *BlockChain) GetBody(hash common.Hash) *types.Body { body := cached.(*types.Body) return body } - body := GetBody(bc.db, hash, bc.hc.GetBlockNumber(hash)) + number := bc.hc.GetBlockNumber(hash) + if number == nil { + return nil + } + body := rawdb.ReadBody(bc.db, hash, *number) if body == nil { return nil } @@ -526,7 +523,11 @@ func (bc *BlockChain) GetBodyRLP(hash common.Hash) rlp.RawValue { if cached, ok := bc.bodyRLPCache.Get(hash); ok { return cached.(rlp.RawValue) } - body := GetBodyRLP(bc.db, hash, bc.hc.GetBlockNumber(hash)) + number := bc.hc.GetBlockNumber(hash) + if number == nil { + return nil + } + body := rawdb.ReadBodyRLP(bc.db, hash, *number) if len(body) == 0 { return nil } @@ -540,8 +541,7 @@ func (bc *BlockChain) HasBlock(hash common.Hash, number uint64) bool { if bc.blockCache.Contains(hash) { return true } - ok, _ := bc.db.Has(blockBodyKey(hash, number)) - return ok + return rawdb.HasBody(bc.db, hash, number) } // HasState checks if state trie is fully present in the database or not. @@ -568,7 +568,7 @@ func (bc *BlockChain) GetBlock(hash common.Hash, number uint64) *types.Block { if block, ok := bc.blockCache.Get(hash); ok { return block.(*types.Block) } - block := GetBlock(bc.db, hash, number) + block := rawdb.ReadBlock(bc.db, hash, number) if block == nil { return nil } @@ -579,13 +579,17 @@ func (bc *BlockChain) GetBlock(hash common.Hash, number uint64) *types.Block { // GetBlockByHash retrieves a block from the database by hash, caching it if found. func (bc *BlockChain) GetBlockByHash(hash common.Hash) *types.Block { - return bc.GetBlock(hash, bc.hc.GetBlockNumber(hash)) + number := bc.hc.GetBlockNumber(hash) + if number == nil { + return nil + } + return bc.GetBlock(hash, *number) } // GetBlockByNumber retrieves a block from the database by number, caching it // (associated with its hash) if found. func (bc *BlockChain) GetBlockByNumber(number uint64) *types.Block { - hash := GetCanonicalHash(bc.db, number) + hash := rawdb.ReadCanonicalHash(bc.db, number) if hash == (common.Hash{}) { return nil } @@ -594,21 +598,28 @@ func (bc *BlockChain) GetBlockByNumber(number uint64) *types.Block { // GetReceiptsByHash retrieves the receipts for all transactions in a given block. func (bc *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts { - return GetBlockReceipts(bc.db, hash, GetBlockNumber(bc.db, hash)) + number := rawdb.ReadHeaderNumber(bc.db, hash) + if number == nil { + return nil + } + return rawdb.ReadReceipts(bc.db, hash, *number) } // GetBlocksFromHash returns the block corresponding to hash and up to n-1 ancestors. // [deprecated by eth/62] func (bc *BlockChain) GetBlocksFromHash(hash common.Hash, n int) (blocks []*types.Block) { number := bc.hc.GetBlockNumber(hash) + if number == nil { + return nil + } for i := 0; i < n; i++ { - block := bc.GetBlock(hash, number) + block := bc.GetBlock(hash, *number) if block == nil { break } blocks = append(blocks, block) hash = block.ParentHash() - number-- + *number-- } return } @@ -713,12 +724,12 @@ func (bc *BlockChain) Rollback(chain []common.Hash) { if currentFastBlock := bc.CurrentFastBlock(); currentFastBlock.Hash() == hash { newFastBlock := bc.GetBlock(currentFastBlock.ParentHash(), currentFastBlock.NumberU64()-1) bc.currentFastBlock.Store(newFastBlock) - WriteHeadFastBlockHash(bc.db, newFastBlock.Hash()) + rawdb.WriteHeadFastBlockHash(bc.db, newFastBlock.Hash()) } if currentBlock := bc.CurrentBlock(); currentBlock.Hash() == hash { newBlock := bc.GetBlock(currentBlock.ParentHash(), currentBlock.NumberU64()-1) bc.currentBlock.Store(newBlock) - WriteHeadBlockHash(bc.db, newBlock.Hash()) + rawdb.WriteHeadBlockHash(bc.db, newBlock.Hash()) } } } @@ -803,15 +814,10 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ return i, fmt.Errorf("failed to set receipts data: %v", err) } // Write all the data out into the database - if err := WriteBody(batch, block.Hash(), block.NumberU64(), block.Body()); err != nil { - return i, fmt.Errorf("failed to write block body: %v", err) - } - if err := WriteBlockReceipts(batch, block.Hash(), block.NumberU64(), receipts); err != nil { - return i, fmt.Errorf("failed to write block receipts: %v", err) - } - if err := WriteTxLookupEntries(batch, block); err != nil { - return i, fmt.Errorf("failed to write lookup metadata: %v", err) - } + rawdb.WriteBody(batch, block.Hash(), block.NumberU64(), block.Body()) + rawdb.WriteReceipts(batch, block.Hash(), block.NumberU64(), receipts) + rawdb.WriteTxLookupEntries(batch, block) + stats.processed++ if batch.ValueSize() >= ethdb.IdealBatchSize { @@ -835,9 +841,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ if td := bc.GetTd(head.Hash(), head.NumberU64()); td != nil { // Rewind may have occurred, skip in that case currentFastBlock := bc.CurrentFastBlock() if bc.GetTd(currentFastBlock.Hash(), currentFastBlock.NumberU64()).Cmp(td) < 0 { - if err := WriteHeadFastBlockHash(bc.db, head.Hash()); err != nil { - log.Crit("Failed to update head fast block hash", "err", err) - } + rawdb.WriteHeadFastBlockHash(bc.db, head.Hash()) bc.currentFastBlock.Store(head) } } @@ -865,9 +869,8 @@ func (bc *BlockChain) WriteBlockWithoutState(block *types.Block, td *big.Int) (e if err := bc.hc.WriteTd(block.Hash(), block.NumberU64(), td); err != nil { return err } - if err := WriteBlock(bc.db, block); err != nil { - return err - } + rawdb.WriteBlock(bc.db, block) + return nil } @@ -895,9 +898,8 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. } // Write other block data using a batch. batch := bc.db.NewBatch() - if err := WriteBlock(batch, block); err != nil { - return NonStatTy, err - } + rawdb.WriteBlock(batch, block) + root, err := state.Commit(bc.chainConfig.IsEIP158(block.Number())) if err != nil { return NonStatTy, err @@ -954,9 +956,8 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. } } } - if err := WriteBlockReceipts(batch, block.Hash(), block.NumberU64(), receipts); err != nil { - return NonStatTy, err - } + rawdb.WriteReceipts(batch, block.Hash(), block.NumberU64(), receipts) + // If the total difficulty is higher than our known, add it to the canonical chain // Second clause in the if statement reduces the vulnerability to selfish mining. // Please refer to http://www.cs.cornell.edu/~ie53/publications/btcProcFC.pdf @@ -973,14 +974,10 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. return NonStatTy, err } } - // Write the positional metadata for transaction and receipt lookups - if err := WriteTxLookupEntries(batch, block); err != nil { - return NonStatTy, err - } - // Write hash preimages - if err := WritePreimages(bc.db, block.NumberU64(), state.Preimages()); err != nil { - return NonStatTy, err - } + // Write the positional metadata for transaction/receipt lookups and preimages + rawdb.WriteTxLookupEntries(batch, block) + rawdb.WritePreimages(batch, block.NumberU64(), state.Preimages()) + status = CanonStatTy } else { status = SideStatTy @@ -1260,9 +1257,13 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { // collectLogs collects the logs that were generated during the // processing of the block that corresponds with the given hash. // These logs are later announced as deleted. - collectLogs = func(h common.Hash) { + collectLogs = func(hash common.Hash) { // Coalesce logs and set 'Removed'. - receipts := GetBlockReceipts(bc.db, h, bc.hc.GetBlockNumber(h)) + number := bc.hc.GetBlockNumber(hash) + if number == nil { + return + } + receipts := rawdb.ReadReceipts(bc.db, hash, *number) for _, receipt := range receipts { for _, log := range receipt.Logs { del := *log @@ -1331,9 +1332,7 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { // insert the block in the canonical way, re-writing history bc.insert(newChain[i]) // write lookup entries for hash based transaction/receipt searches - if err := WriteTxLookupEntries(bc.db, newChain[i]); err != nil { - return err - } + rawdb.WriteTxLookupEntries(bc.db, newChain[i]) addedTxs = append(addedTxs, newChain[i].Transactions()...) } // calculate the difference between deleted and added transactions @@ -1341,7 +1340,7 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { for _, tx := range diff { // When transactions get deleted from the database that means the // receipts that were created in the fork must also be deleted - DeleteTxLookupEntry(bc.db, tx.Hash()) + rawdb.DeleteTxLookupEntry(bc.db, tx.Hash()) // Let ReturnData subscribers know when a transaction is removed from canonical chain bc.txPostFeed.Send(TransactionEvent{TxHash: tx.Hash(), RetData: &types.ReturnData{TxHash: tx.Hash(), Removed: true}}) } diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 24e81a0cb9..9c8b668a92 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" @@ -128,8 +129,8 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error { return err } blockchain.mu.Lock() - WriteTd(blockchain.db, block.Hash(), block.NumberU64(), new(big.Int).Add(block.Difficulty(), blockchain.GetTdByHash(block.ParentHash()))) - WriteBlock(blockchain.db, block) + rawdb.WriteTd(blockchain.db, block.Hash(), block.NumberU64(), new(big.Int).Add(block.Difficulty(), blockchain.GetTdByHash(block.ParentHash()))) + rawdb.WriteBlock(blockchain.db, block) statedb.Commit(false) blockchain.mu.Unlock() } @@ -146,8 +147,8 @@ func testHeaderChainImport(chain []*types.Header, blockchain *BlockChain) error } // Manually insert the header into the database, but don't reorganise (allows subsequent testing) blockchain.mu.Lock() - WriteTd(blockchain.db, header.Hash(), header.Number.Uint64(), new(big.Int).Add(header.Difficulty, blockchain.GetTdByHash(header.ParentHash))) - WriteHeader(blockchain.db, header) + rawdb.WriteTd(blockchain.db, header.Hash(), header.Number.Uint64(), new(big.Int).Add(header.Difficulty, blockchain.GetTdByHash(header.ParentHash))) + rawdb.WriteHeader(blockchain.db, header) blockchain.mu.Unlock() } return nil @@ -173,7 +174,7 @@ func TestLastBlock(t *testing.T) { if _, err := blockchain.InsertChain(blocks); err != nil { t.Fatalf("Failed to insert block: %v", err) } - if blocks[len(blocks)-1].Hash() != GetHeadBlockHash(blockchain.db) { + if blocks[len(blocks)-1].Hash() != rawdb.ReadHeadBlockHash(blockchain.db) { t.Fatalf("Write/Get HeadBlockHash failed") } } @@ -568,11 +569,11 @@ func testInsertNonceError(t *testing.T, full bool) { func TestFastVsFullChains(t *testing.T) { // Configure and generate a sample block chain var ( - gendb, _ = ethdb.NewMemDatabase() - key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - address = crypto.PubkeyToAddress(key.PublicKey) - funds = big.NewInt(1000000000) - gspec = &Genesis{ + gendb = ethdb.NewMemDatabase() + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + funds = big.NewInt(1000000000) + gspec = &Genesis{ Config: params.TestChainConfig, Alloc: GenesisAlloc{address: {Balance: funds}}, } @@ -598,7 +599,7 @@ func TestFastVsFullChains(t *testing.T) { } }) // Import the chain as an archive node for the comparison baseline - archiveDb, _ := ethdb.NewMemDatabase() + archiveDb := ethdb.NewMemDatabase() gspec.MustCommit(archiveDb) archive, _ := NewBlockChain(archiveDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) defer archive.Stop() @@ -607,7 +608,7 @@ func TestFastVsFullChains(t *testing.T) { t.Fatalf("failed to process block %d: %v", n, err) } // Fast import the chain as a non-archive node to test - fastDb, _ := ethdb.NewMemDatabase() + fastDb := ethdb.NewMemDatabase() gspec.MustCommit(fastDb) fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) defer fast.Stop() @@ -639,13 +640,13 @@ func TestFastVsFullChains(t *testing.T) { } else if types.CalcUncleHash(fblock.Uncles()) != types.CalcUncleHash(ablock.Uncles()) { t.Errorf("block #%d [%x]: uncles mismatch: have %v, want %v", num, hash, fblock.Uncles(), ablock.Uncles()) } - if freceipts, areceipts := GetBlockReceipts(fastDb, hash, GetBlockNumber(fastDb, hash)), GetBlockReceipts(archiveDb, hash, GetBlockNumber(archiveDb, hash)); types.DeriveSha(freceipts) != types.DeriveSha(areceipts) { + if freceipts, areceipts := rawdb.ReadReceipts(fastDb, hash, *rawdb.ReadHeaderNumber(fastDb, hash)), rawdb.ReadReceipts(archiveDb, hash, *rawdb.ReadHeaderNumber(archiveDb, hash)); types.DeriveSha(freceipts) != types.DeriveSha(areceipts) { t.Errorf("block #%d [%x]: receipts mismatch: have %v, want %v", num, hash, freceipts, areceipts) } } // Check that the canonical chains are the same between the databases for i := 0; i < len(blocks)+1; i++ { - if fhash, ahash := GetCanonicalHash(fastDb, uint64(i)), GetCanonicalHash(archiveDb, uint64(i)); fhash != ahash { + if fhash, ahash := rawdb.ReadCanonicalHash(fastDb, uint64(i)), rawdb.ReadCanonicalHash(archiveDb, uint64(i)); fhash != ahash { t.Errorf("block #%d: canonical hash mismatch: have %v, want %v", i, fhash, ahash) } } @@ -656,12 +657,12 @@ func TestFastVsFullChains(t *testing.T) { func TestLightVsFastVsFullChainHeads(t *testing.T) { // Configure and generate a sample block chain var ( - gendb, _ = ethdb.NewMemDatabase() - key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - address = crypto.PubkeyToAddress(key.PublicKey) - funds = big.NewInt(1000000000) - gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{address: {Balance: funds}}} - genesis = gspec.MustCommit(gendb) + gendb = ethdb.NewMemDatabase() + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + funds = big.NewInt(1000000000) + gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{address: {Balance: funds}}} + genesis = gspec.MustCommit(gendb) ) height := uint64(1024) blocks, receipts := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), gendb, int(height), nil) @@ -684,7 +685,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { } } // Import the chain as an archive node and ensure all pointers are updated - archiveDb, _ := ethdb.NewMemDatabase() + archiveDb := ethdb.NewMemDatabase() gspec.MustCommit(archiveDb) archive, _ := NewBlockChain(archiveDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) @@ -698,7 +699,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { assert(t, "archive", archive, height/2, height/2, height/2) // Import the chain as a non-archive node and ensure all pointers are updated - fastDb, _ := ethdb.NewMemDatabase() + fastDb := ethdb.NewMemDatabase() gspec.MustCommit(fastDb) fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) defer fast.Stop() @@ -718,7 +719,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { assert(t, "fast", fast, height/2, height/2, 0) // Import the chain as a light node and ensure all pointers are updated - lightDb, _ := ethdb.NewMemDatabase() + lightDb := ethdb.NewMemDatabase() gspec.MustCommit(lightDb) light, _ := NewBlockChain(lightDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) @@ -741,7 +742,7 @@ func TestChainTxReorgs(t *testing.T) { addr1 = crypto.PubkeyToAddress(key1.PublicKey) addr2 = crypto.PubkeyToAddress(key2.PublicKey) addr3 = crypto.PubkeyToAddress(key3.PublicKey) - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() gspec = &Genesis{ Config: params.TestChainConfig, GasLimit: 3141592, @@ -821,28 +822,28 @@ func TestChainTxReorgs(t *testing.T) { // removed tx for i, tx := range (types.Transactions{pastDrop, freshDrop}) { - if txn, _, _, _ := GetTransaction(db, tx.Hash()); txn != nil { + if txn, _, _, _ := rawdb.ReadTransaction(db, tx.Hash()); txn != nil { t.Errorf("drop %d: tx %v found while shouldn't have been", i, txn) } - if rcpt, _, _, _ := GetReceipt(db, tx.Hash()); rcpt != nil { + if rcpt, _, _, _ := rawdb.ReadReceipt(db, tx.Hash()); rcpt != nil { t.Errorf("drop %d: receipt %v found while shouldn't have been", i, rcpt) } } // added tx for i, tx := range (types.Transactions{pastAdd, freshAdd, futureAdd}) { - if txn, _, _, _ := GetTransaction(db, tx.Hash()); txn == nil { + if txn, _, _, _ := rawdb.ReadTransaction(db, tx.Hash()); txn == nil { t.Errorf("add %d: expected tx to be found", i) } - if rcpt, _, _, _ := GetReceipt(db, tx.Hash()); rcpt == nil { + if rcpt, _, _, _ := rawdb.ReadReceipt(db, tx.Hash()); rcpt == nil { t.Errorf("add %d: expected receipt to be found", i) } } // shared tx for i, tx := range (types.Transactions{postponed, swapped}) { - if txn, _, _, _ := GetTransaction(db, tx.Hash()); txn == nil { + if txn, _, _, _ := rawdb.ReadTransaction(db, tx.Hash()); txn == nil { t.Errorf("share %d: expected tx to be found", i) } - if rcpt, _, _, _ := GetReceipt(db, tx.Hash()); rcpt == nil { + if rcpt, _, _, _ := rawdb.ReadReceipt(db, tx.Hash()); rcpt == nil { t.Errorf("share %d: expected receipt to be found", i) } } @@ -853,7 +854,7 @@ func TestLogReorgs(t *testing.T) { var ( key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() // this code generates a log code = common.Hex2Bytes("60606040525b7f24ec1d3ff24c2f6ff210738839dbc339cd45a5294d85c79361016243157aae7b60405180905060405180910390a15b600a8060416000396000f360606040526008565b00") gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000)}}} @@ -897,7 +898,7 @@ func TestLogReorgs(t *testing.T) { func TestReorgSideEvent(t *testing.T) { var ( - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) gspec = &Genesis{ @@ -997,14 +998,14 @@ func TestCanonicalBlockRetrieval(t *testing.T) { // try to retrieve a block by its canonical hash and see if the block data can be retrieved. for { - ch := GetCanonicalHash(blockchain.db, block.NumberU64()) + ch := rawdb.ReadCanonicalHash(blockchain.db, block.NumberU64()) if ch == (common.Hash{}) { continue // busy wait for canonical hash to be written } if ch != block.Hash() { t.Fatalf("unknown canonical hash, want %s, got %s", block.Hash().Hex(), ch.Hex()) } - fb := GetBlock(blockchain.db, ch, block.NumberU64()) + fb := rawdb.ReadBlock(blockchain.db, ch, block.NumberU64()) if fb == nil { t.Fatalf("unable to retrieve block %d for canonical hash: %s", block.NumberU64(), ch.Hex()) } @@ -1025,7 +1026,7 @@ func TestCanonicalBlockRetrieval(t *testing.T) { func TestEIP155Transition(t *testing.T) { // Configure and generate a sample block chain var ( - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") address = crypto.PubkeyToAddress(key.PublicKey) funds = big.NewInt(1000000000) @@ -1129,7 +1130,7 @@ func TestEIP155Transition(t *testing.T) { func TestEIP161AccountRemoval(t *testing.T) { // Configure and generate a sample block chain var ( - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") address = crypto.PubkeyToAddress(key.PublicKey) funds = big.NewInt(1000000000) @@ -1201,7 +1202,7 @@ func TestBlockchainHeaderchainReorgConsistency(t *testing.T) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() genesis := new(Genesis).MustCommit(db) blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 64, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) @@ -1217,7 +1218,7 @@ func TestBlockchainHeaderchainReorgConsistency(t *testing.T) { } // Import the canonical and fork chain side by side, verifying the current block // and current header consistency - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() new(Genesis).MustCommit(diskdb) chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}) @@ -1246,7 +1247,7 @@ func TestTrieForkGC(t *testing.T) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() genesis := new(Genesis).MustCommit(db) blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 2*triesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) @@ -1261,7 +1262,7 @@ func TestTrieForkGC(t *testing.T) { forks[i] = fork[0] } // Import the canonical and fork chain side by side, forcing the trie cache to cache both - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() new(Genesis).MustCommit(diskdb) chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}) @@ -1292,7 +1293,7 @@ func TestLargeReorgTrieGC(t *testing.T) { // Generate the original common chain segment and the two competing forks engine := ethash.NewFaker() - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() genesis := new(Genesis).MustCommit(db) shared, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 64, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) @@ -1300,7 +1301,7 @@ func TestLargeReorgTrieGC(t *testing.T) { competitor, _ := GenerateChain(params.TestChainConfig, shared[len(shared)-1], engine, db, 2*triesInMemory+1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{3}) }) // Import the shared chain and the original canonical one - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() new(Genesis).MustCommit(diskdb) chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}) @@ -1360,7 +1361,7 @@ func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks in ) // Generate the original common chain segment and the two competing forks engine := ethash.NewFaker() - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() genesis := gspec.MustCommit(db) blockGenerator := func(i int, block *BlockGen) { @@ -1382,7 +1383,7 @@ func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks in b.ResetTimer() for i := 0; i < b.N; i++ { // Import the shared chain and the original canonical one - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() gspec.MustCommit(diskdb) chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}) diff --git a/core/chain_indexer.go b/core/chain_indexer.go index 158ed83245..0b927116d0 100644 --- a/core/chain_indexer.go +++ b/core/chain_indexer.go @@ -24,6 +24,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -206,7 +207,7 @@ func (c *ChainIndexer) eventLoop(currentHeader *types.Header, events chan ChainE // TODO(karalabe): This operation is expensive and might block, causing the event system to // potentially also lock up. We need to do with on a different thread somehow. - if h := FindCommonAncestor(c.chainDb, prevHeader, header); h != nil { + if h := rawdb.FindCommonAncestor(c.chainDb, prevHeader, header); h != nil { c.newHead(h.Number.Uint64(), true) } } @@ -349,11 +350,11 @@ func (c *ChainIndexer) processSection(section uint64, lastHead common.Hash) (com } for number := section * c.sectionSize; number < (section+1)*c.sectionSize; number++ { - hash := GetCanonicalHash(c.chainDb, number) + hash := rawdb.ReadCanonicalHash(c.chainDb, number) if hash == (common.Hash{}) { return common.Hash{}, fmt.Errorf("canonical block #%d unknown", number) } - header := GetHeader(c.chainDb, hash, number) + header := rawdb.ReadHeader(c.chainDb, hash, number) if header == nil { return common.Hash{}, fmt.Errorf("block #%d [%x…] not found", number, hash[:4]) } else if header.ParentHash != lastHead { diff --git a/core/chain_indexer_test.go b/core/chain_indexer_test.go index 9fc09eda51..550caf5567 100644 --- a/core/chain_indexer_test.go +++ b/core/chain_indexer_test.go @@ -24,6 +24,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" ) @@ -47,7 +48,7 @@ func TestChainIndexerWithChildren(t *testing.T) { // multiple backends. The section size and required confirmation count parameters // are randomized. func testChainIndexer(t *testing.T, count int) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() defer db.Close() // Create a chain of indexers and ensure they all report empty @@ -92,10 +93,10 @@ func testChainIndexer(t *testing.T, count int) { inject := func(number uint64) { header := &types.Header{Number: big.NewInt(int64(number)), Extra: big.NewInt(rand.Int63()).Bytes()} if number > 0 { - header.ParentHash = GetCanonicalHash(db, number-1) + header.ParentHash = rawdb.ReadCanonicalHash(db, number-1) } - WriteHeader(db, header) - WriteCanonicalHash(db, header.Hash(), number) + rawdb.WriteHeader(db, header) + rawdb.WriteCanonicalHash(db, header.Hash(), number) } // Start indexer with an already existing chain for i := uint64(0); i <= 100; i++ { diff --git a/core/chain_makers.go b/core/chain_makers.go index 18a63a074b..9fe00885a3 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -256,11 +256,12 @@ func makeHeader(chain consensus.ChainReader, parent *types.Block, state *state.S // chain. Depending on the full flag, if creates either a full block chain or a // header only chain. func newCanonical(engine consensus.Engine, n int, full bool) (ethdb.Database, *BlockChain, error) { - // Initialize a fresh chain with only a genesis block - gspec := new(Genesis) - db, _ := ethdb.NewMemDatabase() - genesis := gspec.MustCommit(db) + var ( + db = ethdb.NewMemDatabase() + genesis = new(Genesis).MustCommit(db) + ) + // Initialize a fresh chain with only a genesis block blockchain, _ := NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}) // Create and inject the requested chain if n == 0 { diff --git a/core/chain_makers_test.go b/core/chain_makers_test.go index 93be43ddce..5015d1f48e 100644 --- a/core/chain_makers_test.go +++ b/core/chain_makers_test.go @@ -36,7 +36,7 @@ func ExampleGenerateChain() { addr1 = crypto.PubkeyToAddress(key1.PublicKey) addr2 = crypto.PubkeyToAddress(key2.PublicKey) addr3 = crypto.PubkeyToAddress(key3.PublicKey) - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() ) // Ensure that key1 has some funds in the genesis block. diff --git a/core/dao_test.go b/core/dao_test.go index e0a3e3ff37..284b1d98bd 100644 --- a/core/dao_test.go +++ b/core/dao_test.go @@ -32,13 +32,13 @@ func TestDAOForkRangeExtradata(t *testing.T) { forkBlock := big.NewInt(32) // Generate a common prefix for both pro-forkers and non-forkers - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() gspec := new(Genesis) genesis := gspec.MustCommit(db) prefix, _ := GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, int(forkBlock.Int64()-1), func(i int, gen *BlockGen) {}) // Create the concurrent, conflicting two nodes - proDb, _ := ethdb.NewMemDatabase() + proDb := ethdb.NewMemDatabase() gspec.MustCommit(proDb) proConf := *params.TestChainConfig @@ -48,7 +48,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { proBc, _ := NewBlockChain(proDb, nil, &proConf, ethash.NewFaker(), vm.Config{}) defer proBc.Stop() - conDb, _ := ethdb.NewMemDatabase() + conDb := ethdb.NewMemDatabase() gspec.MustCommit(conDb) conConf := *params.TestChainConfig @@ -67,7 +67,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Try to expand both pro-fork and non-fork chains iteratively with other camp's blocks for i := int64(0); i < params.DAOForkExtraRange.Int64(); i++ { // Create a pro-fork block, and try to feed into the no-fork chain - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() gspec.MustCommit(db) bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{}) defer bc.Stop() @@ -92,7 +92,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { t.Fatalf("contra-fork chain didn't accepted no-fork block: %v", err) } // Create a no-fork block, and try to feed into the pro-fork chain - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() gspec.MustCommit(db) bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{}) defer bc.Stop() @@ -118,7 +118,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { } } // Verify that contra-forkers accept pro-fork extra-datas after forking finishes - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() gspec.MustCommit(db) bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{}) defer bc.Stop() @@ -138,7 +138,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { t.Fatalf("contra-fork chain didn't accept pro-fork block post-fork: %v", err) } // Verify that pro-forkers accept contra-fork extra-datas after forking finishes - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() gspec.MustCommit(db) bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{}) defer bc.Stop() diff --git a/core/database_util.go b/core/database_util.go deleted file mode 100644 index 8c46989854..0000000000 --- a/core/database_util.go +++ /dev/null @@ -1,652 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package core - -import ( - "bytes" - "encoding/binary" - "encoding/json" - "errors" - "fmt" - "math/big" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/metrics" - "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/rlp" -) - -// DatabaseReader wraps the Get method of a backing data store. -type DatabaseReader interface { - Get(key []byte) (value []byte, err error) -} - -// DatabaseDeleter wraps the Delete method of a backing data store. -type DatabaseDeleter interface { - Delete(key []byte) error -} - -var ( - headHeaderKey = []byte("LastHeader") - headBlockKey = []byte("LastBlock") - headFastKey = []byte("LastFast") - trieSyncKey = []byte("TrieSync") - - // Data item prefixes (use single byte to avoid mixing data types, avoid `i`). - headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header - tdSuffix = []byte("t") // headerPrefix + num (uint64 big endian) + hash + tdSuffix -> td - numSuffix = []byte("n") // headerPrefix + num (uint64 big endian) + numSuffix -> hash - blockHashPrefix = []byte("H") // blockHashPrefix + hash -> num (uint64 big endian) - bodyPrefix = []byte("b") // bodyPrefix + num (uint64 big endian) + hash -> block body - blockReceiptsPrefix = []byte("r") // blockReceiptsPrefix + num (uint64 big endian) + hash -> block receipts - lookupPrefix = []byte("l") // lookupPrefix + hash -> transaction/receipt lookup metadata - bloomBitsPrefix = []byte("B") // bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash -> bloom bits - - preimagePrefix = "secure-key-" // preimagePrefix + hash -> preimage - configPrefix = []byte("ethereum-config-") // config prefix for the db - - // Chain index prefixes (use `i` + single byte to avoid mixing data types). - BloomBitsIndexPrefix = []byte("iB") // BloomBitsIndexPrefix is the data table of a chain indexer to track its progress - - // used by old db, now only used for conversion - oldReceiptsPrefix = []byte("receipts-") - oldTxMetaSuffix = []byte{0x01} - - ErrChainConfigNotFound = errors.New("ChainConfig not found") // general config not found error - - preimageCounter = metrics.NewRegisteredCounter("db/preimage/total", nil) - preimageHitCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil) -) - -// TxLookupEntry is a positional metadata to help looking up the data content of -// a transaction or receipt given only its hash. -type TxLookupEntry struct { - BlockHash common.Hash - BlockIndex uint64 - Index uint64 -} - -// encodeBlockNumber encodes a block number as big endian uint64 -func encodeBlockNumber(number uint64) []byte { - enc := make([]byte, 8) - binary.BigEndian.PutUint64(enc, number) - return enc -} - -// GetCanonicalHash retrieves a hash assigned to a canonical block number. -func GetCanonicalHash(db DatabaseReader, number uint64) common.Hash { - data, _ := db.Get(append(append(headerPrefix, encodeBlockNumber(number)...), numSuffix...)) - if len(data) == 0 { - return common.Hash{} - } - return common.BytesToHash(data) -} - -// missingNumber is returned by GetBlockNumber if no header with the -// given block hash has been stored in the database -const missingNumber = uint64(0xffffffffffffffff) - -// GetBlockNumber returns the block number assigned to a block hash -// if the corresponding header is present in the database -func GetBlockNumber(db DatabaseReader, hash common.Hash) uint64 { - data, _ := db.Get(append(blockHashPrefix, hash.Bytes()...)) - if len(data) != 8 { - return missingNumber - } - return binary.BigEndian.Uint64(data) -} - -// GetHeadHeaderHash retrieves the hash of the current canonical head block's -// header. The difference between this and GetHeadBlockHash is that whereas the -// last block hash is only updated upon a full block import, the last header -// hash is updated already at header import, allowing head tracking for the -// light synchronization mechanism. -func GetHeadHeaderHash(db DatabaseReader) common.Hash { - data, _ := db.Get(headHeaderKey) - if len(data) == 0 { - return common.Hash{} - } - return common.BytesToHash(data) -} - -// GetHeadBlockHash retrieves the hash of the current canonical head block. -func GetHeadBlockHash(db DatabaseReader) common.Hash { - data, _ := db.Get(headBlockKey) - if len(data) == 0 { - return common.Hash{} - } - return common.BytesToHash(data) -} - -// GetHeadFastBlockHash retrieves the hash of the current canonical head block during -// fast synchronization. The difference between this and GetHeadBlockHash is that -// whereas the last block hash is only updated upon a full block import, the last -// fast hash is updated when importing pre-processed blocks. -func GetHeadFastBlockHash(db DatabaseReader) common.Hash { - data, _ := db.Get(headFastKey) - if len(data) == 0 { - return common.Hash{} - } - return common.BytesToHash(data) -} - -// GetTrieSyncProgress retrieves the number of tries nodes fast synced to allow -// reportinc correct numbers across restarts. -func GetTrieSyncProgress(db DatabaseReader) uint64 { - data, _ := db.Get(trieSyncKey) - if len(data) == 0 { - return 0 - } - return new(big.Int).SetBytes(data).Uint64() -} - -// GetHeaderRLP retrieves a block header in its raw RLP database encoding, or nil -// if the header's not found. -func GetHeaderRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue { - data, _ := db.Get(headerKey(hash, number)) - return data -} - -// GetHeader retrieves the block header corresponding to the hash, nil if none -// found. -func GetHeader(db DatabaseReader, hash common.Hash, number uint64) *types.Header { - data := GetHeaderRLP(db, hash, number) - if len(data) == 0 { - return nil - } - header := new(types.Header) - if err := rlp.Decode(bytes.NewReader(data), header); err != nil { - log.Error("Invalid block header RLP", "hash", hash, "err", err) - return nil - } - return header -} - -// GetBodyRLP retrieves the block body (transactions and uncles) in RLP encoding. -func GetBodyRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue { - data, _ := db.Get(blockBodyKey(hash, number)) - return data -} - -func headerKey(hash common.Hash, number uint64) []byte { - return append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...) -} - -func blockBodyKey(hash common.Hash, number uint64) []byte { - return append(append(bodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...) -} - -// GetBody retrieves the block body (transactons, uncles) corresponding to the -// hash, nil if none found. -func GetBody(db DatabaseReader, hash common.Hash, number uint64) *types.Body { - data := GetBodyRLP(db, hash, number) - if len(data) == 0 { - return nil - } - body := new(types.Body) - if err := rlp.Decode(bytes.NewReader(data), body); err != nil { - log.Error("Invalid block body RLP", "hash", hash, "err", err) - return nil - } - return body -} - -// GetTd retrieves a block's total difficulty corresponding to the hash, nil if -// none found. -func GetTd(db DatabaseReader, hash common.Hash, number uint64) *big.Int { - data, _ := db.Get(append(append(append(headerPrefix, encodeBlockNumber(number)...), hash[:]...), tdSuffix...)) - if len(data) == 0 { - return nil - } - td := new(big.Int) - if err := rlp.Decode(bytes.NewReader(data), td); err != nil { - log.Error("Invalid block total difficulty RLP", "hash", hash, "err", err) - return nil - } - return td -} - -// GetBlock retrieves an entire block corresponding to the hash, assembling it -// back from the stored header and body. If either the header or body could not -// be retrieved nil is returned. -// -// Note, due to concurrent download of header and block body the header and thus -// canonical hash can be stored in the database but the body data not (yet). -func GetBlock(db DatabaseReader, hash common.Hash, number uint64) *types.Block { - // Retrieve the block header and body contents - header := GetHeader(db, hash, number) - if header == nil { - return nil - } - body := GetBody(db, hash, number) - if body == nil { - return nil - } - // Reassemble the block and return - return types.NewBlockWithHeader(header).WithBody(body.Transactions, body.Uncles) -} - -// GetBlockReceipts retrieves the receipts generated by the transactions included -// in a block given by its hash. -func GetBlockReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Receipts { - data, _ := db.Get(append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash[:]...)) - if len(data) == 0 { - return nil - } - storageReceipts := []*types.ReceiptForStorage{} - if err := rlp.DecodeBytes(data, &storageReceipts); err != nil { - log.Error("Invalid receipt array RLP", "hash", hash, "err", err) - return nil - } - receipts := make(types.Receipts, len(storageReceipts)) - for i, receipt := range storageReceipts { - receipts[i] = (*types.Receipt)(receipt) - } - return receipts -} - -// GetTxLookupEntry retrieves the positional metadata associated with a transaction -// hash to allow retrieving the transaction or receipt by hash. -func GetTxLookupEntry(db DatabaseReader, hash common.Hash) (common.Hash, uint64, uint64) { - // Load the positional metadata from disk and bail if it fails - data, _ := db.Get(append(lookupPrefix, hash.Bytes()...)) - if len(data) == 0 { - return common.Hash{}, 0, 0 - } - // Parse and return the contents of the lookup entry - var entry TxLookupEntry - if err := rlp.DecodeBytes(data, &entry); err != nil { - log.Error("Invalid lookup entry RLP", "hash", hash, "err", err) - return common.Hash{}, 0, 0 - } - return entry.BlockHash, entry.BlockIndex, entry.Index -} - -// GetTransaction retrieves a specific transaction from the database, along with -// its added positional metadata. -func GetTransaction(db DatabaseReader, hash common.Hash) (*types.Transaction, common.Hash, uint64, uint64) { - // Retrieve the lookup metadata and resolve the transaction from the body - blockHash, blockNumber, txIndex := GetTxLookupEntry(db, hash) - - if blockHash != (common.Hash{}) { - body := GetBody(db, blockHash, blockNumber) - if body == nil || len(body.Transactions) <= int(txIndex) { - log.Error("Transaction referenced missing", "number", blockNumber, "hash", blockHash, "index", txIndex) - return nil, common.Hash{}, 0, 0 - } - return body.Transactions[txIndex], blockHash, blockNumber, txIndex - } - // Old transaction representation, load the transaction and it's metadata separately - data, _ := db.Get(hash.Bytes()) - if len(data) == 0 { - return nil, common.Hash{}, 0, 0 - } - var tx types.Transaction - if err := rlp.DecodeBytes(data, &tx); err != nil { - return nil, common.Hash{}, 0, 0 - } - // Retrieve the blockchain positional metadata - data, _ = db.Get(append(hash.Bytes(), oldTxMetaSuffix...)) - if len(data) == 0 { - return nil, common.Hash{}, 0, 0 - } - var entry TxLookupEntry - if err := rlp.DecodeBytes(data, &entry); err != nil { - return nil, common.Hash{}, 0, 0 - } - return &tx, entry.BlockHash, entry.BlockIndex, entry.Index -} - -// GetReceipt retrieves a specific transaction receipt from the database, along with -// its added positional metadata. -func GetReceipt(db DatabaseReader, hash common.Hash) (*types.Receipt, common.Hash, uint64, uint64) { - // Retrieve the lookup metadata and resolve the receipt from the receipts - blockHash, blockNumber, receiptIndex := GetTxLookupEntry(db, hash) - - if blockHash != (common.Hash{}) { - receipts := GetBlockReceipts(db, blockHash, blockNumber) - if len(receipts) <= int(receiptIndex) { - log.Error("Receipt refereced missing", "number", blockNumber, "hash", blockHash, "index", receiptIndex) - return nil, common.Hash{}, 0, 0 - } - return receipts[receiptIndex], blockHash, blockNumber, receiptIndex - } - // Old receipt representation, load the receipt and set an unknown metadata - data, _ := db.Get(append(oldReceiptsPrefix, hash[:]...)) - if len(data) == 0 { - return nil, common.Hash{}, 0, 0 - } - var receipt types.ReceiptForStorage - err := rlp.DecodeBytes(data, &receipt) - if err != nil { - log.Error("Invalid receipt RLP", "hash", hash, "err", err) - } - return (*types.Receipt)(&receipt), common.Hash{}, 0, 0 -} - -// GetBloomBits retrieves the compressed bloom bit vector belonging to the given -// section and bit index from the. -func GetBloomBits(db DatabaseReader, bit uint, section uint64, head common.Hash) ([]byte, error) { - key := append(append(bloomBitsPrefix, make([]byte, 10)...), head.Bytes()...) - - binary.BigEndian.PutUint16(key[1:], uint16(bit)) - binary.BigEndian.PutUint64(key[3:], section) - - return db.Get(key) -} - -// WriteCanonicalHash stores the canonical hash for the given block number. -func WriteCanonicalHash(db ethdb.Putter, hash common.Hash, number uint64) error { - key := append(append(headerPrefix, encodeBlockNumber(number)...), numSuffix...) - if err := db.Put(key, hash.Bytes()); err != nil { - log.Crit("Failed to store number to hash mapping", "err", err) - } - return nil -} - -// WriteHeadHeaderHash stores the head header's hash. -func WriteHeadHeaderHash(db ethdb.Putter, hash common.Hash) error { - if err := db.Put(headHeaderKey, hash.Bytes()); err != nil { - log.Crit("Failed to store last header's hash", "err", err) - } - return nil -} - -// WriteHeadBlockHash stores the head block's hash. -func WriteHeadBlockHash(db ethdb.Putter, hash common.Hash) error { - if err := db.Put(headBlockKey, hash.Bytes()); err != nil { - log.Crit("Failed to store last block's hash", "err", err) - } - return nil -} - -// WriteHeadFastBlockHash stores the fast head block's hash. -func WriteHeadFastBlockHash(db ethdb.Putter, hash common.Hash) error { - if err := db.Put(headFastKey, hash.Bytes()); err != nil { - log.Crit("Failed to store last fast block's hash", "err", err) - } - return nil -} - -// WriteTrieSyncProgress stores the fast sync trie process counter to support -// retrieving it across restarts. -func WriteTrieSyncProgress(db ethdb.Putter, count uint64) error { - if err := db.Put(trieSyncKey, new(big.Int).SetUint64(count).Bytes()); err != nil { - log.Crit("Failed to store fast sync trie progress", "err", err) - } - return nil -} - -// WriteHeader serializes a block header into the database. -func WriteHeader(db ethdb.Putter, header *types.Header) error { - data, err := rlp.EncodeToBytes(header) - if err != nil { - return err - } - hash := header.Hash().Bytes() - num := header.Number.Uint64() - encNum := encodeBlockNumber(num) - key := append(blockHashPrefix, hash...) - if err := db.Put(key, encNum); err != nil { - log.Crit("Failed to store hash to number mapping", "err", err) - } - key = append(append(headerPrefix, encNum...), hash...) - if err := db.Put(key, data); err != nil { - log.Crit("Failed to store header", "err", err) - } - return nil -} - -// WriteBody serializes the body of a block into the database. -func WriteBody(db ethdb.Putter, hash common.Hash, number uint64, body *types.Body) error { - data, err := rlp.EncodeToBytes(body) - if err != nil { - return err - } - return WriteBodyRLP(db, hash, number, data) -} - -// WriteBodyRLP writes a serialized body of a block into the database. -func WriteBodyRLP(db ethdb.Putter, hash common.Hash, number uint64, rlp rlp.RawValue) error { - key := append(append(bodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...) - if err := db.Put(key, rlp); err != nil { - log.Crit("Failed to store block body", "err", err) - } - return nil -} - -// WriteTd serializes the total difficulty of a block into the database. -func WriteTd(db ethdb.Putter, hash common.Hash, number uint64, td *big.Int) error { - data, err := rlp.EncodeToBytes(td) - if err != nil { - return err - } - key := append(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...), tdSuffix...) - if err := db.Put(key, data); err != nil { - log.Crit("Failed to store block total difficulty", "err", err) - } - return nil -} - -// WriteBlock serializes a block into the database, header and body separately. -func WriteBlock(db ethdb.Putter, block *types.Block) error { - // Store the body first to retain database consistency - if err := WriteBody(db, block.Hash(), block.NumberU64(), block.Body()); err != nil { - return err - } - // Store the header too, signaling full block ownership - if err := WriteHeader(db, block.Header()); err != nil { - return err - } - return nil -} - -// WriteBlockReceipts stores all the transaction receipts belonging to a block -// as a single receipt slice. This is used during chain reorganisations for -// rescheduling dropped transactions. -func WriteBlockReceipts(db ethdb.Putter, hash common.Hash, number uint64, receipts types.Receipts) error { - // Convert the receipts into their storage form and serialize them - storageReceipts := make([]*types.ReceiptForStorage, len(receipts)) - for i, receipt := range receipts { - storageReceipts[i] = (*types.ReceiptForStorage)(receipt) - } - bytes, err := rlp.EncodeToBytes(storageReceipts) - if err != nil { - return err - } - // Store the flattened receipt slice - key := append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash.Bytes()...) - if err := db.Put(key, bytes); err != nil { - log.Crit("Failed to store block receipts", "err", err) - } - return nil -} - -// WriteTxLookupEntries stores a positional metadata for every transaction from -// a block, enabling hash based transaction and receipt lookups. -func WriteTxLookupEntries(db ethdb.Putter, block *types.Block) error { - // Iterate over each transaction and encode its metadata - for i, tx := range block.Transactions() { - entry := TxLookupEntry{ - BlockHash: block.Hash(), - BlockIndex: block.NumberU64(), - Index: uint64(i), - } - data, err := rlp.EncodeToBytes(entry) - if err != nil { - return err - } - if err := db.Put(append(lookupPrefix, tx.Hash().Bytes()...), data); err != nil { - return err - } - } - return nil -} - -// WriteBloomBits writes the compressed bloom bits vector belonging to the given -// section and bit index. -func WriteBloomBits(db ethdb.Putter, bit uint, section uint64, head common.Hash, bits []byte) { - key := append(append(bloomBitsPrefix, make([]byte, 10)...), head.Bytes()...) - - binary.BigEndian.PutUint16(key[1:], uint16(bit)) - binary.BigEndian.PutUint64(key[3:], section) - - if err := db.Put(key, bits); err != nil { - log.Crit("Failed to store bloom bits", "err", err) - } -} - -// DeleteCanonicalHash removes the number to hash canonical mapping. -func DeleteCanonicalHash(db DatabaseDeleter, number uint64) { - db.Delete(append(append(headerPrefix, encodeBlockNumber(number)...), numSuffix...)) -} - -// DeleteHeader removes all block header data associated with a hash. -func DeleteHeader(db DatabaseDeleter, hash common.Hash, number uint64) { - db.Delete(append(blockHashPrefix, hash.Bytes()...)) - db.Delete(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) -} - -// DeleteBody removes all block body data associated with a hash. -func DeleteBody(db DatabaseDeleter, hash common.Hash, number uint64) { - db.Delete(append(append(bodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) -} - -// DeleteTd removes all block total difficulty data associated with a hash. -func DeleteTd(db DatabaseDeleter, hash common.Hash, number uint64) { - db.Delete(append(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...), tdSuffix...)) -} - -// DeleteBlock removes all block data associated with a hash. -func DeleteBlock(db DatabaseDeleter, hash common.Hash, number uint64) { - DeleteBlockReceipts(db, hash, number) - DeleteHeader(db, hash, number) - DeleteBody(db, hash, number) - DeleteTd(db, hash, number) -} - -// DeleteBlockReceipts removes all receipt data associated with a block hash. -func DeleteBlockReceipts(db DatabaseDeleter, hash common.Hash, number uint64) { - db.Delete(append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) -} - -// DeleteTxLookupEntry removes all transaction data associated with a hash. -func DeleteTxLookupEntry(db DatabaseDeleter, hash common.Hash) { - db.Delete(append(lookupPrefix, hash.Bytes()...)) -} - -// PreimageTable returns a Database instance with the key prefix for preimage entries. -func PreimageTable(db ethdb.Database) ethdb.Database { - return ethdb.NewTable(db, preimagePrefix) -} - -// WritePreimages writes the provided set of preimages to the database. `number` is the -// current block number, and is used for debug messages only. -func WritePreimages(db ethdb.Database, number uint64, preimages map[common.Hash][]byte) error { - table := PreimageTable(db) - batch := table.NewBatch() - hitCount := 0 - for hash, preimage := range preimages { - if _, err := table.Get(hash.Bytes()); err != nil { - batch.Put(hash.Bytes(), preimage) - hitCount++ - } - } - preimageCounter.Inc(int64(len(preimages))) - preimageHitCounter.Inc(int64(hitCount)) - if hitCount > 0 { - if err := batch.Write(); err != nil { - return fmt.Errorf("preimage write fail for block %d: %v", number, err) - } - } - return nil -} - -// GetBlockChainVersion reads the version number from db. -func GetBlockChainVersion(db DatabaseReader) int { - var vsn uint - enc, _ := db.Get([]byte("BlockchainVersion")) - rlp.DecodeBytes(enc, &vsn) - return int(vsn) -} - -// WriteBlockChainVersion writes vsn as the version number to db. -func WriteBlockChainVersion(db ethdb.Putter, vsn int) { - enc, _ := rlp.EncodeToBytes(uint(vsn)) - db.Put([]byte("BlockchainVersion"), enc) -} - -// WriteChainConfig writes the chain config settings to the database. -func WriteChainConfig(db ethdb.Putter, hash common.Hash, cfg *params.ChainConfig) error { - // short circuit and ignore if nil config. GetChainConfig - // will return a default. - if cfg == nil { - return nil - } - - jsonChainConfig, err := json.Marshal(cfg) - if err != nil { - return err - } - - return db.Put(append(configPrefix, hash[:]...), jsonChainConfig) -} - -// GetChainConfig will fetch the network settings based on the given hash. -func GetChainConfig(db DatabaseReader, hash common.Hash) (*params.ChainConfig, error) { - jsonChainConfig, _ := db.Get(append(configPrefix, hash[:]...)) - if len(jsonChainConfig) == 0 { - return nil, ErrChainConfigNotFound - } - - var config params.ChainConfig - if err := json.Unmarshal(jsonChainConfig, &config); err != nil { - return nil, err - } - - return &config, nil -} - -// FindCommonAncestor returns the last common ancestor of two block headers -func FindCommonAncestor(db DatabaseReader, a, b *types.Header) *types.Header { - for bn := b.Number.Uint64(); a.Number.Uint64() > bn; { - a = GetHeader(db, a.ParentHash, a.Number.Uint64()-1) - if a == nil { - return nil - } - } - for an := a.Number.Uint64(); an < b.Number.Uint64(); { - b = GetHeader(db, b.ParentHash, b.Number.Uint64()-1) - if b == nil { - return nil - } - } - for a.Hash() != b.Hash() { - a = GetHeader(db, a.ParentHash, a.Number.Uint64()-1) - if a == nil { - return nil - } - b = GetHeader(db, b.ParentHash, b.Number.Uint64()-1) - if b == nil { - return nil - } - } - return a -} diff --git a/core/genesis.go b/core/genesis.go index b6ead2250a..9190e2ba22 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" @@ -155,7 +156,7 @@ func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig } // Just commit the new block if there is no stored genesis block. - stored := GetCanonicalHash(db, 0) + stored := rawdb.ReadCanonicalHash(db, 0) if (stored == common.Hash{}) { if genesis == nil { log.Info("Writing default main-net genesis block") @@ -177,14 +178,11 @@ func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig // Get the existing chain configuration. newcfg := genesis.configOrDefault(stored) - storedcfg, err := GetChainConfig(db, stored) - if err != nil { - if err == ErrChainConfigNotFound { - // This case happens if a genesis write was interrupted. - log.Warn("Found genesis block without chain config") - err = WriteChainConfig(db, stored, newcfg) - } - return newcfg, stored, err + storedcfg := rawdb.ReadChainConfig(db, stored) + if storedcfg == nil { + log.Warn("Found genesis block without chain config") + rawdb.WriteChainConfig(db, stored, newcfg) + return newcfg, stored, nil } // Special case: don't change the existing config of a non-mainnet chain if no new // config is supplied. These chains would get AllProtocolChanges (and a compat error) @@ -195,15 +193,16 @@ func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig // Check config compatibility and write the config. Compatibility errors // are returned to the caller unless we're already at block zero. - height := GetBlockNumber(db, GetHeadHeaderHash(db)) - if height == missingNumber { + height := rawdb.ReadHeaderNumber(db, rawdb.ReadHeadHeaderHash(db)) + if height == nil { return newcfg, stored, fmt.Errorf("missing block number for head header hash") } - compatErr := storedcfg.CheckCompatible(newcfg, height) - if compatErr != nil && height != 0 && compatErr.RewindTo != 0 { + compatErr := storedcfg.CheckCompatible(newcfg, *height) + if compatErr != nil && *height != 0 && compatErr.RewindTo != 0 { return newcfg, stored, compatErr } - return newcfg, stored, WriteChainConfig(db, stored, newcfg) + rawdb.WriteChainConfig(db, stored, newcfg) + return newcfg, stored, nil } func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig { @@ -223,7 +222,7 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig { // to the given database (or discards it if nil). func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { if db == nil { - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() } statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) for addr, account := range g.Alloc { @@ -267,29 +266,19 @@ func (g *Genesis) Commit(db ethdb.Database) (*types.Block, error) { if block.Number().Sign() != 0 { return nil, fmt.Errorf("can't commit genesis block with number > 0") } - if err := WriteTd(db, block.Hash(), block.NumberU64(), g.Difficulty); err != nil { - return nil, err - } - if err := WriteBlock(db, block); err != nil { - return nil, err - } - if err := WriteBlockReceipts(db, block.Hash(), block.NumberU64(), nil); err != nil { - return nil, err - } - if err := WriteCanonicalHash(db, block.Hash(), block.NumberU64()); err != nil { - return nil, err - } - if err := WriteHeadBlockHash(db, block.Hash()); err != nil { - return nil, err - } - if err := WriteHeadHeaderHash(db, block.Hash()); err != nil { - return nil, err - } + rawdb.WriteTd(db, block.Hash(), block.NumberU64(), g.Difficulty) + rawdb.WriteBlock(db, block) + rawdb.WriteReceipts(db, block.Hash(), block.NumberU64(), nil) + rawdb.WriteCanonicalHash(db, block.Hash(), block.NumberU64()) + rawdb.WriteHeadBlockHash(db, block.Hash()) + rawdb.WriteHeadHeaderHash(db, block.Hash()) + config := g.Config if config == nil { config = params.AllEthashProtocolChanges } - return block, WriteChainConfig(db, block.Hash(), config) + rawdb.WriteChainConfig(db, block.Hash(), config) + return block, nil } // MustCommit writes the genesis block and state to db, panicking on error. diff --git a/core/genesis_test.go b/core/genesis_test.go index 052ded6991..2d7f94f8f8 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -24,6 +24,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" @@ -140,7 +141,7 @@ func TestSetupGenesis(t *testing.T) { } for _, test := range tests { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() config, hash, err := test.fn(db) // Check the return values. if !reflect.DeepEqual(err, test.wantErr) { @@ -154,7 +155,7 @@ func TestSetupGenesis(t *testing.T) { t.Errorf("%s: returned hash %s, want %s", test.name, hash.Hex(), test.wantHash.Hex()) } else if err == nil { // Check database content. - stored := GetBlock(db, test.wantHash, 0) + stored := rawdb.ReadBlock(db, test.wantHash, 0) if stored.Hash() != test.wantHash { t.Errorf("%s: block in DB has hash %s, want %s", test.name, stored.Hash(), test.wantHash) } diff --git a/core/headerchain.go b/core/headerchain.go index 2d1b0a2a18..2ac0cccc72 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" @@ -97,7 +98,7 @@ func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, engine c } hc.currentHeader.Store(hc.genesisHeader) - if head := GetHeadBlockHash(chainDb); head != (common.Hash{}) { + if head := rawdb.ReadHeadBlockHash(chainDb); head != (common.Hash{}) { if chead := hc.GetHeaderByHash(head); chead != nil { hc.currentHeader.Store(chead) } @@ -109,13 +110,14 @@ func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, engine c // GetBlockNumber retrieves the block number belonging to the given hash // from the cache or database -func (hc *HeaderChain) GetBlockNumber(hash common.Hash) uint64 { +func (hc *HeaderChain) GetBlockNumber(hash common.Hash) *uint64 { if cached, ok := hc.numberCache.Get(hash); ok { - return cached.(uint64) + number := cached.(uint64) + return &number } - number := GetBlockNumber(hc.chainDb, hash) - if number != missingNumber { - hc.numberCache.Add(hash, number) + number := rawdb.ReadHeaderNumber(hc.chainDb, hash) + if number != nil { + hc.numberCache.Add(hash, *number) } return number } @@ -147,20 +149,19 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er if err := hc.WriteTd(hash, number, externTd); err != nil { log.Crit("Failed to write header total difficulty", "err", err) } - if err := WriteHeader(hc.chainDb, header); err != nil { - log.Crit("Failed to write header content", "err", err) - } + rawdb.WriteHeader(hc.chainDb, header) + // If the total difficulty is higher than our known, add it to the canonical chain // Second clause in the if statement reduces the vulnerability to selfish mining. // Please refer to http://www.cs.cornell.edu/~ie53/publications/btcProcFC.pdf if externTd.Cmp(localTd) > 0 || (externTd.Cmp(localTd) == 0 && mrand.Float64() < 0.5) { // Delete any canonical number assignments above the new head for i := number + 1; ; i++ { - hash := GetCanonicalHash(hc.chainDb, i) + hash := rawdb.ReadCanonicalHash(hc.chainDb, i) if hash == (common.Hash{}) { break } - DeleteCanonicalHash(hc.chainDb, i) + rawdb.DeleteCanonicalHash(hc.chainDb, i) } // Overwrite any stale canonical number assignments var ( @@ -168,20 +169,17 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er headNumber = header.Number.Uint64() - 1 headHeader = hc.GetHeader(headHash, headNumber) ) - for GetCanonicalHash(hc.chainDb, headNumber) != headHash { - WriteCanonicalHash(hc.chainDb, headHash, headNumber) + for rawdb.ReadCanonicalHash(hc.chainDb, headNumber) != headHash { + rawdb.WriteCanonicalHash(hc.chainDb, headHash, headNumber) headHash = headHeader.ParentHash headNumber = headHeader.Number.Uint64() - 1 headHeader = hc.GetHeader(headHash, headNumber) } // Extend the canonical chain with the new header - if err := WriteCanonicalHash(hc.chainDb, hash, number); err != nil { - log.Crit("Failed to insert header number", "err", err) - } - if err := WriteHeadHeaderHash(hc.chainDb, hash); err != nil { - log.Crit("Failed to insert head header hash", "err", err) - } + rawdb.WriteCanonicalHash(hc.chainDb, hash, number) + rawdb.WriteHeadHeaderHash(hc.chainDb, hash) + hc.currentHeaderHash = hash hc.currentHeader.Store(types.CopyHeader(header)) @@ -316,7 +314,7 @@ func (hc *HeaderChain) GetTd(hash common.Hash, number uint64) *big.Int { if cached, ok := hc.tdCache.Get(hash); ok { return cached.(*big.Int) } - td := GetTd(hc.chainDb, hash, number) + td := rawdb.ReadTd(hc.chainDb, hash, number) if td == nil { return nil } @@ -328,15 +326,17 @@ func (hc *HeaderChain) GetTd(hash common.Hash, number uint64) *big.Int { // GetTdByHash retrieves a block's total difficulty in the canonical chain from the // database by hash, caching it if found. func (hc *HeaderChain) GetTdByHash(hash common.Hash) *big.Int { - return hc.GetTd(hash, hc.GetBlockNumber(hash)) + number := hc.GetBlockNumber(hash) + if number == nil { + return nil + } + return hc.GetTd(hash, *number) } // WriteTd stores a block's total difficulty into the database, also caching it // along the way. func (hc *HeaderChain) WriteTd(hash common.Hash, number uint64, td *big.Int) error { - if err := WriteTd(hc.chainDb, hash, number, td); err != nil { - return err - } + rawdb.WriteTd(hc.chainDb, hash, number, td) hc.tdCache.Add(hash, new(big.Int).Set(td)) return nil } @@ -348,7 +348,7 @@ func (hc *HeaderChain) GetHeader(hash common.Hash, number uint64) *types.Header if header, ok := hc.headerCache.Get(hash); ok { return header.(*types.Header) } - header := GetHeader(hc.chainDb, hash, number) + header := rawdb.ReadHeader(hc.chainDb, hash, number) if header == nil { return nil } @@ -360,7 +360,11 @@ func (hc *HeaderChain) GetHeader(hash common.Hash, number uint64) *types.Header // GetHeaderByHash retrieves a block header from the database by hash, caching it if // found. func (hc *HeaderChain) GetHeaderByHash(hash common.Hash) *types.Header { - return hc.GetHeader(hash, hc.GetBlockNumber(hash)) + number := hc.GetBlockNumber(hash) + if number == nil { + return nil + } + return hc.GetHeader(hash, *number) } // HasHeader checks if a block header is present in the database or not. @@ -368,14 +372,13 @@ func (hc *HeaderChain) HasHeader(hash common.Hash, number uint64) bool { if hc.numberCache.Contains(hash) || hc.headerCache.Contains(hash) { return true } - ok, _ := hc.chainDb.Has(headerKey(hash, number)) - return ok + return rawdb.HasHeader(hc.chainDb, hash, number) } // GetHeaderByNumber retrieves a block header from the database by number, // caching it (associated with its hash) if found. func (hc *HeaderChain) GetHeaderByNumber(number uint64) *types.Header { - hash := GetCanonicalHash(hc.chainDb, number) + hash := rawdb.ReadCanonicalHash(hc.chainDb, number) if hash == (common.Hash{}) { return nil } @@ -390,9 +393,8 @@ func (hc *HeaderChain) CurrentHeader() *types.Header { // SetCurrentHeader sets the current head header of the canonical chain. func (hc *HeaderChain) SetCurrentHeader(head *types.Header) { - if err := WriteHeadHeaderHash(hc.chainDb, head.Hash()); err != nil { - log.Crit("Failed to insert head header hash", "err", err) - } + rawdb.WriteHeadHeaderHash(hc.chainDb, head.Hash()) + hc.currentHeader.Store(head) hc.currentHeaderHash = head.Hash() } @@ -416,13 +418,14 @@ func (hc *HeaderChain) SetHead(head uint64, delFn DeleteCallback) { if delFn != nil { delFn(hash, num) } - DeleteHeader(hc.chainDb, hash, num) - DeleteTd(hc.chainDb, hash, num) + rawdb.DeleteHeader(hc.chainDb, hash, num) + rawdb.DeleteTd(hc.chainDb, hash, num) + hc.currentHeader.Store(hc.GetHeader(hdr.ParentHash, hdr.Number.Uint64()-1)) } // Roll back the canonical chain numbering for i := height; i > head; i-- { - DeleteCanonicalHash(hc.chainDb, i) + rawdb.DeleteCanonicalHash(hc.chainDb, i) } // Clear out any stale content from the caches hc.headerCache.Purge() @@ -434,9 +437,7 @@ func (hc *HeaderChain) SetHead(head uint64, delFn DeleteCallback) { } hc.currentHeaderHash = hc.CurrentHeader().Hash() - if err := WriteHeadHeaderHash(hc.chainDb, hc.currentHeaderHash); err != nil { - log.Crit("Failed to reset head header hash", "err", err) - } + rawdb.WriteHeadHeaderHash(hc.chainDb, hc.currentHeaderHash) } // SetGenesis sets a new genesis block header for the chain diff --git a/core/helper_test.go b/core/helper_test.go index 698a2924cb..051384d854 100644 --- a/core/helper_test.go +++ b/core/helper_test.go @@ -18,7 +18,6 @@ package core import ( "container/list" - "fmt" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" @@ -77,18 +76,11 @@ func (tm *TestManager) Db() ethdb.Database { } func NewTestManager() *TestManager { - db, err := ethdb.NewMemDatabase() - if err != nil { - fmt.Println("Could not create mem-db, failing") - return nil - } - testManager := &TestManager{} testManager.eventMux = new(event.TypeMux) - testManager.db = db + testManager.db = ethdb.NewMemDatabase() // testManager.txPool = NewTxPool(testManager) // testManager.blockChain = NewBlockChain(testManager) // testManager.stateManager = NewStateManager(testManager) - return testManager } diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go new file mode 100644 index 0000000000..a26a42ba7c --- /dev/null +++ b/core/rawdb/accessors_chain.go @@ -0,0 +1,381 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "bytes" + "encoding/binary" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" +) + +// ReadCanonicalHash retrieves the hash assigned to a canonical block number. +func ReadCanonicalHash(db DatabaseReader, number uint64) common.Hash { + data, _ := db.Get(append(append(headerPrefix, encodeBlockNumber(number)...), headerHashSuffix...)) + if len(data) == 0 { + return common.Hash{} + } + return common.BytesToHash(data) +} + +// WriteCanonicalHash stores the hash assigned to a canonical block number. +func WriteCanonicalHash(db DatabaseWriter, hash common.Hash, number uint64) { + key := append(append(headerPrefix, encodeBlockNumber(number)...), headerHashSuffix...) + if err := db.Put(key, hash.Bytes()); err != nil { + log.Crit("Failed to store number to hash mapping", "err", err) + } +} + +// DeleteCanonicalHash removes the number to hash canonical mapping. +func DeleteCanonicalHash(db DatabaseDeleter, number uint64) { + if err := db.Delete(append(append(headerPrefix, encodeBlockNumber(number)...), headerHashSuffix...)); err != nil { + log.Crit("Failed to delete number to hash mapping", "err", err) + } +} + +// ReadHeaderNumber returns the header number assigned to a hash. +func ReadHeaderNumber(db DatabaseReader, hash common.Hash) *uint64 { + data, _ := db.Get(append(headerNumberPrefix, hash.Bytes()...)) + if len(data) != 8 { + return nil + } + number := binary.BigEndian.Uint64(data) + return &number +} + +// ReadHeadHeaderHash retrieves the hash of the current canonical head header. +func ReadHeadHeaderHash(db DatabaseReader) common.Hash { + data, _ := db.Get(headHeaderKey) + if len(data) == 0 { + return common.Hash{} + } + return common.BytesToHash(data) +} + +// WriteHeadHeaderHash stores the hash of the current canonical head header. +func WriteHeadHeaderHash(db DatabaseWriter, hash common.Hash) { + if err := db.Put(headHeaderKey, hash.Bytes()); err != nil { + log.Crit("Failed to store last header's hash", "err", err) + } +} + +// ReadHeadBlockHash retrieves the hash of the current canonical head block. +func ReadHeadBlockHash(db DatabaseReader) common.Hash { + data, _ := db.Get(headBlockKey) + if len(data) == 0 { + return common.Hash{} + } + return common.BytesToHash(data) +} + +// WriteHeadBlockHash stores the head block's hash. +func WriteHeadBlockHash(db DatabaseWriter, hash common.Hash) { + if err := db.Put(headBlockKey, hash.Bytes()); err != nil { + log.Crit("Failed to store last block's hash", "err", err) + } +} + +// ReadHeadFastBlockHash retrieves the hash of the current fast-sync head block. +func ReadHeadFastBlockHash(db DatabaseReader) common.Hash { + data, _ := db.Get(headFastBlockKey) + if len(data) == 0 { + return common.Hash{} + } + return common.BytesToHash(data) +} + +// WriteHeadFastBlockHash stores the hash of the current fast-sync head block. +func WriteHeadFastBlockHash(db DatabaseWriter, hash common.Hash) { + if err := db.Put(headFastBlockKey, hash.Bytes()); err != nil { + log.Crit("Failed to store last fast block's hash", "err", err) + } +} + +// ReadFastTrieProgress retrieves the number of tries nodes fast synced to allow +// reporting correct numbers across restarts. +func ReadFastTrieProgress(db DatabaseReader) uint64 { + data, _ := db.Get(fastTrieProgressKey) + if len(data) == 0 { + return 0 + } + return new(big.Int).SetBytes(data).Uint64() +} + +// WriteFastTrieProgress stores the fast sync trie process counter to support +// retrieving it across restarts. +func WriteFastTrieProgress(db DatabaseWriter, count uint64) { + if err := db.Put(fastTrieProgressKey, new(big.Int).SetUint64(count).Bytes()); err != nil { + log.Crit("Failed to store fast sync trie progress", "err", err) + } +} + +// ReadHeaderRLP retrieves a block header in its raw RLP database encoding. +func ReadHeaderRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue { + data, _ := db.Get(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) + return data +} + +// HasHeader verifies the existence of a block header corresponding to the hash. +func HasHeader(db DatabaseReader, hash common.Hash, number uint64) bool { + key := append(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) + if has, err := db.Has(key); !has || err != nil { + return false + } + return true +} + +// ReadHeader retrieves the block header corresponding to the hash. +func ReadHeader(db DatabaseReader, hash common.Hash, number uint64) *types.Header { + data := ReadHeaderRLP(db, hash, number) + if len(data) == 0 { + return nil + } + header := new(types.Header) + if err := rlp.Decode(bytes.NewReader(data), header); err != nil { + log.Error("Invalid block header RLP", "hash", hash, "err", err) + return nil + } + return header +} + +// WriteHeader stores a block header into the database and also stores the hash- +// to-number mapping. +func WriteHeader(db DatabaseWriter, header *types.Header) { + // Write the hash -> number mapping + var ( + hash = header.Hash().Bytes() + number = header.Number.Uint64() + encoded = encodeBlockNumber(number) + ) + key := append(headerNumberPrefix, hash...) + if err := db.Put(key, encoded); err != nil { + log.Crit("Failed to store hash to number mapping", "err", err) + } + // Write the encoded header + data, err := rlp.EncodeToBytes(header) + if err != nil { + log.Crit("Failed to RLP encode header", "err", err) + } + key = append(append(headerPrefix, encoded...), hash...) + if err := db.Put(key, data); err != nil { + log.Crit("Failed to store header", "err", err) + } +} + +// DeleteHeader removes all block header data associated with a hash. +func DeleteHeader(db DatabaseDeleter, hash common.Hash, number uint64) { + if err := db.Delete(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...)); err != nil { + log.Crit("Failed to delete header", "err", err) + } + if err := db.Delete(append(headerNumberPrefix, hash.Bytes()...)); err != nil { + log.Crit("Failed to delete hash to number mapping", "err", err) + } +} + +// ReadBodyRLP retrieves the block body (transactions and uncles) in RLP encoding. +func ReadBodyRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue { + data, _ := db.Get(append(append(blockBodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) + return data +} + +// WriteBodyRLP stores an RLP encoded block body into the database. +func WriteBodyRLP(db DatabaseWriter, hash common.Hash, number uint64, rlp rlp.RawValue) { + key := append(append(blockBodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...) + if err := db.Put(key, rlp); err != nil { + log.Crit("Failed to store block body", "err", err) + } +} + +// HasBody verifies the existence of a block body corresponding to the hash. +func HasBody(db DatabaseReader, hash common.Hash, number uint64) bool { + key := append(append(blockBodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...) + if has, err := db.Has(key); !has || err != nil { + return false + } + return true +} + +// ReadBody retrieves the block body corresponding to the hash. +func ReadBody(db DatabaseReader, hash common.Hash, number uint64) *types.Body { + data := ReadBodyRLP(db, hash, number) + if len(data) == 0 { + return nil + } + body := new(types.Body) + if err := rlp.Decode(bytes.NewReader(data), body); err != nil { + log.Error("Invalid block body RLP", "hash", hash, "err", err) + return nil + } + return body +} + +// WriteBody storea a block body into the database. +func WriteBody(db DatabaseWriter, hash common.Hash, number uint64, body *types.Body) { + data, err := rlp.EncodeToBytes(body) + if err != nil { + log.Crit("Failed to RLP encode body", "err", err) + } + WriteBodyRLP(db, hash, number, data) +} + +// DeleteBody removes all block body data associated with a hash. +func DeleteBody(db DatabaseDeleter, hash common.Hash, number uint64) { + if err := db.Delete(append(append(blockBodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...)); err != nil { + log.Crit("Failed to delete block body", "err", err) + } +} + +// ReadTd retrieves a block's total difficulty corresponding to the hash. +func ReadTd(db DatabaseReader, hash common.Hash, number uint64) *big.Int { + data, _ := db.Get(append(append(append(headerPrefix, encodeBlockNumber(number)...), hash[:]...), headerTDSuffix...)) + if len(data) == 0 { + return nil + } + td := new(big.Int) + if err := rlp.Decode(bytes.NewReader(data), td); err != nil { + log.Error("Invalid block total difficulty RLP", "hash", hash, "err", err) + return nil + } + return td +} + +// WriteTd stores the total difficulty of a block into the database. +func WriteTd(db DatabaseWriter, hash common.Hash, number uint64, td *big.Int) { + data, err := rlp.EncodeToBytes(td) + if err != nil { + log.Crit("Failed to RLP encode block total difficulty", "err", err) + } + key := append(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...), headerTDSuffix...) + if err := db.Put(key, data); err != nil { + log.Crit("Failed to store block total difficulty", "err", err) + } +} + +// DeleteTd removes all block total difficulty data associated with a hash. +func DeleteTd(db DatabaseDeleter, hash common.Hash, number uint64) { + if err := db.Delete(append(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...), headerTDSuffix...)); err != nil { + log.Crit("Failed to delete block total difficulty", "err", err) + } +} + +// ReadReceipts retrieves all the transaction receipts belonging to a block. +func ReadReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Receipts { + // Retrieve the flattened receipt slice + data, _ := db.Get(append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash[:]...)) + if len(data) == 0 { + return nil + } + // Convert the revceipts from their storage form to their internal representation + storageReceipts := []*types.ReceiptForStorage{} + if err := rlp.DecodeBytes(data, &storageReceipts); err != nil { + log.Error("Invalid receipt array RLP", "hash", hash, "err", err) + return nil + } + receipts := make(types.Receipts, len(storageReceipts)) + for i, receipt := range storageReceipts { + receipts[i] = (*types.Receipt)(receipt) + } + return receipts +} + +// WriteReceipts stores all the transaction receipts belonging to a block. +func WriteReceipts(db DatabaseWriter, hash common.Hash, number uint64, receipts types.Receipts) { + // Convert the receipts into their storage form and serialize them + storageReceipts := make([]*types.ReceiptForStorage, len(receipts)) + for i, receipt := range receipts { + storageReceipts[i] = (*types.ReceiptForStorage)(receipt) + } + bytes, err := rlp.EncodeToBytes(storageReceipts) + if err != nil { + log.Crit("Failed to encode block receipts", "err", err) + } + // Store the flattened receipt slice + key := append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash.Bytes()...) + if err := db.Put(key, bytes); err != nil { + log.Crit("Failed to store block receipts", "err", err) + } +} + +// DeleteReceipts removes all receipt data associated with a block hash. +func DeleteReceipts(db DatabaseDeleter, hash common.Hash, number uint64) { + if err := db.Delete(append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash.Bytes()...)); err != nil { + log.Crit("Failed to delete block receipts", "err", err) + } +} + +// ReadBlock retrieves an entire block corresponding to the hash, assembling it +// back from the stored header and body. If either the header or body could not +// be retrieved nil is returned. +// +// Note, due to concurrent download of header and block body the header and thus +// canonical hash can be stored in the database but the body data not (yet). +func ReadBlock(db DatabaseReader, hash common.Hash, number uint64) *types.Block { + header := ReadHeader(db, hash, number) + if header == nil { + return nil + } + body := ReadBody(db, hash, number) + if body == nil { + return nil + } + return types.NewBlockWithHeader(header).WithBody(body.Transactions, body.Uncles) +} + +// WriteBlock serializes a block into the database, header and body separately. +func WriteBlock(db DatabaseWriter, block *types.Block) { + WriteBody(db, block.Hash(), block.NumberU64(), block.Body()) + WriteHeader(db, block.Header()) +} + +// DeleteBlock removes all block data associated with a hash. +func DeleteBlock(db DatabaseDeleter, hash common.Hash, number uint64) { + DeleteReceipts(db, hash, number) + DeleteHeader(db, hash, number) + DeleteBody(db, hash, number) + DeleteTd(db, hash, number) +} + +// FindCommonAncestor returns the last common ancestor of two block headers +func FindCommonAncestor(db DatabaseReader, a, b *types.Header) *types.Header { + for bn := b.Number.Uint64(); a.Number.Uint64() > bn; { + a = ReadHeader(db, a.ParentHash, a.Number.Uint64()-1) + if a == nil { + return nil + } + } + for an := a.Number.Uint64(); an < b.Number.Uint64(); { + b = ReadHeader(db, b.ParentHash, b.Number.Uint64()-1) + if b == nil { + return nil + } + } + for a.Hash() != b.Hash() { + a = ReadHeader(db, a.ParentHash, a.Number.Uint64()-1) + if a == nil { + return nil + } + b = ReadHeader(db, b.ParentHash, b.Number.Uint64()-1) + if b == nil { + return nil + } + } + return a +} diff --git a/core/database_util_test.go b/core/rawdb/accessors_chain_test.go similarity index 59% rename from core/database_util_test.go rename to core/rawdb/accessors_chain_test.go index aa87fa6f86..9ddae6e2b5 100644 --- a/core/database_util_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 The go-ethereum Authors +// Copyright 2018 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package core +package rawdb import ( "bytes" @@ -30,23 +30,21 @@ import ( // Tests block header storage and retrieval operations. func TestHeaderStorage(t *testing.T) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() // Create a test header to move around the database and make sure it's really new header := &types.Header{Number: big.NewInt(42), Extra: []byte("test header")} - if entry := GetHeader(db, header.Hash(), header.Number.Uint64()); entry != nil { + if entry := ReadHeader(db, header.Hash(), header.Number.Uint64()); entry != nil { t.Fatalf("Non existent header returned: %v", entry) } // Write and verify the header in the database - if err := WriteHeader(db, header); err != nil { - t.Fatalf("Failed to write header into database: %v", err) - } - if entry := GetHeader(db, header.Hash(), header.Number.Uint64()); entry == nil { + WriteHeader(db, header) + if entry := ReadHeader(db, header.Hash(), header.Number.Uint64()); entry == nil { t.Fatalf("Stored header not found") } else if entry.Hash() != header.Hash() { t.Fatalf("Retrieved header mismatch: have %v, want %v", entry, header) } - if entry := GetHeaderRLP(db, header.Hash(), header.Number.Uint64()); entry == nil { + if entry := ReadHeaderRLP(db, header.Hash(), header.Number.Uint64()); entry == nil { t.Fatalf("Stored header RLP not found") } else { hasher := sha3.NewKeccak256() @@ -58,14 +56,14 @@ func TestHeaderStorage(t *testing.T) { } // Delete the header and verify the execution DeleteHeader(db, header.Hash(), header.Number.Uint64()) - if entry := GetHeader(db, header.Hash(), header.Number.Uint64()); entry != nil { + if entry := ReadHeader(db, header.Hash(), header.Number.Uint64()); entry != nil { t.Fatalf("Deleted header returned: %v", entry) } } // Tests block body storage and retrieval operations. func TestBodyStorage(t *testing.T) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() // Create a test body to move around the database and make sure it's really new body := &types.Body{Uncles: []*types.Header{{Extra: []byte("test header")}}} @@ -74,19 +72,17 @@ func TestBodyStorage(t *testing.T) { rlp.Encode(hasher, body) hash := common.BytesToHash(hasher.Sum(nil)) - if entry := GetBody(db, hash, 0); entry != nil { + if entry := ReadBody(db, hash, 0); entry != nil { t.Fatalf("Non existent body returned: %v", entry) } // Write and verify the body in the database - if err := WriteBody(db, hash, 0, body); err != nil { - t.Fatalf("Failed to write body into database: %v", err) - } - if entry := GetBody(db, hash, 0); entry == nil { + WriteBody(db, hash, 0, body) + if entry := ReadBody(db, hash, 0); entry == nil { t.Fatalf("Stored body not found") } else if types.DeriveSha(types.Transactions(entry.Transactions)) != types.DeriveSha(types.Transactions(body.Transactions)) || types.CalcUncleHash(entry.Uncles) != types.CalcUncleHash(body.Uncles) { t.Fatalf("Retrieved body mismatch: have %v, want %v", entry, body) } - if entry := GetBodyRLP(db, hash, 0); entry == nil { + if entry := ReadBodyRLP(db, hash, 0); entry == nil { t.Fatalf("Stored body RLP not found") } else { hasher := sha3.NewKeccak256() @@ -98,14 +94,14 @@ func TestBodyStorage(t *testing.T) { } // Delete the body and verify the execution DeleteBody(db, hash, 0) - if entry := GetBody(db, hash, 0); entry != nil { + if entry := ReadBody(db, hash, 0); entry != nil { t.Fatalf("Deleted body returned: %v", entry) } } // Tests block storage and retrieval operations. func TestBlockStorage(t *testing.T) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() // Create a test block to move around the database and make sure it's really new block := types.NewBlockWithHeader(&types.Header{ @@ -114,50 +110,48 @@ func TestBlockStorage(t *testing.T) { TxHash: types.EmptyRootHash, ReceiptHash: types.EmptyRootHash, }) - if entry := GetBlock(db, block.Hash(), block.NumberU64()); entry != nil { + if entry := ReadBlock(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Non existent block returned: %v", entry) } - if entry := GetHeader(db, block.Hash(), block.NumberU64()); entry != nil { + if entry := ReadHeader(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Non existent header returned: %v", entry) } - if entry := GetBody(db, block.Hash(), block.NumberU64()); entry != nil { + if entry := ReadBody(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Non existent body returned: %v", entry) } // Write and verify the block in the database - if err := WriteBlock(db, block); err != nil { - t.Fatalf("Failed to write block into database: %v", err) - } - if entry := GetBlock(db, block.Hash(), block.NumberU64()); entry == nil { + WriteBlock(db, block) + if entry := ReadBlock(db, block.Hash(), block.NumberU64()); entry == nil { t.Fatalf("Stored block not found") } else if entry.Hash() != block.Hash() { t.Fatalf("Retrieved block mismatch: have %v, want %v", entry, block) } - if entry := GetHeader(db, block.Hash(), block.NumberU64()); entry == nil { + if entry := ReadHeader(db, block.Hash(), block.NumberU64()); entry == nil { t.Fatalf("Stored header not found") } else if entry.Hash() != block.Header().Hash() { t.Fatalf("Retrieved header mismatch: have %v, want %v", entry, block.Header()) } - if entry := GetBody(db, block.Hash(), block.NumberU64()); entry == nil { + if entry := ReadBody(db, block.Hash(), block.NumberU64()); entry == nil { t.Fatalf("Stored body not found") } else if types.DeriveSha(types.Transactions(entry.Transactions)) != types.DeriveSha(block.Transactions()) || types.CalcUncleHash(entry.Uncles) != types.CalcUncleHash(block.Uncles()) { t.Fatalf("Retrieved body mismatch: have %v, want %v", entry, block.Body()) } // Delete the block and verify the execution DeleteBlock(db, block.Hash(), block.NumberU64()) - if entry := GetBlock(db, block.Hash(), block.NumberU64()); entry != nil { + if entry := ReadBlock(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Deleted block returned: %v", entry) } - if entry := GetHeader(db, block.Hash(), block.NumberU64()); entry != nil { + if entry := ReadHeader(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Deleted header returned: %v", entry) } - if entry := GetBody(db, block.Hash(), block.NumberU64()); entry != nil { + if entry := ReadBody(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Deleted body returned: %v", entry) } } // Tests that partial block contents don't get reassembled into full blocks. func TestPartialBlockStorage(t *testing.T) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() block := types.NewBlockWithHeader(&types.Header{ Extra: []byte("test block"), UncleHash: types.EmptyUncleHash, @@ -165,31 +159,24 @@ func TestPartialBlockStorage(t *testing.T) { ReceiptHash: types.EmptyRootHash, }) // Store a header and check that it's not recognized as a block - if err := WriteHeader(db, block.Header()); err != nil { - t.Fatalf("Failed to write header into database: %v", err) - } - if entry := GetBlock(db, block.Hash(), block.NumberU64()); entry != nil { + WriteHeader(db, block.Header()) + if entry := ReadBlock(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Non existent block returned: %v", entry) } DeleteHeader(db, block.Hash(), block.NumberU64()) // Store a body and check that it's not recognized as a block - if err := WriteBody(db, block.Hash(), block.NumberU64(), block.Body()); err != nil { - t.Fatalf("Failed to write body into database: %v", err) - } - if entry := GetBlock(db, block.Hash(), block.NumberU64()); entry != nil { + WriteBody(db, block.Hash(), block.NumberU64(), block.Body()) + if entry := ReadBlock(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Non existent block returned: %v", entry) } DeleteBody(db, block.Hash(), block.NumberU64()) // Store a header and a body separately and check reassembly - if err := WriteHeader(db, block.Header()); err != nil { - t.Fatalf("Failed to write header into database: %v", err) - } - if err := WriteBody(db, block.Hash(), block.NumberU64(), block.Body()); err != nil { - t.Fatalf("Failed to write body into database: %v", err) - } - if entry := GetBlock(db, block.Hash(), block.NumberU64()); entry == nil { + WriteHeader(db, block.Header()) + WriteBody(db, block.Hash(), block.NumberU64(), block.Body()) + + if entry := ReadBlock(db, block.Hash(), block.NumberU64()); entry == nil { t.Fatalf("Stored block not found") } else if entry.Hash() != block.Hash() { t.Fatalf("Retrieved block mismatch: have %v, want %v", entry, block) @@ -198,142 +185,88 @@ func TestPartialBlockStorage(t *testing.T) { // Tests block total difficulty storage and retrieval operations. func TestTdStorage(t *testing.T) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() // Create a test TD to move around the database and make sure it's really new hash, td := common.Hash{}, big.NewInt(314) - if entry := GetTd(db, hash, 0); entry != nil { + if entry := ReadTd(db, hash, 0); entry != nil { t.Fatalf("Non existent TD returned: %v", entry) } // Write and verify the TD in the database - if err := WriteTd(db, hash, 0, td); err != nil { - t.Fatalf("Failed to write TD into database: %v", err) - } - if entry := GetTd(db, hash, 0); entry == nil { + WriteTd(db, hash, 0, td) + if entry := ReadTd(db, hash, 0); entry == nil { t.Fatalf("Stored TD not found") } else if entry.Cmp(td) != 0 { t.Fatalf("Retrieved TD mismatch: have %v, want %v", entry, td) } // Delete the TD and verify the execution DeleteTd(db, hash, 0) - if entry := GetTd(db, hash, 0); entry != nil { + if entry := ReadTd(db, hash, 0); entry != nil { t.Fatalf("Deleted TD returned: %v", entry) } } // Tests that canonical numbers can be mapped to hashes and retrieved. func TestCanonicalMappingStorage(t *testing.T) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() // Create a test canonical number and assinged hash to move around hash, number := common.Hash{0: 0xff}, uint64(314) - if entry := GetCanonicalHash(db, number); entry != (common.Hash{}) { + if entry := ReadCanonicalHash(db, number); entry != (common.Hash{}) { t.Fatalf("Non existent canonical mapping returned: %v", entry) } // Write and verify the TD in the database - if err := WriteCanonicalHash(db, hash, number); err != nil { - t.Fatalf("Failed to write canonical mapping into database: %v", err) - } - if entry := GetCanonicalHash(db, number); entry == (common.Hash{}) { + WriteCanonicalHash(db, hash, number) + if entry := ReadCanonicalHash(db, number); entry == (common.Hash{}) { t.Fatalf("Stored canonical mapping not found") } else if entry != hash { t.Fatalf("Retrieved canonical mapping mismatch: have %v, want %v", entry, hash) } // Delete the TD and verify the execution DeleteCanonicalHash(db, number) - if entry := GetCanonicalHash(db, number); entry != (common.Hash{}) { + if entry := ReadCanonicalHash(db, number); entry != (common.Hash{}) { t.Fatalf("Deleted canonical mapping returned: %v", entry) } } // Tests that head headers and head blocks can be assigned, individually. func TestHeadStorage(t *testing.T) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() blockHead := types.NewBlockWithHeader(&types.Header{Extra: []byte("test block header")}) blockFull := types.NewBlockWithHeader(&types.Header{Extra: []byte("test block full")}) blockFast := types.NewBlockWithHeader(&types.Header{Extra: []byte("test block fast")}) // Check that no head entries are in a pristine database - if entry := GetHeadHeaderHash(db); entry != (common.Hash{}) { + if entry := ReadHeadHeaderHash(db); entry != (common.Hash{}) { t.Fatalf("Non head header entry returned: %v", entry) } - if entry := GetHeadBlockHash(db); entry != (common.Hash{}) { + if entry := ReadHeadBlockHash(db); entry != (common.Hash{}) { t.Fatalf("Non head block entry returned: %v", entry) } - if entry := GetHeadFastBlockHash(db); entry != (common.Hash{}) { + if entry := ReadHeadFastBlockHash(db); entry != (common.Hash{}) { t.Fatalf("Non fast head block entry returned: %v", entry) } // Assign separate entries for the head header and block - if err := WriteHeadHeaderHash(db, blockHead.Hash()); err != nil { - t.Fatalf("Failed to write head header hash: %v", err) - } - if err := WriteHeadBlockHash(db, blockFull.Hash()); err != nil { - t.Fatalf("Failed to write head block hash: %v", err) - } - if err := WriteHeadFastBlockHash(db, blockFast.Hash()); err != nil { - t.Fatalf("Failed to write fast head block hash: %v", err) - } + WriteHeadHeaderHash(db, blockHead.Hash()) + WriteHeadBlockHash(db, blockFull.Hash()) + WriteHeadFastBlockHash(db, blockFast.Hash()) + // Check that both heads are present, and different (i.e. two heads maintained) - if entry := GetHeadHeaderHash(db); entry != blockHead.Hash() { + if entry := ReadHeadHeaderHash(db); entry != blockHead.Hash() { t.Fatalf("Head header hash mismatch: have %v, want %v", entry, blockHead.Hash()) } - if entry := GetHeadBlockHash(db); entry != blockFull.Hash() { + if entry := ReadHeadBlockHash(db); entry != blockFull.Hash() { t.Fatalf("Head block hash mismatch: have %v, want %v", entry, blockFull.Hash()) } - if entry := GetHeadFastBlockHash(db); entry != blockFast.Hash() { + if entry := ReadHeadFastBlockHash(db); entry != blockFast.Hash() { t.Fatalf("Fast head block hash mismatch: have %v, want %v", entry, blockFast.Hash()) } } -// Tests that positional lookup metadata can be stored and retrieved. -func TestLookupStorage(t *testing.T) { - db, _ := ethdb.NewMemDatabase() - - tx1 := types.NewTransaction(1, common.BytesToAddress([]byte{0x11}), big.NewInt(111), 1111, big.NewInt(11111), []byte{0x11, 0x11, 0x11}) - tx2 := types.NewTransaction(2, common.BytesToAddress([]byte{0x22}), big.NewInt(222), 2222, big.NewInt(22222), []byte{0x22, 0x22, 0x22}) - tx3 := types.NewTransaction(3, common.BytesToAddress([]byte{0x33}), big.NewInt(333), 3333, big.NewInt(33333), []byte{0x33, 0x33, 0x33}) - txs := []*types.Transaction{tx1, tx2, tx3} - - block := types.NewBlock(&types.Header{Number: big.NewInt(314)}, txs, nil, nil) - - // Check that no transactions entries are in a pristine database - for i, tx := range txs { - if txn, _, _, _ := GetTransaction(db, tx.Hash()); txn != nil { - t.Fatalf("tx #%d [%x]: non existent transaction returned: %v", i, tx.Hash(), txn) - } - } - // Insert all the transactions into the database, and verify contents - if err := WriteBlock(db, block); err != nil { - t.Fatalf("failed to write block contents: %v", err) - } - if err := WriteTxLookupEntries(db, block); err != nil { - t.Fatalf("failed to write transactions: %v", err) - } - for i, tx := range txs { - if txn, hash, number, index := GetTransaction(db, tx.Hash()); txn == nil { - t.Fatalf("tx #%d [%x]: transaction not found", i, tx.Hash()) - } else { - if hash != block.Hash() || number != block.NumberU64() || index != uint64(i) { - t.Fatalf("tx #%d [%x]: positional metadata mismatch: have %x/%d/%d, want %x/%v/%v", i, tx.Hash(), hash, number, index, block.Hash(), block.NumberU64(), i) - } - if tx.Hash() != txn.Hash() { - t.Fatalf("tx #%d [%x]: transaction mismatch: have %v, want %v", i, tx.Hash(), txn, tx) - } - } - } - // Delete the transactions and check purge - for i, tx := range txs { - DeleteTxLookupEntry(db, tx.Hash()) - if txn, _, _, _ := GetTransaction(db, tx.Hash()); txn != nil { - t.Fatalf("tx #%d [%x]: deleted transaction returned: %v", i, tx.Hash(), txn) - } - } -} - // Tests that receipts associated with a single block can be stored and retrieved. func TestBlockReceiptStorage(t *testing.T) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() receipt1 := &types.Receipt{ Status: types.ReceiptStatusFailed, @@ -361,14 +294,12 @@ func TestBlockReceiptStorage(t *testing.T) { // Check that no receipt entries are in a pristine database hash := common.BytesToHash([]byte{0x03, 0x14}) - if rs := GetBlockReceipts(db, hash, 0); len(rs) != 0 { + if rs := ReadReceipts(db, hash, 0); len(rs) != 0 { t.Fatalf("non existent receipts returned: %v", rs) } // Insert the receipt slice into the database and check presence - if err := WriteBlockReceipts(db, hash, 0, receipts); err != nil { - t.Fatalf("failed to write block receipts: %v", err) - } - if rs := GetBlockReceipts(db, hash, 0); len(rs) == 0 { + WriteReceipts(db, hash, 0, receipts) + if rs := ReadReceipts(db, hash, 0); len(rs) == 0 { t.Fatalf("no receipts returned") } else { for i := 0; i < len(receipts); i++ { @@ -381,8 +312,8 @@ func TestBlockReceiptStorage(t *testing.T) { } } // Delete the receipt slice and check purge - DeleteBlockReceipts(db, hash, 0) - if rs := GetBlockReceipts(db, hash, 0); len(rs) != 0 { + DeleteReceipts(db, hash, 0) + if rs := ReadReceipts(db, hash, 0); len(rs) != 0 { t.Fatalf("deleted receipts returned: %v", rs) } } diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go new file mode 100644 index 0000000000..9abad14e0c --- /dev/null +++ b/core/rawdb/accessors_indexes.go @@ -0,0 +1,119 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "encoding/binary" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" +) + +// ReadTxLookupEntry retrieves the positional metadata associated with a transaction +// hash to allow retrieving the transaction or receipt by hash. +func ReadTxLookupEntry(db DatabaseReader, hash common.Hash) (common.Hash, uint64, uint64) { + data, _ := db.Get(append(txLookupPrefix, hash.Bytes()...)) + if len(data) == 0 { + return common.Hash{}, 0, 0 + } + var entry TxLookupEntry + if err := rlp.DecodeBytes(data, &entry); err != nil { + log.Error("Invalid transaction lookup entry RLP", "hash", hash, "err", err) + return common.Hash{}, 0, 0 + } + return entry.BlockHash, entry.BlockIndex, entry.Index +} + +// WriteTxLookupEntries stores a positional metadata for every transaction from +// a block, enabling hash based transaction and receipt lookups. +func WriteTxLookupEntries(db DatabaseWriter, block *types.Block) { + for i, tx := range block.Transactions() { + entry := TxLookupEntry{ + BlockHash: block.Hash(), + BlockIndex: block.NumberU64(), + Index: uint64(i), + } + data, err := rlp.EncodeToBytes(entry) + if err != nil { + log.Crit("Failed to encode transaction lookup entry", "err", err) + } + if err := db.Put(append(txLookupPrefix, tx.Hash().Bytes()...), data); err != nil { + log.Crit("Failed to store transaction lookup entry", "err", err) + } + } +} + +// DeleteTxLookupEntry removes all transaction data associated with a hash. +func DeleteTxLookupEntry(db DatabaseDeleter, hash common.Hash) { + db.Delete(append(txLookupPrefix, hash.Bytes()...)) +} + +// ReadTransaction retrieves a specific transaction from the database, along with +// its added positional metadata. +func ReadTransaction(db DatabaseReader, hash common.Hash) (*types.Transaction, common.Hash, uint64, uint64) { + blockHash, blockNumber, txIndex := ReadTxLookupEntry(db, hash) + if blockHash == (common.Hash{}) { + return nil, common.Hash{}, 0, 0 + } + body := ReadBody(db, blockHash, blockNumber) + if body == nil || len(body.Transactions) <= int(txIndex) { + log.Error("Transaction referenced missing", "number", blockNumber, "hash", blockHash, "index", txIndex) + return nil, common.Hash{}, 0, 0 + } + return body.Transactions[txIndex], blockHash, blockNumber, txIndex +} + +// ReadReceipt retrieves a specific transaction receipt from the database, along with +// its added positional metadata. +func ReadReceipt(db DatabaseReader, hash common.Hash) (*types.Receipt, common.Hash, uint64, uint64) { + blockHash, blockNumber, receiptIndex := ReadTxLookupEntry(db, hash) + if blockHash == (common.Hash{}) { + return nil, common.Hash{}, 0, 0 + } + receipts := ReadReceipts(db, blockHash, blockNumber) + if len(receipts) <= int(receiptIndex) { + log.Error("Receipt refereced missing", "number", blockNumber, "hash", blockHash, "index", receiptIndex) + return nil, common.Hash{}, 0, 0 + } + return receipts[receiptIndex], blockHash, blockNumber, receiptIndex +} + +// ReadBloomBits retrieves the compressed bloom bit vector belonging to the given +// section and bit index from the. +func ReadBloomBits(db DatabaseReader, bit uint, section uint64, head common.Hash) ([]byte, error) { + key := append(append(bloomBitsPrefix, make([]byte, 10)...), head.Bytes()...) + + binary.BigEndian.PutUint16(key[1:], uint16(bit)) + binary.BigEndian.PutUint64(key[3:], section) + + return db.Get(key) +} + +// WriteBloomBits stores the compressed bloom bits vector belonging to the given +// section and bit index. +func WriteBloomBits(db DatabaseWriter, bit uint, section uint64, head common.Hash, bits []byte) { + key := append(append(bloomBitsPrefix, make([]byte, 10)...), head.Bytes()...) + + binary.BigEndian.PutUint16(key[1:], uint16(bit)) + binary.BigEndian.PutUint64(key[3:], section) + + if err := db.Put(key, bits); err != nil { + log.Crit("Failed to store bloom bits", "err", err) + } +} diff --git a/core/rawdb/accessors_indexes_test.go b/core/rawdb/accessors_indexes_test.go new file mode 100644 index 0000000000..d9c10e1490 --- /dev/null +++ b/core/rawdb/accessors_indexes_test.go @@ -0,0 +1,68 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" +) + +// Tests that positional lookup metadata can be stored and retrieved. +func TestLookupStorage(t *testing.T) { + db := ethdb.NewMemDatabase() + + tx1 := types.NewTransaction(1, common.BytesToAddress([]byte{0x11}), big.NewInt(111), 1111, big.NewInt(11111), []byte{0x11, 0x11, 0x11}) + tx2 := types.NewTransaction(2, common.BytesToAddress([]byte{0x22}), big.NewInt(222), 2222, big.NewInt(22222), []byte{0x22, 0x22, 0x22}) + tx3 := types.NewTransaction(3, common.BytesToAddress([]byte{0x33}), big.NewInt(333), 3333, big.NewInt(33333), []byte{0x33, 0x33, 0x33}) + txs := []*types.Transaction{tx1, tx2, tx3} + + block := types.NewBlock(&types.Header{Number: big.NewInt(314)}, txs, nil, nil) + + // Check that no transactions entries are in a pristine database + for i, tx := range txs { + if txn, _, _, _ := ReadTransaction(db, tx.Hash()); txn != nil { + t.Fatalf("tx #%d [%x]: non existent transaction returned: %v", i, tx.Hash(), txn) + } + } + // Insert all the transactions into the database, and verify contents + WriteBlock(db, block) + WriteTxLookupEntries(db, block) + + for i, tx := range txs { + if txn, hash, number, index := ReadTransaction(db, tx.Hash()); txn == nil { + t.Fatalf("tx #%d [%x]: transaction not found", i, tx.Hash()) + } else { + if hash != block.Hash() || number != block.NumberU64() || index != uint64(i) { + t.Fatalf("tx #%d [%x]: positional metadata mismatch: have %x/%d/%d, want %x/%v/%v", i, tx.Hash(), hash, number, index, block.Hash(), block.NumberU64(), i) + } + if tx.Hash() != txn.Hash() { + t.Fatalf("tx #%d [%x]: transaction mismatch: have %v, want %v", i, tx.Hash(), txn, tx) + } + } + } + // Delete the transactions and check purge + for i, tx := range txs { + DeleteTxLookupEntry(db, tx.Hash()) + if txn, _, _, _ := ReadTransaction(db, tx.Hash()); txn != nil { + t.Fatalf("tx #%d [%x]: deleted transaction returned: %v", i, tx.Hash(), txn) + } + } +} diff --git a/core/rawdb/accessors_metadata.go b/core/rawdb/accessors_metadata.go new file mode 100644 index 0000000000..73ab983f2b --- /dev/null +++ b/core/rawdb/accessors_metadata.go @@ -0,0 +1,90 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "encoding/json" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" +) + +// ReadDatabaseVersion retrieves the version number of the database. +func ReadDatabaseVersion(db DatabaseReader) int { + var version int + + enc, _ := db.Get(databaseVerisionKey) + rlp.DecodeBytes(enc, &version) + + return version +} + +// WriteDatabaseVersion stores the version number of the database +func WriteDatabaseVersion(db DatabaseWriter, version int) { + enc, _ := rlp.EncodeToBytes(version) + if err := db.Put(databaseVerisionKey, enc); err != nil { + log.Crit("Failed to store the database version", "err", err) + } +} + +// ReadChainConfig retrieves the consensus settings based on the given genesis hash. +func ReadChainConfig(db DatabaseReader, hash common.Hash) *params.ChainConfig { + data, _ := db.Get(append(configPrefix, hash[:]...)) + if len(data) == 0 { + return nil + } + var config params.ChainConfig + if err := json.Unmarshal(data, &config); err != nil { + log.Error("Invalid chain config JSON", "hash", hash, "err", err) + return nil + } + return &config +} + +// WriteChainConfig writes the chain config settings to the database. +func WriteChainConfig(db DatabaseWriter, hash common.Hash, cfg *params.ChainConfig) { + if cfg == nil { + return + } + data, err := json.Marshal(cfg) + if err != nil { + log.Crit("Failed to JSON encode chain config", "err", err) + } + if err := db.Put(append(configPrefix, hash[:]...), data); err != nil { + log.Crit("Failed to store chain config", "err", err) + } +} + +// ReadPreimage retrieves a single preimage of the provided hash. +func ReadPreimage(db DatabaseReader, hash common.Hash) []byte { + data, _ := db.Get(append(preimagePrefix, hash.Bytes()...)) + return data +} + +// WritePreimages writes the provided set of preimages to the database. `number` is the +// current block number, and is used for debug messages only. +func WritePreimages(db DatabaseWriter, number uint64, preimages map[common.Hash][]byte) { + for hash, preimage := range preimages { + if err := db.Put(append(preimagePrefix, hash.Bytes()...), preimage); err != nil { + log.Crit("Failed to store trie preimage", "err", err) + } + } + preimageCounter.Inc(int64(len(preimages))) + preimageHitCounter.Inc(int64(len(preimages))) +} diff --git a/core/rawdb/interfaces.go b/core/rawdb/interfaces.go new file mode 100644 index 0000000000..3bdf55124a --- /dev/null +++ b/core/rawdb/interfaces.go @@ -0,0 +1,33 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +// DatabaseReader wraps the Has and Get method of a backing data store. +type DatabaseReader interface { + Has(key []byte) (bool, error) + Get(key []byte) ([]byte, error) +} + +// DatabaseWriter wraps the Put method of a backing data store. +type DatabaseWriter interface { + Put(key []byte, value []byte) error +} + +// DatabaseDeleter wraps the Delete method of a backing data store. +type DatabaseDeleter interface { + Delete(key []byte) error +} diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go new file mode 100644 index 0000000000..a4b1596fde --- /dev/null +++ b/core/rawdb/schema.go @@ -0,0 +1,79 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package rawdb contains a collection of low level database accessors. +package rawdb + +import ( + "encoding/binary" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/metrics" +) + +// The fields below define the low level database schema prefixing. +var ( + // databaseVerisionKey tracks the current database version. + databaseVerisionKey = []byte("DatabaseVersion") + + // headHeaderKey tracks the latest know header's hash. + headHeaderKey = []byte("LastHeader") + + // headBlockKey tracks the latest know full block's hash. + headBlockKey = []byte("LastBlock") + + // headFastBlockKey tracks the latest known incomplete block's hash duirng fast sync. + headFastBlockKey = []byte("LastFast") + + // fastTrieProgressKey tracks the number of trie entries imported during fast sync. + fastTrieProgressKey = []byte("TrieSync") + + // Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes). + headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header + headerTDSuffix = []byte("t") // headerPrefix + num (uint64 big endian) + hash + headerTDSuffix -> td + headerHashSuffix = []byte("n") // headerPrefix + num (uint64 big endian) + headerHashSuffix -> hash + headerNumberPrefix = []byte("H") // headerNumberPrefix + hash -> num (uint64 big endian) + + blockBodyPrefix = []byte("b") // blockBodyPrefix + num (uint64 big endian) + hash -> block body + blockReceiptsPrefix = []byte("r") // blockReceiptsPrefix + num (uint64 big endian) + hash -> block receipts + + txLookupPrefix = []byte("l") // txLookupPrefix + hash -> transaction/receipt lookup metadata + bloomBitsPrefix = []byte("B") // bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash -> bloom bits + + preimagePrefix = []byte("secure-key-") // preimagePrefix + hash -> preimage + configPrefix = []byte("ethereum-config-") // config prefix for the db + + // Chain index prefixes (use `i` + single byte to avoid mixing data types). + BloomBitsIndexPrefix = []byte("iB") // BloomBitsIndexPrefix is the data table of a chain indexer to track its progress + + preimageCounter = metrics.NewRegisteredCounter("db/preimage/total", nil) + preimageHitCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil) +) + +// TxLookupEntry is a positional metadata to help looking up the data content of +// a transaction or receipt given only its hash. +type TxLookupEntry struct { + BlockHash common.Hash + BlockIndex uint64 + Index uint64 +} + +// encodeBlockNumber encodes a block number as big endian uint64 +func encodeBlockNumber(number uint64) []byte { + enc := make([]byte, 8) + binary.BigEndian.PutUint64(enc, number) + return enc +} diff --git a/core/state/managed_state_test.go b/core/state/managed_state_test.go index 1cfdd3a890..3d9c4e8676 100644 --- a/core/state/managed_state_test.go +++ b/core/state/managed_state_test.go @@ -26,8 +26,7 @@ import ( var addr = common.BytesToAddress([]byte("test")) func create() (*ManagedState, *account) { - db, _ := ethdb.NewMemDatabase() - statedb, _ := New(common.Hash{}, NewDatabase(db)) + statedb, _ := New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase())) ms := ManageState(statedb) ms.StateDB.SetNonce(addr, 100) ms.accounts[addr] = newAccount(ms.StateDB.getStateObject(addr)) diff --git a/core/state/state_object.go b/core/state/state_object.go index 3c40c20415..5d203fddd8 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -178,9 +178,7 @@ func (self *stateObject) GetState(db Database, key common.Hash) common.Hash { } value.SetBytes(content) } - if (value != common.Hash{}) { - self.cachedStorage[key] = value - } + self.cachedStorage[key] = value return value } @@ -197,7 +195,6 @@ func (self *stateObject) SetState(db Database, key, value common.Hash) { func (self *stateObject) setState(key, value common.Hash) { self.cachedStorage[key] = value self.dirtyStorage[key] = value - } // updateTrie writes cached storage modifications into the object's storage trie. diff --git a/core/state/state_test.go b/core/state/state_test.go index 6d42d63d82..12778f6f12 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -87,7 +87,7 @@ func (s *StateSuite) TestDump(c *checker.C) { } func (s *StateSuite) SetUpTest(c *checker.C) { - s.db, _ = ethdb.NewMemDatabase() + s.db = ethdb.NewMemDatabase() s.state, _ = New(common.Hash{}, NewDatabase(s.db)) } @@ -133,8 +133,7 @@ func (s *StateSuite) TestSnapshotEmpty(c *checker.C) { // use testing instead of checker because checker does not support // printing/logging in tests (-check.vv does not work) func TestSnapshot2(t *testing.T) { - db, _ := ethdb.NewMemDatabase() - state, _ := New(common.Hash{}, NewDatabase(db)) + state, _ := New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase())) stateobjaddr0 := toAddr([]byte("so0")) stateobjaddr1 := toAddr([]byte("so1")) diff --git a/core/state/statedb.go b/core/state/statedb.go index 3ae6843d87..a952027d6d 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -572,27 +572,6 @@ func (self *StateDB) Prepare(thash, bhash common.Hash, ti int) { self.txIndex = ti } -// DeleteSuicides flags the suicided objects for deletion so that it -// won't be referenced again when called / queried up on. -// -// DeleteSuicides should not be used for consensus related updates -// under any circumstances. -func (s *StateDB) DeleteSuicides() { - // Reset refund so that any used-gas calculations can use this method. - s.clearJournalAndRefund() - - for addr := range s.stateObjectsDirty { - stateObject := s.stateObjects[addr] - - // If the object has been removed by a suicide - // flag the object as deleted. - if stateObject.suicided { - stateObject.deleted = true - } - delete(s.stateObjectsDirty, addr) - } -} - func (s *StateDB) clearJournalAndRefund() { s.journal = newJournal() s.validRevisions = s.validRevisions[:0] diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 420ca745cc..e2b349de86 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -39,7 +39,7 @@ import ( // actually committing the state. func TestUpdateLeaks(t *testing.T) { // Create an empty state database - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() state, _ := New(common.Hash{}, NewDatabase(db)) // Update it with some accounts @@ -66,8 +66,8 @@ func TestUpdateLeaks(t *testing.T) { // only the one right before the commit. func TestIntermediateLeaks(t *testing.T) { // Create two state databases, one transitioning to the final state, the other final from the beginning - transDb, _ := ethdb.NewMemDatabase() - finalDb, _ := ethdb.NewMemDatabase() + transDb := ethdb.NewMemDatabase() + finalDb := ethdb.NewMemDatabase() transState, _ := New(common.Hash{}, NewDatabase(transDb)) finalState, _ := New(common.Hash{}, NewDatabase(finalDb)) @@ -122,8 +122,7 @@ func TestIntermediateLeaks(t *testing.T) { // https://github.com/ethereum/go-ethereum/pull/15549. func TestCopy(t *testing.T) { // Create a random state test to copy and modify "independently" - db, _ := ethdb.NewMemDatabase() - orig, _ := New(common.Hash{}, NewDatabase(db)) + orig, _ := New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase())) for i := byte(0); i < 255; i++ { obj := orig.GetOrNewStateObject(common.BytesToAddress([]byte{i})) @@ -334,8 +333,7 @@ func (test *snapshotTest) String() string { func (test *snapshotTest) run() bool { // Run all actions and create snapshots. var ( - db, _ = ethdb.NewMemDatabase() - state, _ = New(common.Hash{}, NewDatabase(db)) + state, _ = New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase())) snapshotRevs = make([]int, len(test.snapshots)) sindex = 0 ) @@ -426,8 +424,7 @@ func (s *StateSuite) TestTouchDelete(c *check.C) { // TestCopyOfCopy tests that modified objects are carried over to the copy, and the copy of the copy. // See https://github.com/ethereum/go-ethereum/pull/15225#issuecomment-380191512 func TestCopyOfCopy(t *testing.T) { - db, _ := ethdb.NewMemDatabase() - sdb, _ := New(common.Hash{}, NewDatabase(db)) + sdb, _ := New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase())) addr := common.HexToAddress("aaaa") sdb.SetBalance(addr, big.NewInt(42)) diff --git a/core/state/sync_test.go b/core/state/sync_test.go index 8f14a44e7a..3177401608 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -38,8 +38,7 @@ type testAccount struct { // makeTestState create a sample test state to test node-wise reconstruction. func makeTestState() (Database, common.Hash, []*testAccount) { // Create an empty state - diskdb, _ := ethdb.NewMemDatabase() - db := NewDatabase(diskdb) + db := NewDatabase(ethdb.NewMemDatabase()) state, _ := New(common.Hash{}, db) // Fill it with some arbitrary data @@ -125,8 +124,7 @@ func checkStateConsistency(db ethdb.Database, root common.Hash) error { // Tests that an empty state is not scheduled for syncing. func TestEmptyStateSync(t *testing.T) { empty := common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") - db, _ := ethdb.NewMemDatabase() - if req := NewStateSync(empty, db).Missing(1); len(req) != 0 { + if req := NewStateSync(empty, ethdb.NewMemDatabase()).Missing(1); len(req) != 0 { t.Errorf("content requested for empty state: %v", req) } } @@ -141,7 +139,7 @@ func testIterativeStateSync(t *testing.T, batch int) { srcDb, srcRoot, srcAccounts := makeTestState() // Create a destination state and sync with the scheduler - dstDb, _ := ethdb.NewMemDatabase() + dstDb := ethdb.NewMemDatabase() sched := NewStateSync(srcRoot, dstDb) queue := append([]common.Hash{}, sched.Missing(batch)...) @@ -173,7 +171,7 @@ func TestIterativeDelayedStateSync(t *testing.T) { srcDb, srcRoot, srcAccounts := makeTestState() // Create a destination state and sync with the scheduler - dstDb, _ := ethdb.NewMemDatabase() + dstDb := ethdb.NewMemDatabase() sched := NewStateSync(srcRoot, dstDb) queue := append([]common.Hash{}, sched.Missing(0)...) @@ -210,7 +208,7 @@ func testIterativeRandomStateSync(t *testing.T, batch int) { srcDb, srcRoot, srcAccounts := makeTestState() // Create a destination state and sync with the scheduler - dstDb, _ := ethdb.NewMemDatabase() + dstDb := ethdb.NewMemDatabase() sched := NewStateSync(srcRoot, dstDb) queue := make(map[common.Hash]struct{}) @@ -250,7 +248,7 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) { srcDb, srcRoot, srcAccounts := makeTestState() // Create a destination state and sync with the scheduler - dstDb, _ := ethdb.NewMemDatabase() + dstDb := ethdb.NewMemDatabase() sched := NewStateSync(srcRoot, dstDb) queue := make(map[common.Hash]struct{}) @@ -297,7 +295,7 @@ func TestIncompleteStateSync(t *testing.T) { checkTrieConsistency(srcDb.TrieDB().DiskDB().(ethdb.Database), srcRoot) // Create a destination state and sync with the scheduler - dstDb, _ := ethdb.NewMemDatabase() + dstDb := ethdb.NewMemDatabase() sched := NewStateSync(srcRoot, dstDb) added := []common.Hash{} diff --git a/core/tx_pool.go b/core/tx_pool.go index b21384458a..388b40058f 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -618,7 +618,7 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (bool, error) { // If the transaction pool is full, discard underpriced transactions if uint64(len(pool.all)) >= pool.config.GlobalSlots+pool.config.GlobalQueue { // If the new transaction is underpriced, don't accept it - if pool.priced.Underpriced(tx, pool.locals) { + if !local && pool.priced.Underpriced(tx, pool.locals) { log.Trace("Discarding underpriced transaction", "hash", hash, "price", tx.GasPrice()) underpricedTxCounter.Inc(1) return false, ErrUnderpriced diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index 0cb14cb6a4..e7f52075e5 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -78,8 +78,7 @@ func pricedTransaction(nonce uint64, gaslimit uint64, gasprice *big.Int, key *ec } func setupTxPool() (*TxPool, *ecdsa.PrivateKey) { - diskdb, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(diskdb)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} key, _ := crypto.GenerateKey() @@ -158,8 +157,7 @@ func (c *testChain) State() (*state.StateDB, error) { // a state change between those fetches. stdb := c.statedb if *c.trigger { - db, _ := ethdb.NewMemDatabase() - c.statedb, _ = state.New(common.Hash{}, state.NewDatabase(db)) + c.statedb, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) // simulate that the new head block included tx0 and tx1 c.statedb.SetNonce(c.address, 2) c.statedb.SetBalance(c.address, new(big.Int).SetUint64(params.Ether)) @@ -175,10 +173,9 @@ func TestStateChangeDuringTransactionPoolReset(t *testing.T) { t.Parallel() var ( - db, _ = ethdb.NewMemDatabase() key, _ = crypto.GenerateKey() address = crypto.PubkeyToAddress(key.PublicKey) - statedb, _ = state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) trigger = false ) @@ -332,8 +329,7 @@ func TestTransactionChainFork(t *testing.T) { addr := crypto.PubkeyToAddress(key.PublicKey) resetState := func() { - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) statedb.AddBalance(addr, big.NewInt(100000000000000)) pool.chain = &testBlockChain{statedb, 1000000, new(event.Feed)} @@ -362,8 +358,7 @@ func TestTransactionDoubleNonce(t *testing.T) { addr := crypto.PubkeyToAddress(key.PublicKey) resetState := func() { - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) statedb.AddBalance(addr, big.NewInt(100000000000000)) pool.chain = &testBlockChain{statedb, 1000000, new(event.Feed)} @@ -553,8 +548,7 @@ func TestTransactionPostponing(t *testing.T) { t.Parallel() // Create the pool to test the postponing with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) @@ -769,8 +763,7 @@ func testTransactionQueueGlobalLimiting(t *testing.T, nolocals bool) { t.Parallel() // Create the pool to test the limit enforcement with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -858,8 +851,7 @@ func testTransactionQueueTimeLimiting(t *testing.T, nolocals bool) { evictionInterval = time.Second // Create the pool to test the non-expiration enforcement - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1013,8 +1005,7 @@ func TestTransactionPendingGlobalLimiting(t *testing.T) { t.Parallel() // Create the pool to test the limit enforcement with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1060,8 +1051,7 @@ func TestTransactionCapClearsFromAll(t *testing.T) { t.Parallel() // Create the pool to test the limit enforcement with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1095,8 +1085,7 @@ func TestTransactionPendingMinimumAllowance(t *testing.T) { t.Parallel() // Create the pool to test the limit enforcement with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1144,8 +1133,7 @@ func TestTransactionPoolRepricing(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) @@ -1266,8 +1254,7 @@ func TestTransactionPoolRepricingKeepsLocals(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) @@ -1329,8 +1316,7 @@ func TestTransactionPoolUnderpricing(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1346,7 +1332,7 @@ func TestTransactionPoolUnderpricing(t *testing.T) { defer sub.Unsubscribe() // Create a number of test accounts and fund them - keys := make([]*ecdsa.PrivateKey, 3) + keys := make([]*ecdsa.PrivateKey, 4) for i := 0; i < len(keys); i++ { keys[i], _ = crypto.GenerateKey() pool.currentState.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) @@ -1406,18 +1392,22 @@ func TestTransactionPoolUnderpricing(t *testing.T) { t.Fatalf("pool internal state corrupted: %v", err) } // Ensure that adding local transactions can push out even higher priced ones - tx := pricedTransaction(1, 100000, big.NewInt(0), keys[2]) - if err := pool.AddLocal(tx); err != nil { - t.Fatalf("failed to add underpriced local transaction: %v", err) + ltx = pricedTransaction(1, 100000, big.NewInt(0), keys[2]) + if err := pool.AddLocal(ltx); err != nil { + t.Fatalf("failed to append underpriced local transaction: %v", err) + } + ltx = pricedTransaction(0, 100000, big.NewInt(0), keys[3]) + if err := pool.AddLocal(ltx); err != nil { + t.Fatalf("failed to add new underpriced local transaction: %v", err) } pending, queued = pool.Stats() - if pending != 2 { - t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 2) + if pending != 3 { + t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 3) } - if queued != 2 { - t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 2) + if queued != 1 { + t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 1) } - if err := validateEvents(events, 1); err != nil { + if err := validateEvents(events, 2); err != nil { t.Fatalf("local event firing failed: %v", err) } if err := validateTxPoolInternals(pool); err != nil { @@ -1432,8 +1422,7 @@ func TestTransactionPoolStableUnderpricing(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1499,8 +1488,7 @@ func TestTransactionReplacement(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) @@ -1594,8 +1582,7 @@ func testTransactionJournaling(t *testing.T, nolocals bool) { os.Remove(journal) // Create the original pool to inject transaction into the journal - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1693,8 +1680,7 @@ func TestTransactionStatusCheck(t *testing.T) { t.Parallel() // Create the pool to test the status retrievals with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) diff --git a/core/vm/contract.go b/core/vm/contract.go index 66748e8215..b466681dbd 100644 --- a/core/vm/contract.go +++ b/core/vm/contract.go @@ -139,15 +139,15 @@ func (c *Contract) Value() *big.Int { } // SetCode sets the code to the contract -func (self *Contract) SetCode(hash common.Hash, code []byte) { - self.Code = code - self.CodeHash = hash +func (c *Contract) SetCode(hash common.Hash, code []byte) { + c.Code = code + c.CodeHash = hash } // SetCallCode sets the code of the contract and address of the backing data // object -func (self *Contract) SetCallCode(addr *common.Address, hash common.Hash, code []byte) { - self.Code = code - self.CodeHash = hash - self.CodeAddr = addr +func (c *Contract) SetCallCode(addr *common.Address, hash common.Hash, code []byte) { + c.Code = code + c.CodeHash = hash + c.CodeAddr = addr } diff --git a/core/vm/evm.go b/core/vm/evm.go index 96676c314a..ea46209742 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -160,6 +160,11 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas precompiles = PrecompiledContractsByzantium } if precompiles[addr] == nil && evm.ChainConfig().IsEIP158(evm.BlockNumber) && value.Sign() == 0 { + // Calling a non existing account, don't do antything, but ping the tracer + if evm.vmConfig.Debug && evm.depth == 0 { + evm.vmConfig.Tracer.CaptureStart(caller.Address(), addr, false, input, gas, value) + evm.vmConfig.Tracer.CaptureEnd(ret, 0, 0, nil) + } return nil, gas, nil } evm.StateDB.CreateAccount(addr) diff --git a/core/vm/logger.go b/core/vm/logger.go index dde1903bf2..c32a7b4044 100644 --- a/core/vm/logger.go +++ b/core/vm/logger.go @@ -31,9 +31,9 @@ import ( type Storage map[common.Hash]common.Hash -func (self Storage) Copy() Storage { +func (s Storage) Copy() Storage { cpy := make(Storage) - for key, value := range self { + for key, value := range s { cpy[key] = value } diff --git a/core/vm/memory.go b/core/vm/memory.go index 99a84d2271..d5cc7870bc 100644 --- a/core/vm/memory.go +++ b/core/vm/memory.go @@ -51,14 +51,14 @@ func (m *Memory) Resize(size uint64) { } // Get returns offset + size as a new slice -func (self *Memory) Get(offset, size int64) (cpy []byte) { +func (m *Memory) Get(offset, size int64) (cpy []byte) { if size == 0 { return nil } - if len(self.store) > int(offset) { + if len(m.store) > int(offset) { cpy = make([]byte, size) - copy(cpy, self.store[offset:offset+size]) + copy(cpy, m.store[offset:offset+size]) return } @@ -67,13 +67,13 @@ func (self *Memory) Get(offset, size int64) (cpy []byte) { } // GetPtr returns the offset + size -func (self *Memory) GetPtr(offset, size int64) []byte { +func (m *Memory) GetPtr(offset, size int64) []byte { if size == 0 { return nil } - if len(self.store) > int(offset) { - return self.store[offset : offset+size] + if len(m.store) > int(offset) { + return m.store[offset : offset+size] } return nil diff --git a/core/vm/opcodes.go b/core/vm/opcodes.go index 7fe55b72f6..e3568eb000 100644 --- a/core/vm/opcodes.go +++ b/core/vm/opcodes.go @@ -375,10 +375,10 @@ var opCodeToString = map[OpCode]string{ SWAP: "SWAP", } -func (o OpCode) String() string { - str := opCodeToString[o] +func (op OpCode) String() string { + str := opCodeToString[op] if len(str) == 0 { - return fmt.Sprintf("Missing opcode 0x%x", int(o)) + return fmt.Sprintf("Missing opcode 0x%x", int(op)) } return str diff --git a/core/vm/runtime/runtime.go b/core/vm/runtime/runtime.go index 5ac5464064..c8c5d34d9d 100644 --- a/core/vm/runtime/runtime.go +++ b/core/vm/runtime/runtime.go @@ -99,8 +99,7 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) { setDefaults(cfg) if cfg.State == nil { - db, _ := ethdb.NewMemDatabase() - cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(db)) + cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) } var ( address = common.BytesToAddress([]byte("contract")) @@ -130,8 +129,7 @@ func Create(input []byte, cfg *Config) ([]byte, common.Address, uint64, error) { setDefaults(cfg) if cfg.State == nil { - db, _ := ethdb.NewMemDatabase() - cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(db)) + cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) } var ( vmenv = NewEnv(cfg) diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index 2c4dc50265..ef664bda30 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -94,8 +94,7 @@ func TestExecute(t *testing.T) { } func TestCall(t *testing.T) { - db, _ := ethdb.NewMemDatabase() - state, _ := state.New(common.Hash{}, state.NewDatabase(db)) + state, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) address := common.HexToAddress("0x0a") state.SetCode(address, []byte{ byte(vm.PUSH1), 10, diff --git a/crypto/crypto.go b/crypto/crypto.go index 1c4d5a2e02..76d1ffaf64 100644 --- a/crypto/crypto.go +++ b/crypto/crypto.go @@ -35,8 +35,8 @@ import ( ) var ( - secp256k1_N, _ = new(big.Int).SetString("fffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141", 16) - secp256k1_halfN = new(big.Int).Div(secp256k1_N, big.NewInt(2)) + secp256k1N, _ = new(big.Int).SetString("fffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141", 16) + secp256k1halfN = new(big.Int).Div(secp256k1N, big.NewInt(2)) ) // Keccak256 calculates and returns the Keccak256 hash of the input data. @@ -68,7 +68,7 @@ func Keccak512(data ...[]byte) []byte { return d.Sum(nil) } -// Creates an ethereum address given the bytes and the nonce +// CreateAddress creates an ethereum address given the bytes and the nonce func CreateAddress(b common.Address, nonce uint64) common.Address { data, _ := rlp.EncodeToBytes([]interface{}{b, nonce}) return common.BytesToAddress(Keccak256(data)[12:]) @@ -99,7 +99,7 @@ func toECDSA(d []byte, strict bool) (*ecdsa.PrivateKey, error) { priv.D = new(big.Int).SetBytes(d) // The priv.D must < N - if priv.D.Cmp(secp256k1_N) >= 0 { + if priv.D.Cmp(secp256k1N) >= 0 { return nil, fmt.Errorf("invalid private key, >=N") } // The priv.D must not be zero or negative. @@ -184,11 +184,11 @@ func ValidateSignatureValues(v byte, r, s *big.Int, homestead bool) bool { } // reject upper range of s values (ECDSA malleability) // see discussion in secp256k1/libsecp256k1/include/secp256k1.h - if homestead && s.Cmp(secp256k1_halfN) > 0 { + if homestead && s.Cmp(secp256k1halfN) > 0 { return false } // Frontier: allow s to be in full N range - return r.Cmp(secp256k1_N) < 0 && s.Cmp(secp256k1_N) < 0 && (v == 0 || v == 1) + return r.Cmp(secp256k1N) < 0 && s.Cmp(secp256k1N) < 0 && (v == 0 || v == 1) } func PubkeyToAddress(p ecdsa.PublicKey) common.Address { diff --git a/crypto/crypto_test.go b/crypto/crypto_test.go index 8350354623..804de3fe22 100644 --- a/crypto/crypto_test.go +++ b/crypto/crypto_test.go @@ -154,7 +154,7 @@ func TestValidateSignatureValues(t *testing.T) { minusOne := big.NewInt(-1) one := common.Big1 zero := common.Big0 - secp256k1nMinus1 := new(big.Int).Sub(secp256k1_N, common.Big1) + secp256k1nMinus1 := new(big.Int).Sub(secp256k1N, common.Big1) // correct v,r,s check(true, 0, one, one) @@ -181,9 +181,9 @@ func TestValidateSignatureValues(t *testing.T) { // correct sig with max r,s check(true, 0, secp256k1nMinus1, secp256k1nMinus1) // correct v, combinations of incorrect r,s at upper limit - check(false, 0, secp256k1_N, secp256k1nMinus1) - check(false, 0, secp256k1nMinus1, secp256k1_N) - check(false, 0, secp256k1_N, secp256k1_N) + check(false, 0, secp256k1N, secp256k1nMinus1) + check(false, 0, secp256k1nMinus1, secp256k1N) + check(false, 0, secp256k1N, secp256k1N) // current callers ensures r,s cannot be negative, but let's test for that too // as crypto package could be used stand-alone diff --git a/crypto/secp256k1/curve.go b/crypto/secp256k1/curve.go index f51be5e353..6fdf2be6a4 100644 --- a/crypto/secp256k1/curve.go +++ b/crypto/secp256k1/curve.go @@ -77,7 +77,7 @@ func (BitCurve *BitCurve) Params() *elliptic.CurveParams { } } -// IsOnBitCurve returns true if the given (x,y) lies on the BitCurve. +// IsOnCurve returns true if the given (x,y) lies on the BitCurve. func (BitCurve *BitCurve) IsOnCurve(x, y *big.Int) bool { // y² = x³ + b y2 := new(big.Int).Mul(y, y) //y² diff --git a/crypto/secp256k1/secp256_test.go b/crypto/secp256k1/secp256_test.go index f6582ecd54..b608bcfcf1 100644 --- a/crypto/secp256k1/secp256_test.go +++ b/crypto/secp256k1/secp256_test.go @@ -49,7 +49,7 @@ func randSig() []byte { // tests for malleability // highest bit of signature ECDSA s value must be 0, in the 33th byte func compactSigCheck(t *testing.T, sig []byte) { - var b int = int(sig[32]) + var b = int(sig[32]) if b < 0 { t.Errorf("highest bit is negative: %d", b) } diff --git a/crypto/signature_nocgo.go b/crypto/signature_nocgo.go index f636b23772..e8fa18ed47 100644 --- a/crypto/signature_nocgo.go +++ b/crypto/signature_nocgo.go @@ -88,7 +88,7 @@ func VerifySignature(pubkey, hash, signature []byte) bool { return false } // Reject malleable signatures. libsecp256k1 does this check but btcec doesn't. - if sig.S.Cmp(secp256k1_halfN) > 0 { + if sig.S.Cmp(secp256k1halfN) > 0 { return false } return sig.Verify(hash, key) diff --git a/eth/api.go b/eth/api.go index a345b57e49..247ca7485c 100644 --- a/eth/api.go +++ b/eth/api.go @@ -19,6 +19,7 @@ package eth import ( "compress/gzip" "context" + "errors" "fmt" "io" "math/big" @@ -28,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" @@ -343,8 +345,10 @@ func NewPrivateDebugAPI(config *params.ChainConfig, eth *Ethereum) *PrivateDebug // Preimage is a debug API function that returns the preimage for a sha3 hash, if known. func (api *PrivateDebugAPI) Preimage(ctx context.Context, hash common.Hash) (hexutil.Bytes, error) { - db := core.PreimageTable(api.eth.ChainDb()) - return db.Get(hash.Bytes()) + if preimage := rawdb.ReadPreimage(api.eth.ChainDb(), hash); preimage != nil { + return preimage, nil + } + return nil, errors.New("unknown preimage") } // GetBadBLocks returns a list of the last 'bad blocks' that the client has seen on the network diff --git a/eth/api_backend.go b/eth/api_backend.go index a6f7055445..b889e5e224 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" @@ -36,26 +37,26 @@ import ( "github.com/ethereum/go-ethereum/rpc" ) -// EthApiBackend implements ethapi.Backend for full nodes -type EthApiBackend struct { +// EthAPIBackend implements ethapi.Backend for full nodes +type EthAPIBackend struct { eth *Ethereum gpo *gasprice.Oracle } -func (b *EthApiBackend) ChainConfig() *params.ChainConfig { +func (b *EthAPIBackend) ChainConfig() *params.ChainConfig { return b.eth.chainConfig } -func (b *EthApiBackend) CurrentBlock() *types.Block { +func (b *EthAPIBackend) CurrentBlock() *types.Block { return b.eth.blockchain.CurrentBlock() } -func (b *EthApiBackend) SetHead(number uint64) { +func (b *EthAPIBackend) SetHead(number uint64) { b.eth.protocolManager.downloader.Cancel() b.eth.blockchain.SetHead(number) } -func (b *EthApiBackend) HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) { +func (b *EthAPIBackend) HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) { // Pending block is only known by the miner if blockNr == rpc.PendingBlockNumber { block := b.eth.miner.PendingBlock() @@ -68,7 +69,7 @@ func (b *EthApiBackend) HeaderByNumber(ctx context.Context, blockNr rpc.BlockNum return b.eth.blockchain.GetHeaderByNumber(uint64(blockNr)), nil } -func (b *EthApiBackend) BlockByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Block, error) { +func (b *EthAPIBackend) BlockByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Block, error) { // Pending block is only known by the miner if blockNr == rpc.PendingBlockNumber { block := b.eth.miner.PendingBlock() @@ -81,7 +82,7 @@ func (b *EthApiBackend) BlockByNumber(ctx context.Context, blockNr rpc.BlockNumb return b.eth.blockchain.GetBlockByNumber(uint64(blockNr)), nil } -func (b *EthApiBackend) StateAndHeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*state.StateDB, *types.Header, error) { +func (b *EthAPIBackend) StateAndHeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*state.StateDB, *types.Header, error) { // Pending state is only known by the miner if blockNr == rpc.PendingBlockNumber { block, state := b.eth.miner.Pending() @@ -96,16 +97,23 @@ func (b *EthApiBackend) StateAndHeaderByNumber(ctx context.Context, blockNr rpc. return stateDb, header, err } -func (b *EthApiBackend) GetBlock(ctx context.Context, blockHash common.Hash) (*types.Block, error) { - return b.eth.blockchain.GetBlockByHash(blockHash), nil +func (b *EthAPIBackend) GetBlock(ctx context.Context, hash common.Hash) (*types.Block, error) { + return b.eth.blockchain.GetBlockByHash(hash), nil } -func (b *EthApiBackend) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) { - return core.GetBlockReceipts(b.eth.chainDb, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash)), nil +func (b *EthAPIBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { + if number := rawdb.ReadHeaderNumber(b.eth.chainDb, hash); number != nil { + return rawdb.ReadReceipts(b.eth.chainDb, hash, *number), nil + } + return nil, nil } -func (b *EthApiBackend) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) { - receipts := core.GetBlockReceipts(b.eth.chainDb, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash)) +func (b *EthAPIBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { + number := rawdb.ReadHeaderNumber(b.eth.chainDb, hash) + if number == nil { + return nil, nil + } + receipts := rawdb.ReadReceipts(b.eth.chainDb, hash, *number) if receipts == nil { return nil, nil } @@ -116,11 +124,11 @@ func (b *EthApiBackend) GetLogs(ctx context.Context, blockHash common.Hash) ([][ return logs, nil } -func (b *EthApiBackend) GetTd(blockHash common.Hash) *big.Int { +func (b *EthAPIBackend) GetTd(blockHash common.Hash) *big.Int { return b.eth.blockchain.GetTdByHash(blockHash) } -func (b *EthApiBackend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) { +func (b *EthAPIBackend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) { state.SetBalance(msg.From(), math.MaxBig256) vmError := func() error { return nil } @@ -128,19 +136,19 @@ func (b *EthApiBackend) GetEVM(ctx context.Context, msg core.Message, state *sta return vm.NewEVM(context, state, b.eth.chainConfig, vmCfg), vmError, nil } -func (b *EthApiBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { +func (b *EthAPIBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { return b.eth.BlockChain().SubscribeRemovedLogsEvent(ch) } -func (b *EthApiBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { +func (b *EthAPIBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { return b.eth.BlockChain().SubscribeChainEvent(ch) } -func (b *EthApiBackend) SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription { +func (b *EthAPIBackend) SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription { return b.eth.BlockChain().SubscribeChainHeadEvent(ch) } -func (b *EthApiBackend) SubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription { +func (b *EthAPIBackend) SubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription { return b.eth.BlockChain().SubscribeChainSideEvent(ch) } @@ -148,15 +156,15 @@ func (b *EthApiBackend) SubscribeTransactionEvent(ch chan<- core.TransactionEven return b.eth.BlockChain().SubscribeTransactionEvent(ch) } -func (b *EthApiBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { +func (b *EthAPIBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { return b.eth.BlockChain().SubscribeLogsEvent(ch) } -func (b *EthApiBackend) SendTx(ctx context.Context, signedTx *types.Transaction) error { +func (b *EthAPIBackend) SendTx(ctx context.Context, signedTx *types.Transaction) error { return b.eth.txPool.AddLocal(signedTx) } -func (b *EthApiBackend) GetPoolTransactions() (types.Transactions, error) { +func (b *EthAPIBackend) GetPoolTransactions() (types.Transactions, error) { pending, err := b.eth.txPool.Pending() if err != nil { return nil, err @@ -168,56 +176,56 @@ func (b *EthApiBackend) GetPoolTransactions() (types.Transactions, error) { return txs, nil } -func (b *EthApiBackend) GetPoolTransaction(hash common.Hash) *types.Transaction { +func (b *EthAPIBackend) GetPoolTransaction(hash common.Hash) *types.Transaction { return b.eth.txPool.Get(hash) } -func (b *EthApiBackend) GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) { +func (b *EthAPIBackend) GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) { return b.eth.txPool.State().GetNonce(addr), nil } -func (b *EthApiBackend) Stats() (pending int, queued int) { +func (b *EthAPIBackend) Stats() (pending int, queued int) { return b.eth.txPool.Stats() } -func (b *EthApiBackend) TxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions) { +func (b *EthAPIBackend) TxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions) { return b.eth.TxPool().Content() } -func (b *EthApiBackend) SubscribeTxPreEvent(ch chan<- core.TxPreEvent) event.Subscription { +func (b *EthAPIBackend) SubscribeTxPreEvent(ch chan<- core.TxPreEvent) event.Subscription { return b.eth.TxPool().SubscribeTxPreEvent(ch) } -func (b *EthApiBackend) Downloader() *downloader.Downloader { +func (b *EthAPIBackend) Downloader() *downloader.Downloader { return b.eth.Downloader() } -func (b *EthApiBackend) ProtocolVersion() int { +func (b *EthAPIBackend) ProtocolVersion() int { return b.eth.EthVersion() } -func (b *EthApiBackend) SuggestPrice(ctx context.Context) (*big.Int, error) { +func (b *EthAPIBackend) SuggestPrice(ctx context.Context) (*big.Int, error) { return b.gpo.SuggestPrice(ctx) } -func (b *EthApiBackend) ChainDb() ethdb.Database { +func (b *EthAPIBackend) ChainDb() ethdb.Database { return b.eth.ChainDb() } -func (b *EthApiBackend) EventMux() *event.TypeMux { +func (b *EthAPIBackend) EventMux() *event.TypeMux { return b.eth.EventMux() } -func (b *EthApiBackend) AccountManager() *accounts.Manager { +func (b *EthAPIBackend) AccountManager() *accounts.Manager { return b.eth.AccountManager() } -func (b *EthApiBackend) BloomStatus() (uint64, uint64) { +func (b *EthAPIBackend) BloomStatus() (uint64, uint64) { sections, _, _ := b.eth.bloomIndexer.Sections() return params.BloomBitsBlocks, sections } -func (b *EthApiBackend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) { +func (b *EthAPIBackend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) { for i := 0; i < bloomFilterThreads; i++ { go session.Multiplex(bloomRetrievalBatch, bloomRetrievalWait, b.eth.bloomRequests) } diff --git a/eth/api_test.go b/eth/api_test.go index 900a82bb6a..47b062a40c 100644 --- a/eth/api_test.go +++ b/eth/api_test.go @@ -31,8 +31,7 @@ var dumper = spew.ConfigState{Indent: " "} func TestStorageRangeAt(t *testing.T) { // Create a state where account 0x010000... has a few storage entries. var ( - db, _ = ethdb.NewMemDatabase() - state, _ = state.New(common.Hash{}, state.NewDatabase(db)) + state, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) addr = common.Address{0x01} keys = []common.Hash{ // hashes of Keys of storage common.HexToHash("340dd630ad21bf010b4e676dbfa9ba9a02175262d1fa356232cfde6cb5b47ef2"), diff --git a/eth/api_tracer.go b/eth/api_tracer.go index 6d86e50e03..85e4e44c46 100644 --- a/eth/api_tracer.go +++ b/eth/api_tracer.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" @@ -201,7 +202,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl log.Warn("Tracing failed", "hash", tx.Hash(), "block", task.block.NumberU64(), "err", err) break } - task.statedb.DeleteSuicides() + task.statedb.Finalise(true) task.results[i] = &txTraceResult{Result: res} } // Stream the result back to the user or abort on teardown @@ -533,7 +534,7 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (* // and returns them as a JSON object. func (api *PrivateDebugAPI) TraceTransaction(ctx context.Context, hash common.Hash, config *TraceConfig) (interface{}, error) { // Retrieve the transaction and assemble its EVM context - tx, blockHash, _, index := core.GetTransaction(api.eth.ChainDb(), hash) + tx, blockHash, _, index := rawdb.ReadTransaction(api.eth.ChainDb(), hash) if tx == nil { return nil, fmt.Errorf("transaction %x not found", hash) } @@ -640,7 +641,8 @@ func (api *PrivateDebugAPI) computeTxEnv(blockHash common.Hash, txIndex int, ree if _, _, _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { return nil, vm.Context{}, nil, fmt.Errorf("tx %x failed: %v", tx.Hash(), err) } - statedb.DeleteSuicides() + // Ensure any modifications are committed to the state + statedb.Finalise(true) } return nil, vm.Context{}, nil, fmt.Errorf("tx index %d out of range for block %x", txIndex, blockHash) } diff --git a/eth/backend.go b/eth/backend.go index ffd5d85421..ea70e3826c 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/downloader" @@ -63,8 +64,7 @@ type Ethereum struct { chainConfig *params.ChainConfig // Channel for shutting down the service - shutdownChan chan bool // Channel for shutting down the Ethereum - stopDbUpgrade func() error // stop chain db sequential key upgrade + shutdownChan chan bool // Channel for shutting down the Ethereum // Handlers txPool *core.TxPool @@ -82,7 +82,7 @@ type Ethereum struct { bloomRequests chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests bloomIndexer *core.ChainIndexer // Bloom indexer operating during block imports - ApiBackend *EthApiBackend + APIBackend *EthAPIBackend miner *miner.Miner gasPrice *big.Int @@ -112,7 +112,6 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { if err != nil { return nil, err } - stopDbUpgrade := upgradeDeduplicateData(chainDb) chainConfig, genesisHash, genesisErr := core.SetupGenesisBlock(chainDb, config.Genesis) if _, ok := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !ok { return nil, genesisErr @@ -127,7 +126,6 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { accountManager: ctx.AccountManager, engine: CreateConsensusEngine(ctx, &config.Ethash, chainConfig, chainDb), shutdownChan: make(chan bool), - stopDbUpgrade: stopDbUpgrade, networkId: config.NetworkId, gasPrice: config.GasPrice, etherbase: config.Etherbase, @@ -138,11 +136,11 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { log.Info("Initialising Ethereum protocol", "versions", ProtocolVersions, "network", config.NetworkId) if !config.SkipBcVersionCheck { - bcVersion := core.GetBlockChainVersion(chainDb) + bcVersion := rawdb.ReadDatabaseVersion(chainDb) if bcVersion != core.BlockChainVersion && bcVersion != 0 { return nil, fmt.Errorf("Blockchain DB version mismatch (%d / %d). Run geth upgradedb.\n", bcVersion, core.BlockChainVersion) } - core.WriteBlockChainVersion(chainDb, core.BlockChainVersion) + rawdb.WriteDatabaseVersion(chainDb, core.BlockChainVersion) } var ( vmConfig = vm.Config{EnablePreimageRecording: config.EnablePreimageRecording} @@ -156,7 +154,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { if compat, ok := genesisErr.(*params.ConfigCompatError); ok { log.Warn("Rewinding chain to upgrade configuration", "err", compat) eth.blockchain.SetHead(compat.RewindTo) - core.WriteChainConfig(chainDb, genesisHash, chainConfig) + rawdb.WriteChainConfig(chainDb, genesisHash, chainConfig) } eth.bloomIndexer.Start(eth.blockchain) @@ -171,12 +169,12 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { eth.miner = miner.New(eth, eth.chainConfig, eth.EventMux(), eth.engine) eth.miner.SetExtra(makeExtraData(config.ExtraData)) - eth.ApiBackend = &EthApiBackend{eth, nil} + eth.APIBackend = &EthAPIBackend{eth, nil} gpoParams := config.GPO if gpoParams.Default == nil { gpoParams.Default = config.GasPrice } - eth.ApiBackend.gpo = gasprice.NewOracle(eth.ApiBackend, gpoParams) + eth.APIBackend.gpo = gasprice.NewOracle(eth.APIBackend, gpoParams) return eth, nil } @@ -244,7 +242,7 @@ func CreateConsensusEngine(ctx *node.ServiceContext, config *ethash.Config, chai // APIs returns the collection of RPC services the ethereum package offers. // NOTE, some of these services probably need to be moved to somewhere else. func (s *Ethereum) APIs() []rpc.API { - apis := ethapi.GetAPIs(s.ApiBackend) + apis := ethapi.GetAPIs(s.APIBackend) // Append any APIs exposed explicitly by the consensus engine apis = append(apis, s.engine.APIs(s.BlockChain())...) @@ -274,7 +272,7 @@ func (s *Ethereum) APIs() []rpc.API { }, { Namespace: "eth", Version: "1.0", - Service: filters.NewPublicFilterAPI(s.ApiBackend, false), + Service: filters.NewPublicFilterAPI(s.APIBackend, false), Public: true, }, { Namespace: "admin", @@ -325,13 +323,13 @@ func (s *Ethereum) Etherbase() (eb common.Address, err error) { return common.Address{}, fmt.Errorf("etherbase must be explicitly specified") } -// set in js console via admin interface or wrapper from cli flags -func (self *Ethereum) SetEtherbase(etherbase common.Address) { - self.lock.Lock() - self.etherbase = etherbase - self.lock.Unlock() +// SetEtherbase sets the mining reward address. +func (s *Ethereum) SetEtherbase(etherbase common.Address) { + s.lock.Lock() + s.etherbase = etherbase + s.lock.Unlock() - self.miner.SetEtherbase(etherbase) + s.miner.SetEtherbase(etherbase) } func (s *Ethereum) StartMining(local bool) error { @@ -411,9 +409,6 @@ func (s *Ethereum) Start(srvr *p2p.Server) error { // Stop implements node.Service, terminating all internal goroutines used by the // Ethereum protocol. func (s *Ethereum) Stop() error { - if s.stopDbUpgrade != nil { - s.stopDbUpgrade() - } s.bloomIndexer.Close() s.blockchain.Stop() s.protocolManager.Stop() diff --git a/eth/bloombits.go b/eth/bloombits.go index c5597391c5..954239d141 100644 --- a/eth/bloombits.go +++ b/eth/bloombits.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common/bitutil" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" @@ -60,8 +61,8 @@ func (eth *Ethereum) startBloomHandlers() { task := <-request task.Bitsets = make([][]byte, len(task.Sections)) for i, section := range task.Sections { - head := core.GetCanonicalHash(eth.chainDb, (section+1)*params.BloomBitsBlocks-1) - if compVector, err := core.GetBloomBits(eth.chainDb, task.Bit, section, head); err == nil { + head := rawdb.ReadCanonicalHash(eth.chainDb, (section+1)*params.BloomBitsBlocks-1) + if compVector, err := rawdb.ReadBloomBits(eth.chainDb, task.Bit, section, head); err == nil { if blob, err := bitutil.DecompressBytes(compVector, int(params.BloomBitsBlocks)/8); err == nil { task.Bitsets[i] = blob } else { @@ -107,7 +108,7 @@ func NewBloomIndexer(db ethdb.Database, size uint64) *core.ChainIndexer { db: db, size: size, } - table := ethdb.NewTable(db, string(core.BloomBitsIndexPrefix)) + table := ethdb.NewTable(db, string(rawdb.BloomBitsIndexPrefix)) return core.NewChainIndexer(db, table, backend, size, bloomConfirms, bloomThrottling, "bloombits") } @@ -137,7 +138,7 @@ func (b *BloomIndexer) Commit() error { if err != nil { return err } - core.WriteBloomBits(batch, uint(i), b.section, b.head, bitutil.CompressBytes(bits)) + rawdb.WriteBloomBits(batch, uint(i), b.section, b.head, bitutil.CompressBytes(bits)) } return batch.Write() } diff --git a/eth/db_upgrade.go b/eth/db_upgrade.go deleted file mode 100644 index 96c584ac64..0000000000 --- a/eth/db_upgrade.go +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Package eth implements the Ethereum protocol. -package eth - -import ( - "bytes" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rlp" -) - -var deduplicateData = []byte("dbUpgrade_20170714deduplicateData") - -// upgradeDeduplicateData checks the chain database version and -// starts a background process to make upgrades if necessary. -// Returns a stop function that blocks until the process has -// been safely stopped. -func upgradeDeduplicateData(db ethdb.Database) func() error { - // If the database is already converted or empty, bail out - data, _ := db.Get(deduplicateData) - if len(data) > 0 && data[0] == 42 { - return nil - } - if data, _ := db.Get([]byte("LastHeader")); len(data) == 0 { - db.Put(deduplicateData, []byte{42}) - return nil - } - // Start the deduplication upgrade on a new goroutine - log.Warn("Upgrading database to use lookup entries") - stop := make(chan chan error) - - go func() { - // Create an iterator to read the entire database and covert old lookup entires - it := db.(*ethdb.LDBDatabase).NewIterator() - defer func() { - if it != nil { - it.Release() - } - }() - - var ( - converted uint64 - failed error - ) - for failed == nil && it.Next() { - // Skip any entries that don't look like old transaction meta entries (0x01) - key := it.Key() - if len(key) != common.HashLength+1 || key[common.HashLength] != 0x01 { - continue - } - // Skip any entries that don't contain metadata (name clash between 0x01 and ) - var meta struct { - BlockHash common.Hash - BlockIndex uint64 - Index uint64 - } - if err := rlp.DecodeBytes(it.Value(), &meta); err != nil { - continue - } - // Skip any already upgraded entries (clash due to ending with 0x01 (old suffix)) - hash := key[:common.HashLength] - - if hash[0] == byte('l') { - // Potential clash, the "old" `hash` must point to a live transaction. - if tx, _, _, _ := core.GetTransaction(db, common.BytesToHash(hash)); tx == nil || !bytes.Equal(tx.Hash().Bytes(), hash) { - continue - } - } - // Convert the old metadata to a new lookup entry, delete duplicate data - if failed = db.Put(append([]byte("l"), hash...), it.Value()); failed == nil { // Write the new lookup entry - if failed = db.Delete(hash); failed == nil { // Delete the duplicate transaction data - if failed = db.Delete(append([]byte("receipts-"), hash...)); failed == nil { // Delete the duplicate receipt data - if failed = db.Delete(key); failed != nil { // Delete the old transaction metadata - break - } - } - } - } - // Bump the conversion counter, and recreate the iterator occasionally to - // avoid too high memory consumption. - converted++ - if converted%100000 == 0 { - it.Release() - it = db.(*ethdb.LDBDatabase).NewIterator() - it.Seek(key) - - log.Info("Deduplicating database entries", "deduped", converted) - } - // Check for termination, or continue after a bit of a timeout - select { - case errc := <-stop: - errc <- nil - return - case <-time.After(time.Microsecond * 100): - } - } - // Upgrade finished, mark a such and terminate - if failed == nil { - log.Info("Database deduplication successful", "deduped", converted) - db.Put(deduplicateData, []byte{42}) - } else { - log.Error("Database deduplication failed", "deduped", converted, "err", failed) - } - it.Release() - it = nil - - errc := <-stop - errc <- failed - }() - // Assembly the cancellation callback - return func() error { - errc := make(chan error) - stop <- errc - return <-errc - } -} diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 43f0e3db97..dc23354929 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -27,7 +27,7 @@ import ( ethereum "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -224,7 +224,7 @@ func New(mode SyncMode, stateDb ethdb.Database, mux *event.TypeMux, chain BlockC stateCh: make(chan dataPack), stateSyncStart: make(chan *stateSync), syncStatsState: stateSyncStats{ - processed: core.GetTrieSyncProgress(stateDb), + processed: rawdb.ReadFastTrieProgress(stateDb), }, trackStateReq: make(chan *stateReq), } diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index e85e234c0e..d1a9a8694b 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -75,7 +75,7 @@ type downloadTester struct { // newTester creates a new downloader test mocker. func newTester() *downloadTester { - testdb, _ := ethdb.NewMemDatabase() + testdb := ethdb.NewMemDatabase() genesis := core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000)) tester := &downloadTester{ @@ -93,7 +93,7 @@ func newTester() *downloadTester { peerChainTds: make(map[string]map[common.Hash]*big.Int), peerMissingStates: make(map[string]map[common.Hash]bool), } - tester.stateDb, _ = ethdb.NewMemDatabase() + tester.stateDb = ethdb.NewMemDatabase() tester.stateDb.Put(genesis.Root().Bytes(), []byte{0x00}) tester.downloader = New(FullSync, tester.stateDb, new(event.TypeMux), tester, nil, tester.dropPeer) diff --git a/eth/downloader/fakepeer.go b/eth/downloader/fakepeer.go index 5248e7fb0c..59832facaa 100644 --- a/eth/downloader/fakepeer.go +++ b/eth/downloader/fakepeer.go @@ -21,6 +21,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" ) @@ -126,7 +127,7 @@ func (p *FakePeer) RequestBodies(hashes []common.Hash) error { uncles [][]*types.Header ) for _, hash := range hashes { - block := core.GetBlock(p.db, hash, p.hc.GetBlockNumber(hash)) + block := rawdb.ReadBlock(p.db, hash, *p.hc.GetBlockNumber(hash)) txs = append(txs, block.Transactions()) uncles = append(uncles, block.Uncles()) @@ -140,7 +141,7 @@ func (p *FakePeer) RequestBodies(hashes []common.Hash) error { func (p *FakePeer) RequestReceipts(hashes []common.Hash) error { var receipts [][]*types.Receipt for _, hash := range hashes { - receipts = append(receipts, core.GetBlockReceipts(p.db, hash, p.hc.GetBlockNumber(hash))) + receipts = append(receipts, rawdb.ReadReceipts(p.db, hash, *p.hc.GetBlockNumber(hash))) } p.dl.DeliverReceipts(p.id, receipts) return nil diff --git a/eth/downloader/statesync.go b/eth/downloader/statesync.go index 521ee25a02..5b4b9ba1b7 100644 --- a/eth/downloader/statesync.go +++ b/eth/downloader/statesync.go @@ -23,7 +23,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/ethdb" @@ -476,6 +476,6 @@ func (s *stateSync) updateStats(written, duplicate, unexpected int, duration tim log.Info("Imported new state entries", "count", written, "elapsed", common.PrettyDuration(duration), "processed", s.d.syncStatsState.processed, "pending", s.d.syncStatsState.pending, "retry", len(s.tasks), "duplicate", s.d.syncStatsState.duplicate, "unexpected", s.d.syncStatsState.unexpected) } if written > 0 { - core.WriteTrieSyncProgress(s.d.stateDB, s.d.syncStatsState.processed) + rawdb.WriteFastTrieProgress(s.d.stateDB, s.d.syncStatsState.processed) } } diff --git a/eth/fetcher/fetcher_test.go b/eth/fetcher/fetcher_test.go index 1e536fcacb..3d4f0d1e59 100644 --- a/eth/fetcher/fetcher_test.go +++ b/eth/fetcher/fetcher_test.go @@ -34,7 +34,7 @@ import ( ) var ( - testdb, _ = ethdb.NewMemDatabase() + testdb = ethdb.NewMemDatabase() testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") testAddress = crypto.PubkeyToAddress(testKey.PublicKey) genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000)) diff --git a/eth/filters/api.go b/eth/filters/api.go index ec14fb6e6e..9bee4dde15 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -345,14 +345,8 @@ func (api *PublicFilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc } // FilterCriteria represents a request to create a new filter. -// -// TODO(karalabe): Kill this in favor of ethereum.FilterQuery. -type FilterCriteria struct { - FromBlock *big.Int - ToBlock *big.Int - Addresses []common.Address - Topics [][]common.Hash -} +// Same as ethereum.FilterQuery but with UnmarshalJSON() method. +type FilterCriteria ethereum.FilterQuery // NewFilter creates a new filter and returns the filter id. It can be // used to retrieve logs when the state changes. This method cannot be diff --git a/eth/filters/bench_test.go b/eth/filters/bench_test.go index d2aaf42a9e..dfc9978716 100644 --- a/eth/filters/bench_test.go +++ b/eth/filters/bench_test.go @@ -25,8 +25,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/bitutil" - "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -71,20 +71,20 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) { if err != nil { b.Fatalf("error opening database at %v: %v", benchDataDir, err) } - head := core.GetHeadBlockHash(db) + head := rawdb.ReadHeadBlockHash(db) if head == (common.Hash{}) { b.Fatalf("chain data not found at %v", benchDataDir) } clearBloomBits(db) fmt.Println("Generating bloombits data...") - headNum := core.GetBlockNumber(db, head) - if headNum < sectionSize+512 { + headNum := rawdb.ReadHeaderNumber(db, head) + if headNum == nil || *headNum < sectionSize+512 { b.Fatalf("not enough blocks for running a benchmark") } start := time.Now() - cnt := (headNum - 512) / sectionSize + cnt := (*headNum - 512) / sectionSize var dataSize, compSize uint64 for sectionIdx := uint64(0); sectionIdx < cnt; sectionIdx++ { bc, err := bloombits.NewGenerator(uint(sectionSize)) @@ -93,14 +93,14 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) { } var header *types.Header for i := sectionIdx * sectionSize; i < (sectionIdx+1)*sectionSize; i++ { - hash := core.GetCanonicalHash(db, i) - header = core.GetHeader(db, hash, i) + hash := rawdb.ReadCanonicalHash(db, i) + header = rawdb.ReadHeader(db, hash, i) if header == nil { b.Fatalf("Error creating bloomBits data") } bc.AddBloom(uint(i-sectionIdx*sectionSize), header.Bloom) } - sectionHead := core.GetCanonicalHash(db, (sectionIdx+1)*sectionSize-1) + sectionHead := rawdb.ReadCanonicalHash(db, (sectionIdx+1)*sectionSize-1) for i := 0; i < types.BloomBitLength; i++ { data, err := bc.Bitset(uint(i)) if err != nil { @@ -109,7 +109,7 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) { comp := bitutil.CompressBytes(data) dataSize += uint64(len(data)) compSize += uint64(len(comp)) - core.WriteBloomBits(db, uint(i), sectionIdx, sectionHead, comp) + rawdb.WriteBloomBits(db, uint(i), sectionIdx, sectionHead, comp) } //if sectionIdx%50 == 0 { // fmt.Println(" section", sectionIdx, "/", cnt) @@ -180,22 +180,24 @@ func BenchmarkNoBloomBits(b *testing.B) { if err != nil { b.Fatalf("error opening database at %v: %v", benchDataDir, err) } - head := core.GetHeadBlockHash(db) + head := rawdb.ReadHeadBlockHash(db) if head == (common.Hash{}) { b.Fatalf("chain data not found at %v", benchDataDir) } - headNum := core.GetBlockNumber(db, head) + headNum := rawdb.ReadHeaderNumber(db, head) clearBloomBits(db) fmt.Println("Running filter benchmarks...") start := time.Now() mux := new(event.TypeMux) + backend := &testBackend{mux, db, 0, new(event.Feed), new(event.Feed), new(event.Feed), new(event.Feed), new(event.Feed)} - filter := New(backend, 0, int64(headNum), []common.Address{{}}, nil) + filter := New(backend, 0, int64(*headNum), []common.Address{{}}, nil) + filter.Logs(context.Background()) d := time.Since(start) fmt.Println("Finished running filter benchmarks") - fmt.Println(" ", d, "total ", d*time.Duration(1000000)/time.Duration(headNum+1), "per million blocks") + fmt.Println(" ", d, "total ", d*time.Duration(1000000)/time.Duration(*headNum+1), "per million blocks") db.Close() } diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index 2c991f6d8d..a5234bf212 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -28,8 +28,10 @@ import ( ethereum "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" ) @@ -98,8 +100,23 @@ type EventSystem struct { backend Backend lightMode bool lastHead *types.Header - install chan *subscription // install filter for event notification - uninstall chan *subscription // remove filter for event notification + + // Subscriptions + txPreSub event.Subscription // Subscription for pre transaction events + txSub event.Subscription // Subscription for transaction events + logsSub event.Subscription // Subscription for new log event + rmLogsSub event.Subscription // Subscription for removed log event + chainSub event.Subscription // Subscription for new chain event + pendingLogSub *event.TypeMuxSubscription // Subscription for pending log event + + // Channels + install chan *subscription // install filter for event notification + uninstall chan *subscription // remove filter for event notification + txPreCh chan core.TxPreEvent // Channel to recieve new pre transaction events + txCh chan core.TransactionEvent // Channel to receive new transaction events + logsCh chan []*types.Log // Channel to receive new log event + rmLogsCh chan core.RemovedLogsEvent // Channel to receive removed log event + chainCh chan core.ChainEvent // Channel to receive new chain event } // NewEventSystem creates a new manager that listens for event on the given mux, @@ -115,10 +132,29 @@ func NewEventSystem(mux *event.TypeMux, backend Backend, lightMode bool) *EventS lightMode: lightMode, install: make(chan *subscription), uninstall: make(chan *subscription), + txPrech: make(chan core.TxPreEvent, txPreChanSize), + txCh: make(chan core.TransactionEvent, txPostChanSize), + logsCh: make(chan []*types.Log, logsChanSize), + rmLogsCh: make(chan core.RemovedLogsEvent, rmLogsChanSize), + chainCh: make(chan core.ChainEvent, chainEvChanSize), + } + + // Subscribe events + m.txPreSub = m.backend.SubscribeTxPreEvent(m.txPreCh) + m.txSub = m.backend.SubscribeTransactionEvent(m.txCh) + m.logsSub = m.backend.SubscribeLogsEvent(m.logsCh) + m.rmLogsSub = m.backend.SubscribeRemovedLogsEvent(m.rmLogsCh) + m.chainSub = m.backend.SubscribeChainEvent(m.chainCh) + // TODO(rjl493456442): use feed to subscribe pending log event + m.pendingLogSub = m.mux.Subscribe(core.PendingLogsEvent{}) + + // Make sure none of the subscriptions are empty + if m.txSub == nil || m.logsSub == nil || m.rmLogsSub == nil || m.chainSub == nil || + m.pendingLogSub.Closed() { + log.Crit("Subscribe for event system failed") } go m.eventLoop() - return m } @@ -373,11 +409,11 @@ func (es *EventSystem) lightFilterNewHead(newHeader *types.Header, callBack func for oldh.Hash() != newh.Hash() { if oldh.Number.Uint64() >= newh.Number.Uint64() { oldHeaders = append(oldHeaders, oldh) - oldh = core.GetHeader(es.backend.ChainDb(), oldh.ParentHash, oldh.Number.Uint64()-1) + oldh = rawdb.ReadHeader(es.backend.ChainDb(), oldh.ParentHash, oldh.Number.Uint64()-1) } if oldh.Number.Uint64() < newh.Number.Uint64() { newHeaders = append(newHeaders, newh) - newh = core.GetHeader(es.backend.ChainDb(), newh.ParentHash, newh.Number.Uint64()-1) + newh = rawdb.ReadHeader(es.backend.ChainDb(), newh.ParentHash, newh.Number.Uint64()-1) if newh == nil { // happens when CHT syncing, nothing to do newh = oldh @@ -436,58 +472,41 @@ func (es *EventSystem) lightFilterLogs(header *types.Header, addresses []common. // eventLoop (un)installs filters and processes mux events. func (es *EventSystem) eventLoop() { - var ( - index = make(filterIndex) - sub = es.mux.Subscribe(core.PendingLogsEvent{}) - // Subscribe to TxPreEvent from txpool - txPreCh = make(chan core.TxPreEvent, txPreChanSize) - txPreSub = es.backend.SubscribeTxPreEvent(txPreCh) - // Subscribe to TransactionEvent from applyTransaction - txCh = make(chan core.TransactionEvent, txPostChanSize) - txSub = es.backend.SubscribeTransactionEvent(txCh) - // Subscribe RemovedLogsEvent - rmLogsCh = make(chan core.RemovedLogsEvent, rmLogsChanSize) - rmLogsSub = es.backend.SubscribeRemovedLogsEvent(rmLogsCh) - // Subscribe []*types.Log - logsCh = make(chan []*types.Log, logsChanSize) - logsSub = es.backend.SubscribeLogsEvent(logsCh) - // Subscribe ChainEvent - chainEvCh = make(chan core.ChainEvent, chainEvChanSize) - chainEvSub = es.backend.SubscribeChainEvent(chainEvCh) - ) - - // Unsubscribe all events - defer sub.Unsubscribe() - defer txPreSub.Unsubscribe() - defer txSub.Unsubscribe() - defer rmLogsSub.Unsubscribe() - defer logsSub.Unsubscribe() - defer chainEvSub.Unsubscribe() + // Ensure all subscriptions get cleaned up + defer func() { + es.pendingLogSub.Unsubscribe() + es.txPreSub.Unsubscribe() + es.txSub.Unsubscribe() + es.logsSub.Unsubscribe() + es.rmLogsSub.Unsubscribe() + es.chainSub.Unsubscribe() + }() + index := make(filterIndex) + for i := UnknownSubscription; i < LastIndexSubscription; i++ { index[i] = make(map[rpc.ID]*subscription) } for { select { - case ev, active := <-sub.Chan(): + // Handle subscribed events + case ev := <-es.txPreCh: + es.broadcast(index, ev) + case ev := <-es.txCh: + es.broadcast(index, ev) + case ev := <-es.logsCh: + es.broadcast(index, ev) + case ev := <-es.rmLogsCh: + es.broadcast(index, ev) + case ev := <-es.chainCh: + es.broadcast(index, ev) + case ev, active := <-es.pendingLogSub.Chan(): if !active { // system stopped return } es.broadcast(index, ev) - // Handle subscribed events - case ev := <-txPreCh: - es.broadcast(index, ev) - case ev := <-txCh: - es.broadcast(index, ev) - case ev := <-rmLogsCh: - es.broadcast(index, ev) - case ev := <-logsCh: - es.broadcast(index, ev) - case ev := <-chainEvCh: - es.broadcast(index, ev) - case f := <-es.install: if f.typ == MinedAndPendingLogsSubscription { // the type are logs and pending logs subscriptions @@ -497,6 +516,7 @@ func (es *EventSystem) eventLoop() { index[f.typ][f.id] = f } close(f.installed) + case f := <-es.uninstall: if f.typ == MinedAndPendingLogsSubscription { // the type are logs and pending logs subscriptions @@ -508,15 +528,15 @@ func (es *EventSystem) eventLoop() { close(f.err) // System stopped - case <-txPreSub.Err(): + case <-es.txPreSub.Err(): return - case <-txSub.Err(): + case <-es.txSub.Err(): return - case <-rmLogsSub.Err(): + case <-es.logsSub.Err(): return - case <-logsSub.Err(): + case <-es.rmLogsSub.Err(): return - case <-chainEvSub.Err(): + case <-es.chainSub.Err(): return } } diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 3ce7de37ce..1a68262fe4 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -57,26 +58,37 @@ func (b *testBackend) EventMux() *event.TypeMux { } func (b *testBackend) HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) { - var hash common.Hash - var num uint64 + var ( + hash common.Hash + num uint64 + ) if blockNr == rpc.LatestBlockNumber { - hash = core.GetHeadBlockHash(b.db) - num = core.GetBlockNumber(b.db, hash) + hash = rawdb.ReadHeadBlockHash(b.db) + number := rawdb.ReadHeaderNumber(b.db, hash) + if number == nil { + return nil, nil + } + num = *number } else { num = uint64(blockNr) - hash = core.GetCanonicalHash(b.db, num) + hash = rawdb.ReadCanonicalHash(b.db, num) } - return core.GetHeader(b.db, hash, num), nil + return rawdb.ReadHeader(b.db, hash, num), nil } -func (b *testBackend) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) { - number := core.GetBlockNumber(b.db, blockHash) - return core.GetBlockReceipts(b.db, blockHash, number), nil +func (b *testBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { + if number := rawdb.ReadHeaderNumber(b.db, hash); number != nil { + return rawdb.ReadReceipts(b.db, hash, *number), nil + } + return nil, nil } -func (b *testBackend) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) { - number := core.GetBlockNumber(b.db, blockHash) - receipts := core.GetBlockReceipts(b.db, blockHash, number) +func (b *testBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { + number := rawdb.ReadHeaderNumber(b.db, hash) + if number == nil { + return nil, nil + } + receipts := rawdb.ReadReceipts(b.db, hash, *number) logs := make([][]*types.Log, len(receipts)) for i, receipt := range receipts { @@ -126,8 +138,8 @@ func (b *testBackend) ServiceFilter(ctx context.Context, session *bloombits.Matc task.Bitsets = make([][]byte, len(task.Sections)) for i, section := range task.Sections { if rand.Int()%4 != 0 { // Handle occasional missing deliveries - head := core.GetCanonicalHash(b.db, (section+1)*params.BloomBitsBlocks-1) - task.Bitsets[i], _ = core.GetBloomBits(b.db, task.Bit, section, head) + head := rawdb.ReadCanonicalHash(b.db, (section+1)*params.BloomBitsBlocks-1) + task.Bitsets[i], _ = rawdb.ReadBloomBits(b.db, task.Bit, section, head) } } request <- task @@ -146,7 +158,7 @@ func TestBlockSubscription(t *testing.T) { var ( mux = new(event.TypeMux) - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() txPreFeed = new(event.Feed) txPostFeed = new(event.Feed) rmLogsFeed = new(event.Feed) @@ -204,7 +216,7 @@ func TestPendingTxFilter(t *testing.T) { var ( mux = new(event.TypeMux) - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() txPreFeed = new(event.Feed) txPostFeed = new(event.Feed) rmLogsFeed = new(event.Feed) @@ -268,7 +280,7 @@ func TestPendingTxFilter(t *testing.T) { func TestLogFilterCreation(t *testing.T) { var ( mux = new(event.TypeMux) - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() txPreFeed = new(event.Feed) txPostFeed = new(event.Feed) rmLogsFeed = new(event.Feed) @@ -318,9 +330,9 @@ func TestInvalidLogFilterCreation(t *testing.T) { var ( mux = new(event.TypeMux) - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() txPreFeed = new(event.Feed) - txPostFeed = new(event.Feed) + txPostFeed = new(evnt.Feed) rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) chainFeed = new(event.Feed) @@ -349,7 +361,7 @@ func TestLogFilter(t *testing.T) { var ( mux = new(event.TypeMux) - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() txPreFeed = new(event.Feed) txPostFeed = new(event.Feed) rmLogsFeed = new(event.Feed) @@ -469,7 +481,7 @@ func TestPendingLogsSubscription(t *testing.T) { var ( mux = new(event.TypeMux) - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() txPreFeed = new(event.Feed) txPostFeed = new(event.Feed) rmLogsFeed = new(event.Feed) diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index 8c531f93b5..d5ad83d950 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" @@ -85,16 +86,10 @@ func BenchmarkFilters(b *testing.B) { } }) for i, block := range chain { - core.WriteBlock(db, block) - if err := core.WriteCanonicalHash(db, block.Hash(), block.NumberU64()); err != nil { - b.Fatalf("failed to insert block number: %v", err) - } - if err := core.WriteHeadBlockHash(db, block.Hash()); err != nil { - b.Fatalf("failed to insert block number: %v", err) - } - if err := core.WriteBlockReceipts(db, block.Hash(), block.NumberU64(), receipts[i]); err != nil { - b.Fatal("error writing block receipts:", err) - } + rawdb.WriteBlock(db, block) + rawdb.WriteCanonicalHash(db, block.Hash(), block.NumberU64()) + rawdb.WriteHeadBlockHash(db, block.Hash()) + rawdb.WriteReceipts(db, block.Hash(), block.NumberU64(), receipts[i]) } b.ResetTimer() @@ -176,16 +171,10 @@ func TestFilters(t *testing.T) { } }) for i, block := range chain { - core.WriteBlock(db, block) - if err := core.WriteCanonicalHash(db, block.Hash(), block.NumberU64()); err != nil { - t.Fatalf("failed to insert block number: %v", err) - } - if err := core.WriteHeadBlockHash(db, block.Hash()); err != nil { - t.Fatalf("failed to insert block number: %v", err) - } - if err := core.WriteBlockReceipts(db, block.Hash(), block.NumberU64(), receipts[i]); err != nil { - t.Fatal("error writing block receipts:", err) - } + rawdb.WriteBlock(db, block) + rawdb.WriteCanonicalHash(db, block.Hash(), block.NumberU64()) + rawdb.WriteHeadBlockHash(db, block.Hash()) + rawdb.WriteReceipts(db, block.Hash(), block.NumberU64(), receipts[i]) } filter := New(backend, 0, -1, []common.Address{addr}, [][]common.Hash{{hash1, hash2, hash3, hash4}}) diff --git a/eth/handler.go b/eth/handler.go index 4069359c9e..c8f7e13f16 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -725,25 +725,25 @@ func (pm *ProtocolManager) BroadcastTx(hash common.Hash, tx *types.Transaction) } // Mined broadcast loop -func (self *ProtocolManager) minedBroadcastLoop() { +func (pm *ProtocolManager) minedBroadcastLoop() { // automatically stops if unsubscribe - for obj := range self.minedBlockSub.Chan() { + for obj := range pm.minedBlockSub.Chan() { switch ev := obj.Data.(type) { case core.NewMinedBlockEvent: - self.BroadcastBlock(ev.Block, true) // First propagate block to peers - self.BroadcastBlock(ev.Block, false) // Only then announce to the rest + pm.BroadcastBlock(ev.Block, true) // First propagate block to peers + pm.BroadcastBlock(ev.Block, false) // Only then announce to the rest } } } -func (self *ProtocolManager) txBroadcastLoop() { +func (pm *ProtocolManager) txBroadcastLoop() { for { select { - case event := <-self.txCh: - self.BroadcastTx(event.Tx.Hash(), event.Tx) + case event := <-pm.txCh: + pm.BroadcastTx(event.Tx.Hash(), event.Tx) // Err() channel will be closed when unsubscribing. - case <-self.txSub.Err(): + case <-pm.txSub.Err(): return } } @@ -760,13 +760,13 @@ type NodeInfo struct { } // NodeInfo retrieves some protocol metadata about the running host node. -func (self *ProtocolManager) NodeInfo() *NodeInfo { - currentBlock := self.blockchain.CurrentBlock() +func (pm *ProtocolManager) NodeInfo() *NodeInfo { + currentBlock := pm.blockchain.CurrentBlock() return &NodeInfo{ - Network: self.networkId, - Difficulty: self.blockchain.GetTd(currentBlock.Hash(), currentBlock.NumberU64()), - Genesis: self.blockchain.Genesis().Hash(), - Config: self.blockchain.Config(), + Network: pm.networkId, + Difficulty: pm.blockchain.GetTd(currentBlock.Hash(), currentBlock.NumberU64()), + Genesis: pm.blockchain.Genesis().Hash(), + Config: pm.blockchain.Config(), Head: currentBlock.Hash(), } } diff --git a/eth/handler_test.go b/eth/handler_test.go index e336dfa285..fee4114eb6 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -366,7 +366,7 @@ func testGetNodeData(t *testing.T, protocol int) { t.Errorf("data hash mismatch: have %x, want %x", hash, want) } } - statedb, _ := ethdb.NewMemDatabase() + statedb := ethdb.NewMemDatabase() for i := 0; i < len(data); i++ { statedb.Put(hashes[i].Bytes(), data[i]) } @@ -468,7 +468,7 @@ func testDAOChallenge(t *testing.T, localForked, remoteForked bool, timeout bool var ( evmux = new(event.TypeMux) pow = ethash.NewFaker() - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() config = ¶ms.ChainConfig{DAOForkBlock: big.NewInt(1), DAOForkSupport: localForked} gspec = &core.Genesis{Config: config} genesis = gspec.MustCommit(db) diff --git a/eth/helper_test.go b/eth/helper_test.go index 2b05cea801..8a0260fc95 100644 --- a/eth/helper_test.go +++ b/eth/helper_test.go @@ -53,7 +53,7 @@ func newTestProtocolManager(mode downloader.SyncMode, blocks int, generator func var ( evmux = new(event.TypeMux) engine = ethash.NewFaker() - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() gspec = &core.Genesis{ Config: params.TestChainConfig, Alloc: core.GenesisAlloc{testBank: {Balance: big.NewInt(1000000)}}, diff --git a/eth/protocol.go b/eth/protocol.go index cd7db57f23..328d5b9939 100644 --- a/eth/protocol.go +++ b/eth/protocol.go @@ -34,13 +34,13 @@ const ( eth63 = 63 ) -// Official short name of the protocol used during capability negotiation. +// ProtocolName is the official short name of the protocol used during capability negotiation. var ProtocolName = "eth" -// Supported versions of the eth protocol (first is primary). +// ProtocolVersions are the upported versions of the eth protocol (first is primary). var ProtocolVersions = []uint{eth63, eth62} -// Number of implemented message corresponding to different protocol versions. +// ProtocolLengths are the number of implemented message corresponding to different protocol versions. var ProtocolLengths = []uint64{17, 8} const ProtocolMaxMsgSize = 10 * 1024 * 1024 // Maximum cap on the size of a protocol message diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index bf8120228f..d25fc459a1 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -159,8 +159,7 @@ func TestCallTracer(t *testing.T) { GasLimit: uint64(test.Context.GasLimit), GasPrice: tx.GasPrice(), } - db, _ := ethdb.NewMemDatabase() - statedb := tests.MakePreState(db, test.Genesis.Alloc) + statedb := tests.MakePreState(ethdb.NewMemDatabase(), test.Genesis.Alloc) // Create the tracer, the EVM environment and run it tracer, err := New("callTracer") diff --git a/ethdb/database_test.go b/ethdb/database_test.go index 5e4a3ca34a..2deb50988c 100644 --- a/ethdb/database_test.go +++ b/ethdb/database_test.go @@ -53,8 +53,7 @@ func TestLDB_PutGet(t *testing.T) { } func TestMemoryDB_PutGet(t *testing.T) { - db, _ := ethdb.NewMemDatabase() - testPutGet(db, t) + testPutGet(ethdb.NewMemDatabase(), t) } func testPutGet(db ethdb.Database, t *testing.T) { @@ -131,8 +130,7 @@ func TestLDB_ParallelPutGet(t *testing.T) { } func TestMemoryDB_ParallelPutGet(t *testing.T) { - db, _ := ethdb.NewMemDatabase() - testParallelPutGet(db, t) + testParallelPutGet(ethdb.NewMemDatabase(), t) } func testParallelPutGet(db ethdb.Database, t *testing.T) { diff --git a/ethdb/memory_database.go b/ethdb/memory_database.go index 8efd7bf845..c57042920a 100644 --- a/ethdb/memory_database.go +++ b/ethdb/memory_database.go @@ -31,16 +31,16 @@ type MemDatabase struct { lock sync.RWMutex } -func NewMemDatabase() (*MemDatabase, error) { +func NewMemDatabase() *MemDatabase { return &MemDatabase{ db: make(map[string][]byte), - }, nil + } } -func NewMemDatabaseWithCap(size int) (*MemDatabase, error) { +func NewMemDatabaseWithCap(size int) *MemDatabase { return &MemDatabase{ db: make(map[string][]byte, size), - }, nil + } } func (db *MemDatabase) Put(key []byte, value []byte) error { diff --git a/ethstats/ethstats.go b/ethstats/ethstats.go index 13b39c7caf..a1eba1ed3e 100644 --- a/ethstats/ethstats.go +++ b/ethstats/ethstats.go @@ -689,7 +689,7 @@ func (s *Service) reportStats(conn *websocket.Conn) error { sync := s.eth.Downloader().Progress() syncing = s.eth.BlockChain().CurrentHeader().Number.Uint64() >= sync.HighestBlock - price, _ := s.eth.ApiBackend.SuggestPrice(context.Background()) + price, _ := s.eth.APIBackend.SuggestPrice(context.Background()) gasprice = int(price.Uint64()) } else { sync := s.les.Downloader().Progress() diff --git a/event/event.go b/event/event.go index 20d20d1f57..4232787314 100644 --- a/event/event.go +++ b/event/event.go @@ -180,6 +180,12 @@ func (s *TypeMuxSubscription) Unsubscribe() { s.closewait() } +func (s *TypeMuxSubscription) Closed() bool { + s.closeMu.Lock() + defer s.closeMu.Unlock() + return s.closed +} + func (s *TypeMuxSubscription) closewait() { s.closeMu.Lock() defer s.closeMu.Unlock() diff --git a/event/feed.go b/event/feed.go index 78fa3d98d8..f578f00c10 100644 --- a/event/feed.go +++ b/event/feed.go @@ -148,7 +148,9 @@ func (f *Feed) Send(value interface{}) (nsent int) { f.sendCases[i].Send = rvalue } - // Send until all channels except removeSub have been chosen. + // Send until all channels except removeSub have been chosen. 'cases' tracks a prefix + // of sendCases. When a send succeeds, the corresponding case moves to the end of + // 'cases' and it shrinks by one element. cases := f.sendCases for { // Fast path: try sending without blocking before adding to the select set. @@ -170,6 +172,7 @@ func (f *Feed) Send(value interface{}) (nsent int) { index := f.sendCases.find(recv.Interface()) f.sendCases = f.sendCases.delete(index) if index >= 0 && index < len(cases) { + // Shrink 'cases' too because the removed case was still active. cases = f.sendCases[:len(cases)-1] } } else { diff --git a/event/feed_test.go b/event/feed_test.go index a82c103033..be8876932c 100644 --- a/event/feed_test.go +++ b/event/feed_test.go @@ -235,6 +235,45 @@ func TestFeedUnsubscribeBlockedPost(t *testing.T) { wg.Wait() } +// Checks that unsubscribing a channel during Send works even if that +// channel has already been sent on. +func TestFeedUnsubscribeSentChan(t *testing.T) { + var ( + feed Feed + ch1 = make(chan int) + ch2 = make(chan int) + sub1 = feed.Subscribe(ch1) + sub2 = feed.Subscribe(ch2) + wg sync.WaitGroup + ) + defer sub2.Unsubscribe() + + wg.Add(1) + go func() { + feed.Send(0) + wg.Done() + }() + + // Wait for the value on ch1. + <-ch1 + // Unsubscribe ch1, removing it from the send cases. + sub1.Unsubscribe() + + // Receive ch2, finishing Send. + <-ch2 + wg.Wait() + + // Send again. This should send to ch2 only, so the wait group will unblock + // as soon as a value is received on ch2. + wg.Add(1) + go func() { + feed.Send(0) + wg.Done() + }() + <-ch2 + wg.Wait() +} + func TestFeedUnsubscribeFromInbox(t *testing.T) { var ( feed Feed diff --git a/event/filter/filter.go b/event/filter/filter.go index b1fbf30ee4..a6fe46d6a0 100644 --- a/event/filter/filter.go +++ b/event/filter/filter.go @@ -45,37 +45,37 @@ func New() *Filters { } } -func (self *Filters) Start() { - go self.loop() +func (f *Filters) Start() { + go f.loop() } -func (self *Filters) Stop() { - close(self.quit) +func (f *Filters) Stop() { + close(f.quit) } -func (self *Filters) Notify(filter Filter, data interface{}) { - self.ch <- FilterEvent{filter, data} +func (f *Filters) Notify(filter Filter, data interface{}) { + f.ch <- FilterEvent{filter, data} } -func (self *Filters) Install(watcher Filter) int { - self.watchers[self.id] = watcher - self.id++ +func (f *Filters) Install(watcher Filter) int { + f.watchers[f.id] = watcher + f.id++ - return self.id - 1 + return f.id - 1 } -func (self *Filters) Uninstall(id int) { - delete(self.watchers, id) +func (f *Filters) Uninstall(id int) { + delete(f.watchers, id) } -func (self *Filters) loop() { +func (f *Filters) loop() { out: for { select { - case <-self.quit: + case <-f.quit: break out - case event := <-self.ch: - for _, watcher := range self.watchers { + case event := <-f.ch: + for _, watcher := range f.watchers { if reflect.TypeOf(watcher) == reflect.TypeOf(event.filter) { if watcher.Compare(event.filter) { watcher.Trigger(event.data) @@ -86,10 +86,10 @@ out: } } -func (self *Filters) Match(a, b Filter) bool { +func (f *Filters) Match(a, b Filter) bool { return reflect.TypeOf(a) == reflect.TypeOf(b) && a.Compare(b) } -func (self *Filters) Get(i int) Filter { - return self.watchers[i] +func (f *Filters) Get(i int) Filter { + return f.watchers[i] } diff --git a/internal/build/pgp.go b/internal/build/pgp.go index 79ab9c06f1..c7d0d23397 100644 --- a/internal/build/pgp.go +++ b/internal/build/pgp.go @@ -57,3 +57,15 @@ func PGPSignFile(input string, output string, pgpkey string) error { // Generate the signature and return return openpgp.ArmoredDetachSign(out, keys[0], in, nil) } + +// PGPKeyID parses an armored key and returns the key ID. +func PGPKeyID(pgpkey string) (string, error) { + keys, err := openpgp.ReadArmoredKeyRing(bytes.NewBufferString(pgpkey)) + if err != nil { + return "", err + } + if len(keys) != 1 { + return "", fmt.Errorf("key count mismatch: have %d, want %d", len(keys), 1) + } + return keys[0].PrimaryKey.KeyIdString(), nil +} diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 2463682612..c053a38ae4 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" @@ -1006,7 +1007,7 @@ func (s *PublicTransactionPoolAPI) GetTransactionCount(ctx context.Context, addr // GetTransactionByHash returns the transaction for the given hash func (s *PublicTransactionPoolAPI) GetTransactionByHash(ctx context.Context, hash common.Hash) *RPCTransaction { // Try to return an already finalized transaction - if tx, blockHash, blockNumber, index := core.GetTransaction(s.b.ChainDb(), hash); tx != nil { + if tx, blockHash, blockNumber, index := rawdb.ReadTransaction(s.b.ChainDb(), hash); tx != nil { return newRPCTransaction(tx, blockHash, blockNumber, index) } // No finalized transaction, try to retrieve it from the pool @@ -1022,7 +1023,7 @@ func (s *PublicTransactionPoolAPI) GetRawTransactionByHash(ctx context.Context, var tx *types.Transaction // Retrieve a finalized transaction, or a pooled otherwise - if tx, _, _, _ = core.GetTransaction(s.b.ChainDb(), hash); tx == nil { + if tx, _, _, _ = rawdb.ReadTransaction(s.b.ChainDb(), hash); tx == nil { if tx = s.b.GetPoolTransaction(hash); tx == nil { // Transaction not found anywhere, abort return nil, nil @@ -1034,7 +1035,7 @@ func (s *PublicTransactionPoolAPI) GetRawTransactionByHash(ctx context.Context, // GetTransactionReceipt returns the transaction receipt for the given transaction hash. func (s *PublicTransactionPoolAPI) GetTransactionReceipt(ctx context.Context, hash common.Hash) (map[string]interface{}, error) { - tx, blockHash, blockNumber, index := core.GetTransaction(s.b.ChainDb(), hash) + tx, blockHash, blockNumber, index := rawdb.ReadTransaction(s.b.ChainDb(), hash) if tx == nil { return nil, nil } diff --git a/internal/jsre/jsre.go b/internal/jsre/jsre.go index f05865eca6..4c7664f1cc 100644 --- a/internal/jsre/jsre.go +++ b/internal/jsre/jsre.go @@ -102,8 +102,8 @@ func randomSource() *rand.Rand { // call the functions of the otto vm directly to circumvent the queue. These // functions should be used if and only if running a routine that was already // called from JS through an RPC call. -func (self *JSRE) runEventLoop() { - defer close(self.closed) +func (re *JSRE) runEventLoop() { + defer close(re.closed) vm := otto.New() r := randomSource() @@ -202,14 +202,14 @@ loop: break loop } } - case req := <-self.evalQueue: + case req := <-re.evalQueue: // run the code, send the result back req.fn(vm) close(req.done) if waitForCallbacks && (len(registry) == 0) { break loop } - case waitForCallbacks = <-self.stopEventLoop: + case waitForCallbacks = <-re.stopEventLoop: if !waitForCallbacks || (len(registry) == 0) { break loop } @@ -223,31 +223,31 @@ loop: } // Do executes the given function on the JS event loop. -func (self *JSRE) Do(fn func(*otto.Otto)) { +func (re *JSRE) Do(fn func(*otto.Otto)) { done := make(chan bool) req := &evalReq{fn, done} - self.evalQueue <- req + re.evalQueue <- req <-done } // stops the event loop before exit, optionally waits for all timers to expire -func (self *JSRE) Stop(waitForCallbacks bool) { +func (re *JSRE) Stop(waitForCallbacks bool) { select { - case <-self.closed: - case self.stopEventLoop <- waitForCallbacks: - <-self.closed + case <-re.closed: + case re.stopEventLoop <- waitForCallbacks: + <-re.closed } } // Exec(file) loads and runs the contents of a file // if a relative path is given, the jsre's assetPath is used -func (self *JSRE) Exec(file string) error { - code, err := ioutil.ReadFile(common.AbsolutePath(self.assetPath, file)) +func (re *JSRE) Exec(file string) error { + code, err := ioutil.ReadFile(common.AbsolutePath(re.assetPath, file)) if err != nil { return err } var script *otto.Script - self.Do(func(vm *otto.Otto) { + re.Do(func(vm *otto.Otto) { script, err = vm.Compile(file, code) if err != nil { return @@ -259,36 +259,36 @@ func (self *JSRE) Exec(file string) error { // Bind assigns value v to a variable in the JS environment // This method is deprecated, use Set. -func (self *JSRE) Bind(name string, v interface{}) error { - return self.Set(name, v) +func (re *JSRE) Bind(name string, v interface{}) error { + return re.Set(name, v) } // Run runs a piece of JS code. -func (self *JSRE) Run(code string) (v otto.Value, err error) { - self.Do(func(vm *otto.Otto) { v, err = vm.Run(code) }) +func (re *JSRE) Run(code string) (v otto.Value, err error) { + re.Do(func(vm *otto.Otto) { v, err = vm.Run(code) }) return v, err } // Get returns the value of a variable in the JS environment. -func (self *JSRE) Get(ns string) (v otto.Value, err error) { - self.Do(func(vm *otto.Otto) { v, err = vm.Get(ns) }) +func (re *JSRE) Get(ns string) (v otto.Value, err error) { + re.Do(func(vm *otto.Otto) { v, err = vm.Get(ns) }) return v, err } // Set assigns value v to a variable in the JS environment. -func (self *JSRE) Set(ns string, v interface{}) (err error) { - self.Do(func(vm *otto.Otto) { err = vm.Set(ns, v) }) +func (re *JSRE) Set(ns string, v interface{}) (err error) { + re.Do(func(vm *otto.Otto) { err = vm.Set(ns, v) }) return err } // loadScript executes a JS script from inside the currently executing JS code. -func (self *JSRE) loadScript(call otto.FunctionCall) otto.Value { +func (re *JSRE) loadScript(call otto.FunctionCall) otto.Value { file, err := call.Argument(0).ToString() if err != nil { // TODO: throw exception return otto.FalseValue() } - file = common.AbsolutePath(self.assetPath, file) + file = common.AbsolutePath(re.assetPath, file) source, err := ioutil.ReadFile(file) if err != nil { // TODO: throw exception @@ -305,10 +305,10 @@ func (self *JSRE) loadScript(call otto.FunctionCall) otto.Value { // Evaluate executes code and pretty prints the result to the specified output // stream. -func (self *JSRE) Evaluate(code string, w io.Writer) error { +func (re *JSRE) Evaluate(code string, w io.Writer) error { var fail error - self.Do(func(vm *otto.Otto) { + re.Do(func(vm *otto.Otto) { val, err := vm.Run(code) if err != nil { prettyError(vm, err, w) @@ -321,8 +321,8 @@ func (self *JSRE) Evaluate(code string, w io.Writer) error { } // Compile compiles and then runs a piece of JS code. -func (self *JSRE) Compile(filename string, src interface{}) (err error) { - self.Do(func(vm *otto.Otto) { _, err = compileAndRun(vm, filename, src) }) +func (re *JSRE) Compile(filename string, src interface{}) (err error) { + re.Do(func(vm *otto.Otto) { _, err = compileAndRun(vm, filename, src) }) return err } diff --git a/les/api_backend.go b/les/api_backend.go index c8bf925479..1852578987 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" @@ -83,16 +84,22 @@ func (b *LesApiBackend) GetBlock(ctx context.Context, blockHash common.Hash) (*t return b.eth.blockchain.GetBlockByHash(ctx, blockHash) } -func (b *LesApiBackend) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) { - return light.GetBlockReceipts(ctx, b.eth.odr, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash)) +func (b *LesApiBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { + if number := rawdb.ReadHeaderNumber(b.eth.chainDb, hash); number != nil { + return light.GetBlockReceipts(ctx, b.eth.odr, hash, *number) + } + return nil, nil } -func (b *LesApiBackend) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) { - return light.GetBlockLogs(ctx, b.eth.odr, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash)) +func (b *LesApiBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { + if number := rawdb.ReadHeaderNumber(b.eth.chainDb, hash); number != nil { + return light.GetBlockLogs(ctx, b.eth.odr, hash, *number) + } + return nil, nil } -func (b *LesApiBackend) GetTd(blockHash common.Hash) *big.Int { - return b.eth.blockchain.GetTdByHash(blockHash) +func (b *LesApiBackend) GetTd(hash common.Hash) *big.Int { + return b.eth.blockchain.GetTdByHash(hash) } func (b *LesApiBackend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) { diff --git a/les/backend.go b/les/backend.go index 6a324cb04b..2d8ada7b0e 100644 --- a/les/backend.go +++ b/les/backend.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/downloader" @@ -122,7 +123,7 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { if compat, ok := genesisErr.(*params.ConfigCompatError); ok { log.Warn("Rewinding chain to upgrade configuration", "err", compat) leth.blockchain.SetHead(compat.RewindTo) - core.WriteChainConfig(chainDb, genesisHash, chainConfig) + rawdb.WriteChainConfig(chainDb, genesisHash, chainConfig) } leth.txPool = light.NewTxPool(leth.chainConfig, leth.blockchain, leth.relay) diff --git a/les/distributor_test.go b/les/distributor_test.go index 55defb69be..2891bcab49 100644 --- a/les/distributor_test.go +++ b/les/distributor_test.go @@ -97,9 +97,8 @@ func (p *testDistPeer) waitBefore(cost uint64) (time.Duration, float64) { p.lock.RUnlock() if sumCost < testDistBufLimit { return 0, float64(testDistBufLimit-sumCost) / float64(testDistBufLimit) - } else { - return time.Duration(sumCost - testDistBufLimit), 0 } + return time.Duration(sumCost - testDistBufLimit), 0 } func (p *testDistPeer) canQueue() bool { diff --git a/les/fetcher.go b/les/fetcher.go index e12a2c78a2..59d3a2aa3c 100644 --- a/les/fetcher.go +++ b/les/fetcher.go @@ -25,7 +25,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/consensus" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" @@ -280,7 +280,7 @@ func (f *lightFetcher) announce(p *peer, head *announceData) { // if one of root's children is canonical, keep it, delete other branches and root itself var newRoot *fetcherTreeNode for i, nn := range fp.root.children { - if core.GetCanonicalHash(f.pm.chainDb, nn.number) == nn.hash { + if rawdb.ReadCanonicalHash(f.pm.chainDb, nn.number) == nn.hash { fp.root.children = append(fp.root.children[:i], fp.root.children[i+1:]...) nn.parent = nil newRoot = nn @@ -363,7 +363,7 @@ func (f *lightFetcher) peerHasBlock(p *peer, hash common.Hash, number uint64) bo // // when syncing, just check if it is part of the known chain, there is nothing better we // can do since we do not know the most recent block hash yet - return core.GetCanonicalHash(f.pm.chainDb, fp.root.number) == fp.root.hash && core.GetCanonicalHash(f.pm.chainDb, number) == hash + return rawdb.ReadCanonicalHash(f.pm.chainDb, fp.root.number) == fp.root.hash && rawdb.ReadCanonicalHash(f.pm.chainDb, number) == hash } // requestAmount calculates the amount of headers to be downloaded starting diff --git a/les/handler.go b/les/handler.go index 9627f392be..22899eb1bc 100644 --- a/les/handler.go +++ b/les/handler.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/downloader" @@ -528,9 +529,11 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { break } // Retrieve the requested block body, stopping if enough was found - if data := core.GetBodyRLP(pm.chainDb, hash, core.GetBlockNumber(pm.chainDb, hash)); len(data) != 0 { - bodies = append(bodies, data) - bytes += len(data) + if number := rawdb.ReadHeaderNumber(pm.chainDb, hash); number != nil { + if data := rawdb.ReadBodyRLP(pm.chainDb, hash, *number); len(data) != 0 { + bodies = append(bodies, data) + bytes += len(data) + } } } bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost) @@ -579,20 +582,22 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } for _, req := range req.Reqs { // Retrieve the requested state entry, stopping if enough was found - if header := core.GetHeader(pm.chainDb, req.BHash, core.GetBlockNumber(pm.chainDb, req.BHash)); header != nil { - statedb, err := pm.blockchain.State() - if err != nil { - continue - } - account, err := pm.getAccount(statedb, header.Root, common.BytesToHash(req.AccKey)) - if err != nil { - continue - } - code, _ := statedb.Database().TrieDB().Node(common.BytesToHash(account.CodeHash)) + if number := rawdb.ReadHeaderNumber(pm.chainDb, req.BHash); number != nil { + if header := rawdb.ReadHeader(pm.chainDb, req.BHash, *number); header != nil { + statedb, err := pm.blockchain.State() + if err != nil { + continue + } + account, err := pm.getAccount(statedb, header.Root, common.BytesToHash(req.AccKey)) + if err != nil { + continue + } + code, _ := statedb.Database().TrieDB().Node(common.BytesToHash(account.CodeHash)) - data = append(data, code) - if bytes += len(code); bytes >= softResponseLimit { - break + data = append(data, code) + if bytes += len(code); bytes >= softResponseLimit { + break + } } } } @@ -645,7 +650,10 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { break } // Retrieve the requested block's receipts, skipping if unknown to us - results := core.GetBlockReceipts(pm.chainDb, hash, core.GetBlockNumber(pm.chainDb, hash)) + var results types.Receipts + if number := rawdb.ReadHeaderNumber(pm.chainDb, hash); number != nil { + results = rawdb.ReadReceipts(pm.chainDb, hash, *number) + } if results == nil { if header := pm.blockchain.GetHeaderByHash(hash); header == nil || header.ReceiptHash != types.EmptyRootHash { continue @@ -705,28 +713,30 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } for _, req := range req.Reqs { // Retrieve the requested state entry, stopping if enough was found - if header := core.GetHeader(pm.chainDb, req.BHash, core.GetBlockNumber(pm.chainDb, req.BHash)); header != nil { - statedb, err := pm.blockchain.State() - if err != nil { - continue - } - var trie state.Trie - if len(req.AccKey) > 0 { - account, err := pm.getAccount(statedb, header.Root, common.BytesToHash(req.AccKey)) + if number := rawdb.ReadHeaderNumber(pm.chainDb, req.BHash); number != nil { + if header := rawdb.ReadHeader(pm.chainDb, req.BHash, *number); header != nil { + statedb, err := pm.blockchain.State() if err != nil { continue } - trie, _ = statedb.Database().OpenStorageTrie(common.BytesToHash(req.AccKey), account.Root) - } else { - trie, _ = statedb.Database().OpenTrie(header.Root) - } - if trie != nil { - var proof light.NodeList - trie.Prove(req.Key, 0, &proof) + var trie state.Trie + if len(req.AccKey) > 0 { + account, err := pm.getAccount(statedb, header.Root, common.BytesToHash(req.AccKey)) + if err != nil { + continue + } + trie, _ = statedb.Database().OpenStorageTrie(common.BytesToHash(req.AccKey), account.Root) + } else { + trie, _ = statedb.Database().OpenTrie(header.Root) + } + if trie != nil { + var proof light.NodeList + trie.Prove(req.Key, 0, &proof) - proofs = append(proofs, proof) - if bytes += proof.DataSize(); bytes >= softResponseLimit { - break + proofs = append(proofs, proof) + if bytes += proof.DataSize(); bytes >= softResponseLimit { + break + } } } } @@ -763,9 +773,11 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { if statedb == nil || req.BHash != lastBHash { statedb, root, lastBHash = nil, common.Hash{}, req.BHash - if header := core.GetHeader(pm.chainDb, req.BHash, core.GetBlockNumber(pm.chainDb, req.BHash)); header != nil { - statedb, _ = pm.blockchain.State() - root = header.Root + if number := rawdb.ReadHeaderNumber(pm.chainDb, req.BHash); number != nil { + if header := rawdb.ReadHeader(pm.chainDb, req.BHash, *number); header != nil { + statedb, _ = pm.blockchain.State() + root = header.Root + } } } if statedb == nil { @@ -859,7 +871,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { trieDb := trie.NewDatabase(ethdb.NewTable(pm.chainDb, light.ChtTablePrefix)) for _, req := range req.Reqs { if header := pm.blockchain.GetHeaderByNumber(req.BlockNum); header != nil { - sectionHead := core.GetCanonicalHash(pm.chainDb, req.ChtNum*light.CHTFrequencyServer-1) + sectionHead := rawdb.ReadCanonicalHash(pm.chainDb, req.ChtNum*light.CHTFrequencyServer-1) if root := light.GetChtRoot(pm.chainDb, req.ChtNum-1, sectionHead); root != (common.Hash{}) { trie, err := trie.New(root, trieDb) if err != nil { @@ -1114,10 +1126,10 @@ func (pm *ProtocolManager) getAccount(statedb *state.StateDB, root, hash common. func (pm *ProtocolManager) getHelperTrie(id uint, idx uint64) (common.Hash, string) { switch id { case htCanonical: - sectionHead := core.GetCanonicalHash(pm.chainDb, (idx+1)*light.CHTFrequencyClient-1) + sectionHead := rawdb.ReadCanonicalHash(pm.chainDb, (idx+1)*light.CHTFrequencyClient-1) return light.GetChtV2Root(pm.chainDb, idx, sectionHead), light.ChtTablePrefix case htBloomBits: - sectionHead := core.GetCanonicalHash(pm.chainDb, (idx+1)*light.BloomTrieFrequency-1) + sectionHead := rawdb.ReadCanonicalHash(pm.chainDb, (idx+1)*light.BloomTrieFrequency-1) return light.GetBloomTrieRoot(pm.chainDb, idx, sectionHead), light.BloomTrieTablePrefix } return common.Hash{}, "" @@ -1128,8 +1140,8 @@ func (pm *ProtocolManager) getHelperTrieAuxData(req HelperTrieReq) []byte { switch { case req.Type == htCanonical && req.AuxReq == auxHeader && len(req.Key) == 8: blockNum := binary.BigEndian.Uint64(req.Key) - hash := core.GetCanonicalHash(pm.chainDb, blockNum) - return core.GetHeaderRLP(pm.chainDb, hash, blockNum) + hash := rawdb.ReadCanonicalHash(pm.chainDb, blockNum) + return rawdb.ReadHeaderRLP(pm.chainDb, hash, blockNum) } return nil } @@ -1142,9 +1154,9 @@ func (pm *ProtocolManager) txStatus(hashes []common.Hash) []txStatus { // If the transaction is unknown to the pool, try looking it up locally if stat == core.TxStatusUnknown { - if block, number, index := core.GetTxLookupEntry(pm.chainDb, hashes[i]); block != (common.Hash{}) { + if block, number, index := rawdb.ReadTxLookupEntry(pm.chainDb, hashes[i]); block != (common.Hash{}) { stats[i].Status = core.TxStatusIncluded - stats[i].Lookup = &core.TxLookupEntry{BlockHash: block, BlockIndex: number, Index: index} + stats[i].Lookup = &rawdb.TxLookupEntry{BlockHash: block, BlockIndex: number, Index: index} } } } diff --git a/les/handler_test.go b/les/handler_test.go index 9468032f67..31aad3ed45 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/downloader" @@ -50,8 +51,7 @@ func TestGetBlockHeadersLes1(t *testing.T) { testGetBlockHeaders(t, 1) } func TestGetBlockHeadersLes2(t *testing.T) { testGetBlockHeaders(t, 2) } func testGetBlockHeaders(t *testing.T, protocol int) { - db, _ := ethdb.NewMemDatabase() - pm := newTestProtocolManagerMust(t, false, downloader.MaxHashFetch+15, nil, nil, nil, db) + pm := newTestProtocolManagerMust(t, false, downloader.MaxHashFetch+15, nil, nil, nil, ethdb.NewMemDatabase()) bc := pm.blockchain.(*core.BlockChain) peer, _ := newTestPeer(t, "peer", protocol, pm, true) defer peer.close() @@ -180,8 +180,7 @@ func TestGetBlockBodiesLes1(t *testing.T) { testGetBlockBodies(t, 1) } func TestGetBlockBodiesLes2(t *testing.T) { testGetBlockBodies(t, 2) } func testGetBlockBodies(t *testing.T, protocol int) { - db, _ := ethdb.NewMemDatabase() - pm := newTestProtocolManagerMust(t, false, downloader.MaxBlockFetch+15, nil, nil, nil, db) + pm := newTestProtocolManagerMust(t, false, downloader.MaxBlockFetch+15, nil, nil, nil, ethdb.NewMemDatabase()) bc := pm.blockchain.(*core.BlockChain) peer, _ := newTestPeer(t, "peer", protocol, pm, true) defer peer.close() @@ -258,8 +257,7 @@ func TestGetCodeLes2(t *testing.T) { testGetCode(t, 2) } func testGetCode(t *testing.T, protocol int) { // Assemble the test environment - db, _ := ethdb.NewMemDatabase() - pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, db) + pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, ethdb.NewMemDatabase()) bc := pm.blockchain.(*core.BlockChain) peer, _ := newTestPeer(t, "peer", protocol, pm, true) defer peer.close() @@ -292,7 +290,7 @@ func TestGetReceiptLes2(t *testing.T) { testGetReceipt(t, 2) } func testGetReceipt(t *testing.T, protocol int) { // Assemble the test environment - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, db) bc := pm.blockchain.(*core.BlockChain) peer, _ := newTestPeer(t, "peer", protocol, pm, true) @@ -304,7 +302,7 @@ func testGetReceipt(t *testing.T, protocol int) { block := bc.GetBlockByNumber(i) hashes = append(hashes, block.Hash()) - receipts = append(receipts, core.GetBlockReceipts(db, block.Hash(), block.NumberU64())) + receipts = append(receipts, rawdb.ReadReceipts(db, block.Hash(), block.NumberU64())) } // Send the hash request and verify the response cost := peer.GetRequestCost(GetReceiptsMsg, len(hashes)) @@ -320,7 +318,7 @@ func TestGetProofsLes2(t *testing.T) { testGetProofs(t, 2) } func testGetProofs(t *testing.T, protocol int) { // Assemble the test environment - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, db) bc := pm.blockchain.(*core.BlockChain) peer, _ := newTestPeer(t, "peer", protocol, pm, true) @@ -383,7 +381,7 @@ func testGetCHTProofs(t *testing.T, protocol int) { frequency = uint64(light.CHTFrequencyServer) } // Assemble the test environment - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() pm := newTestProtocolManagerMust(t, false, int(frequency)+light.HelperTrieProcessConfirmations, testChainGen, nil, nil, db) bc := pm.blockchain.(*core.BlockChain) peer, _ := newTestPeer(t, "peer", protocol, pm, true) @@ -451,7 +449,7 @@ func testGetCHTProofs(t *testing.T, protocol int) { // Tests that bloombits proofs can be correctly retrieved. func TestGetBloombitsProofs(t *testing.T) { // Assemble the test environment - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() pm := newTestProtocolManagerMust(t, false, light.BloomTrieFrequency+256, testChainGen, nil, nil, db) bc := pm.blockchain.(*core.BlockChain) peer, _ := newTestPeer(t, "peer", 2, pm, true) @@ -490,7 +488,7 @@ func TestGetBloombitsProofs(t *testing.T) { } func TestTransactionStatusLes2(t *testing.T) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() pm := newTestProtocolManagerMust(t, false, 0, nil, nil, nil, db) chain := pm.blockchain.(*core.BlockChain) config := core.DefaultTxPoolConfig @@ -555,9 +553,9 @@ func TestTransactionStatusLes2(t *testing.T) { } // check if their status is included now - block1hash := core.GetCanonicalHash(db, 1) - test(tx1, false, txStatus{Status: core.TxStatusIncluded, Lookup: &core.TxLookupEntry{BlockHash: block1hash, BlockIndex: 1, Index: 0}}) - test(tx2, false, txStatus{Status: core.TxStatusIncluded, Lookup: &core.TxLookupEntry{BlockHash: block1hash, BlockIndex: 1, Index: 1}}) + block1hash := rawdb.ReadCanonicalHash(db, 1) + test(tx1, false, txStatus{Status: core.TxStatusIncluded, Lookup: &rawdb.TxLookupEntry{BlockHash: block1hash, BlockIndex: 1, Index: 0}}) + test(tx2, false, txStatus{Status: core.TxStatusIncluded, Lookup: &rawdb.TxLookupEntry{BlockHash: block1hash, BlockIndex: 1, Index: 1}}) // create a reorg that rolls them back gchain, _ = core.GenerateChain(params.TestChainConfig, chain.GetBlockByNumber(0), ethash.NewFaker(), db, 2, func(i int, block *core.BlockGen) {}) diff --git a/les/odr_requests.go b/les/odr_requests.go index 34d759dd2a..c7f06df94d 100644 --- a/les/odr_requests.go +++ b/les/odr_requests.go @@ -24,7 +24,7 @@ import ( "fmt" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" @@ -110,7 +110,7 @@ func (r *BlockRequest) Validate(db ethdb.Database, msg *Msg) error { body := bodies[0] // Retrieve our stored header and validate block content against it - header := core.GetHeader(db, r.Hash, r.Number) + header := rawdb.ReadHeader(db, r.Hash, r.Number) if header == nil { return errHeaderUnavailable } @@ -166,7 +166,7 @@ func (r *ReceiptsRequest) Validate(db ethdb.Database, msg *Msg) error { receipt := receipts[0] // Retrieve our stored header and validate receipt content against it - header := core.GetHeader(db, r.Hash, r.Number) + header := rawdb.ReadHeader(db, r.Hash, r.Number) if header == nil { return errHeaderUnavailable } diff --git a/les/odr_test.go b/les/odr_test.go index 88e121cda6..983f7262b0 100644 --- a/les/odr_test.go +++ b/les/odr_test.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" @@ -63,9 +64,13 @@ func TestOdrGetReceiptsLes2(t *testing.T) { testOdr(t, 2, 1, odrGetReceipts) } func odrGetReceipts(ctx context.Context, db ethdb.Database, config *params.ChainConfig, bc *core.BlockChain, lc *light.LightChain, bhash common.Hash) []byte { var receipts types.Receipts if bc != nil { - receipts = core.GetBlockReceipts(db, bhash, core.GetBlockNumber(db, bhash)) + if number := rawdb.ReadHeaderNumber(db, bhash); number != nil { + receipts = rawdb.ReadReceipts(db, bhash, *number) + } } else { - receipts, _ = light.GetBlockReceipts(ctx, lc.Odr(), bhash, core.GetBlockNumber(db, bhash)) + if number := rawdb.ReadHeaderNumber(db, bhash); number != nil { + receipts, _ = light.GetBlockReceipts(ctx, lc.Odr(), bhash, *number) + } } if receipts == nil { return nil @@ -160,8 +165,8 @@ func testOdr(t *testing.T, protocol int, expFail uint64, fn odrTestFn) { peers := newPeerSet() dist := newRequestDistributor(peers, make(chan struct{})) rm := newRetrieveManager(peers, dist, nil) - db, _ := ethdb.NewMemDatabase() - ldb, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() + ldb := ethdb.NewMemDatabase() odr := NewLesOdr(ldb, light.NewChtIndexer(db, true), light.NewBloomTrieIndexer(db, true), eth.NewBloomIndexer(db, light.BloomTrieFrequency), rm) pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, db) lpm := newTestProtocolManagerMust(t, true, 0, nil, peers, odr, ldb) @@ -178,7 +183,7 @@ func testOdr(t *testing.T, protocol int, expFail uint64, fn odrTestFn) { test := func(expFail uint64) { for i := uint64(0); i <= pm.blockchain.CurrentHeader().Number.Uint64(); i++ { - bhash := core.GetCanonicalHash(db, i) + bhash := rawdb.ReadCanonicalHash(db, i) b1 := fn(light.NoOdr, db, pm.chainConfig, pm.blockchain.(*core.BlockChain), nil, bhash) ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) diff --git a/les/protocol.go b/les/protocol.go index e1c4625bce..ee4c223988 100644 --- a/les/protocol.go +++ b/les/protocol.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto/secp256k1" "github.com/ethereum/go-ethereum/rlp" @@ -160,9 +161,8 @@ func (a *announceData) checkSignature(pubKey *ecdsa.PublicKey) error { pbytes := elliptic.Marshal(pubKey.Curve, pubKey.X, pubKey.Y) if bytes.Equal(pbytes, recPubkey) { return nil - } else { - return errors.New("Wrong signature") } + return errors.New("Wrong signature") } type blockInfo struct { @@ -224,6 +224,6 @@ type proofsData [][]rlp.RawValue type txStatus struct { Status core.TxStatus - Lookup *core.TxLookupEntry `rlp:"nil"` + Lookup *rawdb.TxLookupEntry `rlp:"nil"` Error string } diff --git a/les/randselect.go b/les/randselect.go index 1a9d0695bd..1cc1d3d3e0 100644 --- a/les/randselect.go +++ b/les/randselect.go @@ -118,17 +118,16 @@ func (n *wrsNode) insert(item wrsItem, weight int64) int { if n.level == 0 { n.items[branch] = item return branch - } else { - var subNode *wrsNode - if n.items[branch] == nil { - subNode = &wrsNode{maxItems: n.maxItems / wrsBranches, level: n.level - 1} - n.items[branch] = subNode - } else { - subNode = n.items[branch].(*wrsNode) - } - subIdx := subNode.insert(item, weight) - return subNode.maxItems*branch + subIdx } + var subNode *wrsNode + if n.items[branch] == nil { + subNode = &wrsNode{maxItems: n.maxItems / wrsBranches, level: n.level - 1} + n.items[branch] = subNode + } else { + subNode = n.items[branch].(*wrsNode) + } + subIdx := subNode.insert(item, weight) + return subNode.maxItems*branch + subIdx } // setWeight updates the weight of a certain item (which should exist) and returns @@ -162,12 +161,10 @@ func (n *wrsNode) choose(val int64) (wrsItem, int64) { if val < w { if n.level == 0 { return n.items[i].(wrsItem), n.weights[i] - } else { - return n.items[i].(*wrsNode).choose(val) } - } else { - val -= w + return n.items[i].(*wrsNode).choose(val) } + val -= w } panic(nil) } diff --git a/les/request_test.go b/les/request_test.go index c13625de8e..ba2f603d8b 100644 --- a/les/request_test.go +++ b/les/request_test.go @@ -22,7 +22,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/ethdb" @@ -58,15 +58,22 @@ func TestTrieEntryAccessLes1(t *testing.T) { testAccess(t, 1, tfTrieEntryAccess) func TestTrieEntryAccessLes2(t *testing.T) { testAccess(t, 2, tfTrieEntryAccess) } func tfTrieEntryAccess(db ethdb.Database, bhash common.Hash, number uint64) light.OdrRequest { - return &light.TrieRequest{Id: light.StateTrieID(core.GetHeader(db, bhash, core.GetBlockNumber(db, bhash))), Key: testBankSecureTrieKey} + if number := rawdb.ReadHeaderNumber(db, bhash); number != nil { + return &light.TrieRequest{Id: light.StateTrieID(rawdb.ReadHeader(db, bhash, *number)), Key: testBankSecureTrieKey} + } + return nil } func TestCodeAccessLes1(t *testing.T) { testAccess(t, 1, tfCodeAccess) } func TestCodeAccessLes2(t *testing.T) { testAccess(t, 2, tfCodeAccess) } -func tfCodeAccess(db ethdb.Database, bhash common.Hash, number uint64) light.OdrRequest { - header := core.GetHeader(db, bhash, core.GetBlockNumber(db, bhash)) +func tfCodeAccess(db ethdb.Database, bhash common.Hash, num uint64) light.OdrRequest { + number := rawdb.ReadHeaderNumber(db, bhash) + if number != nil { + return nil + } + header := rawdb.ReadHeader(db, bhash, *number) if header.Number.Uint64() < testContractDeployed { return nil } @@ -80,8 +87,8 @@ func testAccess(t *testing.T, protocol int, fn accessTestFn) { peers := newPeerSet() dist := newRequestDistributor(peers, make(chan struct{})) rm := newRetrieveManager(peers, dist, nil) - db, _ := ethdb.NewMemDatabase() - ldb, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() + ldb := ethdb.NewMemDatabase() odr := NewLesOdr(ldb, light.NewChtIndexer(db, true), light.NewBloomTrieIndexer(db, true), eth.NewBloomIndexer(db, light.BloomTrieFrequency), rm) pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, db) @@ -99,7 +106,7 @@ func testAccess(t *testing.T, protocol int, fn accessTestFn) { test := func(expFail uint64) { for i := uint64(0); i <= pm.blockchain.CurrentHeader().Number.Uint64(); i++ { - bhash := core.GetCanonicalHash(db, i) + bhash := rawdb.ReadCanonicalHash(db, i) if req := fn(ldb, bhash, i); req != nil { ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) defer cancel() diff --git a/les/server.go b/les/server.go index 28b87008a0..9eb8ee7f97 100644 --- a/les/server.go +++ b/les/server.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/ethdb" @@ -329,11 +330,11 @@ func (pm *ProtocolManager) blockLoop() { header := ev.Block.Header() hash := header.Hash() number := header.Number.Uint64() - td := core.GetTd(pm.chainDb, hash, number) + td := rawdb.ReadTd(pm.chainDb, hash, number) if td != nil && td.Cmp(lastBroadcastTd) > 0 { var reorg uint64 if lastHead != nil { - reorg = lastHead.Number.Uint64() - core.FindCommonAncestor(pm.chainDb, header, lastHead).Number.Uint64() + reorg = lastHead.Number.Uint64() - rawdb.FindCommonAncestor(pm.chainDb, header, lastHead).Number.Uint64() } lastHead = header lastBroadcastTd = td diff --git a/les/serverpool.go b/les/serverpool.go index da73b4b3c5..a39f883554 100644 --- a/les/serverpool.go +++ b/les/serverpool.go @@ -601,9 +601,8 @@ func (e *discoveredEntry) Weight() int64 { t := time.Duration(mclock.Now() - e.lastDiscovered) if t <= discoverExpireStart { return 1000000000 - } else { - return int64(1000000000 * math.Exp(-float64(t-discoverExpireStart)/float64(discoverExpireConst))) } + return int64(1000000000 * math.Exp(-float64(t-discoverExpireStart)/float64(discoverExpireConst))) } // knownEntry implements wrsItem diff --git a/les/sync.go b/les/sync.go index c3d37e2f3f..1ac6455852 100644 --- a/les/sync.go +++ b/les/sync.go @@ -20,7 +20,7 @@ import ( "context" "time" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/light" ) @@ -56,7 +56,7 @@ func (pm *ProtocolManager) syncer() { func (pm *ProtocolManager) needToSync(peerHead blockInfo) bool { head := pm.blockchain.CurrentHeader() - currentTd := core.GetTd(pm.chainDb, head.Hash(), head.Number.Uint64()) + currentTd := rawdb.ReadTd(pm.chainDb, head.Hash(), head.Number.Uint64()) return currentTd != nil && peerHead.Td.Cmp(currentTd) > 0 } diff --git a/light/lightchain.go b/light/lightchain.go index 4596c2f924..c79b5f39b6 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" @@ -143,7 +144,7 @@ func (self *LightChain) Odr() OdrBackend { // loadLastState loads the last known chain state from the database. This method // assumes that the chain manager mutex is held. func (self *LightChain) loadLastState() error { - if head := core.GetHeadHeaderHash(self.chainDb); head == (common.Hash{}) { + if head := rawdb.ReadHeadHeaderHash(self.chainDb); head == (common.Hash{}) { // Corrupt or empty database, init from scratch self.Reset() } else { @@ -190,12 +191,9 @@ func (bc *LightChain) ResetWithGenesisBlock(genesis *types.Block) { defer bc.mu.Unlock() // Prepare the genesis block and reinitialise the chain - if err := core.WriteTd(bc.chainDb, genesis.Hash(), genesis.NumberU64(), genesis.Difficulty()); err != nil { - log.Crit("Failed to write genesis block TD", "err", err) - } - if err := core.WriteBlock(bc.chainDb, genesis); err != nil { - log.Crit("Failed to write genesis block", "err", err) - } + rawdb.WriteTd(bc.chainDb, genesis.Hash(), genesis.NumberU64(), genesis.Difficulty()) + rawdb.WriteBlock(bc.chainDb, genesis) + bc.genesisBlock = genesis bc.hc.SetGenesis(bc.genesisBlock.Header()) bc.hc.SetCurrentHeader(bc.genesisBlock.Header()) @@ -224,7 +222,11 @@ func (self *LightChain) GetBody(ctx context.Context, hash common.Hash) (*types.B body := cached.(*types.Body) return body, nil } - body, err := GetBody(ctx, self.odr, hash, self.hc.GetBlockNumber(hash)) + number := self.hc.GetBlockNumber(hash) + if number == nil { + return nil, errors.New("unknown block") + } + body, err := GetBody(ctx, self.odr, hash, *number) if err != nil { return nil, err } @@ -240,7 +242,11 @@ func (self *LightChain) GetBodyRLP(ctx context.Context, hash common.Hash) (rlp.R if cached, ok := self.bodyRLPCache.Get(hash); ok { return cached.(rlp.RawValue), nil } - body, err := GetBodyRLP(ctx, self.odr, hash, self.hc.GetBlockNumber(hash)) + number := self.hc.GetBlockNumber(hash) + if number == nil { + return nil, errors.New("unknown block") + } + body, err := GetBodyRLP(ctx, self.odr, hash, *number) if err != nil { return nil, err } @@ -275,7 +281,11 @@ func (self *LightChain) GetBlock(ctx context.Context, hash common.Hash, number u // GetBlockByHash retrieves a block from the database or ODR service by hash, // caching it if found. func (self *LightChain) GetBlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) { - return self.GetBlock(ctx, hash, self.hc.GetBlockNumber(hash)) + number := self.hc.GetBlockNumber(hash) + if number == nil { + return nil, errors.New("unknown block") + } + return self.GetBlock(ctx, hash, *number) } // GetBlockByNumber retrieves a block from the database or ODR service by diff --git a/light/lightchain_test.go b/light/lightchain_test.go index 0af7551d41..c0aa51da29 100644 --- a/light/lightchain_test.go +++ b/light/lightchain_test.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" @@ -51,7 +52,7 @@ func makeHeaderChain(parent *types.Header, n int, db ethdb.Database, seed int) [ // chain. Depending on the full flag, if creates either a full block chain or a // header only chain. func newCanonical(n int) (ethdb.Database, *LightChain, error) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() gspec := core.Genesis{Config: params.TestChainConfig} genesis := gspec.MustCommit(db) blockchain, _ := NewLightChain(&dummyOdr{db: db}, gspec.Config, ethash.NewFaker()) @@ -68,7 +69,7 @@ func newCanonical(n int) (ethdb.Database, *LightChain, error) { // newTestLightChain creates a LightChain that doesn't validate anything. func newTestLightChain() *LightChain { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() gspec := &core.Genesis{ Difficulty: big.NewInt(1), Config: params.TestChainConfig, @@ -122,8 +123,8 @@ func testHeaderChainImport(chain []*types.Header, lightchain *LightChain) error } // Manually insert the header into the database, but don't reorganize (allows subsequent testing) lightchain.mu.Lock() - core.WriteTd(lightchain.chainDb, header.Hash(), header.Number.Uint64(), new(big.Int).Add(header.Difficulty, lightchain.GetTdByHash(header.ParentHash))) - core.WriteHeader(lightchain.chainDb, header) + rawdb.WriteTd(lightchain.chainDb, header.Hash(), header.Number.Uint64(), new(big.Int).Add(header.Difficulty, lightchain.GetTdByHash(header.ParentHash))) + rawdb.WriteHeader(lightchain.chainDb, header) lightchain.mu.Unlock() } return nil diff --git a/light/odr.go b/light/odr.go index e2c3d9c5a4..8f1e50b817 100644 --- a/light/odr.go +++ b/light/odr.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" ) @@ -112,7 +113,7 @@ type BlockRequest struct { // StoreResult stores the retrieved data in local database func (req *BlockRequest) StoreResult(db ethdb.Database) { - core.WriteBodyRLP(db, req.Hash, req.Number, req.Rlp) + rawdb.WriteBodyRLP(db, req.Hash, req.Number, req.Rlp) } // ReceiptsRequest is the ODR request type for retrieving block bodies @@ -125,7 +126,7 @@ type ReceiptsRequest struct { // StoreResult stores the retrieved data in local database func (req *ReceiptsRequest) StoreResult(db ethdb.Database) { - core.WriteBlockReceipts(db, req.Hash, req.Number, req.Receipts) + rawdb.WriteReceipts(db, req.Hash, req.Number, req.Receipts) } // ChtRequest is the ODR request type for state/storage trie entries @@ -140,11 +141,11 @@ type ChtRequest struct { // StoreResult stores the retrieved data in local database func (req *ChtRequest) StoreResult(db ethdb.Database) { - // if there is a canonical hash, there is a header too - core.WriteHeader(db, req.Header) hash, num := req.Header.Hash(), req.Header.Number.Uint64() - core.WriteTd(db, hash, num, req.Td) - core.WriteCanonicalHash(db, hash, num) + + rawdb.WriteHeader(db, req.Header) + rawdb.WriteTd(db, hash, num, req.Td) + rawdb.WriteCanonicalHash(db, hash, num) } // BloomRequest is the ODR request type for retrieving bloom filters from a CHT structure @@ -161,11 +162,11 @@ type BloomRequest struct { // StoreResult stores the retrieved data in local database func (req *BloomRequest) StoreResult(db ethdb.Database) { for i, sectionIdx := range req.SectionIdxList { - sectionHead := core.GetCanonicalHash(db, (sectionIdx+1)*BloomTrieFrequency-1) + sectionHead := rawdb.ReadCanonicalHash(db, (sectionIdx+1)*BloomTrieFrequency-1) // if we don't have the canonical hash stored for this section head number, we'll still store it under // a key with a zero sectionHead. GetBloomBits will look there too if we still don't have the canonical // hash. In the unlikely case we've retrieved the section head hash since then, we'll just retrieve the // bit vector again from the network. - core.WriteBloomBits(db, req.BitIdx, sectionIdx, sectionHead, req.BloomBits[i]) + rawdb.WriteBloomBits(db, req.BitIdx, sectionIdx, sectionHead, req.BloomBits[i]) } } diff --git a/light/odr_test.go b/light/odr_test.go index d3f9374fd8..3e7ac10118 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" @@ -70,9 +71,15 @@ func (odr *testOdr) Retrieve(ctx context.Context, req OdrRequest) error { } switch req := req.(type) { case *BlockRequest: - req.Rlp = core.GetBodyRLP(odr.sdb, req.Hash, core.GetBlockNumber(odr.sdb, req.Hash)) + number := rawdb.ReadHeaderNumber(odr.sdb, req.Hash) + if number != nil { + req.Rlp = rawdb.ReadBodyRLP(odr.sdb, req.Hash, *number) + } case *ReceiptsRequest: - req.Receipts = core.GetBlockReceipts(odr.sdb, req.Hash, core.GetBlockNumber(odr.sdb, req.Hash)) + number := rawdb.ReadHeaderNumber(odr.sdb, req.Hash) + if number != nil { + req.Receipts = rawdb.ReadReceipts(odr.sdb, req.Hash, *number) + } case *TrieRequest: t, _ := trie.New(req.Id.Root, trie.NewDatabase(odr.sdb)) nodes := NewNodeSet() @@ -108,9 +115,15 @@ func TestOdrGetReceiptsLes1(t *testing.T) { testChainOdr(t, 1, odrGetReceipts) } func odrGetReceipts(ctx context.Context, db ethdb.Database, bc *core.BlockChain, lc *LightChain, bhash common.Hash) ([]byte, error) { var receipts types.Receipts if bc != nil { - receipts = core.GetBlockReceipts(db, bhash, core.GetBlockNumber(db, bhash)) + number := rawdb.ReadHeaderNumber(db, bhash) + if number != nil { + receipts = rawdb.ReadReceipts(db, bhash, *number) + } } else { - receipts, _ = GetBlockReceipts(ctx, lc.Odr(), bhash, core.GetBlockNumber(db, bhash)) + number := rawdb.ReadHeaderNumber(db, bhash) + if number != nil { + receipts, _ = GetBlockReceipts(ctx, lc.Odr(), bhash, *number) + } } if receipts == nil { return nil, nil @@ -232,8 +245,8 @@ func testChainGen(i int, block *core.BlockGen) { func testChainOdr(t *testing.T, protocol int, fn odrTestFn) { var ( - sdb, _ = ethdb.NewMemDatabase() - ldb, _ = ethdb.NewMemDatabase() + sdb = ethdb.NewMemDatabase() + ldb = ethdb.NewMemDatabase() gspec = core.Genesis{Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}} genesis = gspec.MustCommit(sdb) ) @@ -260,7 +273,7 @@ func testChainOdr(t *testing.T, protocol int, fn odrTestFn) { test := func(expFail int) { for i := uint64(0); i <= blockchain.CurrentHeader().Number.Uint64(); i++ { - bhash := core.GetCanonicalHash(sdb, i) + bhash := rawdb.ReadCanonicalHash(sdb, i) b1, err := fn(NoOdr, sdb, blockchain, nil, bhash) if err != nil { t.Fatalf("error in full-node test for block %d: %v", i, err) diff --git a/light/odr_util.go b/light/odr_util.go index d56330e36b..620af63835 100644 --- a/light/odr_util.go +++ b/light/odr_util.go @@ -22,6 +22,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" @@ -31,10 +32,10 @@ var sha3_nil = crypto.Keccak256Hash(nil) func GetHeaderByNumber(ctx context.Context, odr OdrBackend, number uint64) (*types.Header, error) { db := odr.Database() - hash := core.GetCanonicalHash(db, number) + hash := rawdb.ReadCanonicalHash(db, number) if (hash != common.Hash{}) { // if there is a canonical hash, there is a header too - header := core.GetHeader(db, hash, number) + header := rawdb.ReadHeader(db, hash, number) if header == nil { panic("Canonical hash present but header not found") } @@ -47,14 +48,14 @@ func GetHeaderByNumber(ctx context.Context, odr OdrBackend, number uint64) (*typ ) if odr.ChtIndexer() != nil { chtCount, sectionHeadNum, sectionHead = odr.ChtIndexer().Sections() - canonicalHash := core.GetCanonicalHash(db, sectionHeadNum) + canonicalHash := rawdb.ReadCanonicalHash(db, sectionHeadNum) // if the CHT was injected as a trusted checkpoint, we have no canonical hash yet so we accept zero hash too for chtCount > 0 && canonicalHash != sectionHead && canonicalHash != (common.Hash{}) { chtCount-- if chtCount > 0 { sectionHeadNum = chtCount*CHTFrequencyClient - 1 sectionHead = odr.ChtIndexer().SectionHead(chtCount - 1) - canonicalHash = core.GetCanonicalHash(db, sectionHeadNum) + canonicalHash = rawdb.ReadCanonicalHash(db, sectionHeadNum) } } } @@ -69,7 +70,7 @@ func GetHeaderByNumber(ctx context.Context, odr OdrBackend, number uint64) (*typ } func GetCanonicalHash(ctx context.Context, odr OdrBackend, number uint64) (common.Hash, error) { - hash := core.GetCanonicalHash(odr.Database(), number) + hash := rawdb.ReadCanonicalHash(odr.Database(), number) if (hash != common.Hash{}) { return hash, nil } @@ -82,7 +83,7 @@ func GetCanonicalHash(ctx context.Context, odr OdrBackend, number uint64) (commo // GetBodyRLP retrieves the block body (transactions and uncles) in RLP encoding. func GetBodyRLP(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64) (rlp.RawValue, error) { - if data := core.GetBodyRLP(odr.Database(), hash, number); data != nil { + if data := rawdb.ReadBodyRLP(odr.Database(), hash, number); data != nil { return data, nil } r := &BlockRequest{Hash: hash, Number: number} @@ -111,7 +112,7 @@ func GetBody(ctx context.Context, odr OdrBackend, hash common.Hash, number uint6 // back from the stored header and body. func GetBlock(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64) (*types.Block, error) { // Retrieve the block header and body contents - header := core.GetHeader(odr.Database(), hash, number) + header := rawdb.ReadHeader(odr.Database(), hash, number) if header == nil { return nil, ErrNoHeader } @@ -127,7 +128,7 @@ func GetBlock(ctx context.Context, odr OdrBackend, hash common.Hash, number uint // in a block given by its hash. func GetBlockReceipts(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64) (types.Receipts, error) { // Retrieve the potentially incomplete receipts from disk or network - receipts := core.GetBlockReceipts(odr.Database(), hash, number) + receipts := rawdb.ReadReceipts(odr.Database(), hash, number) if receipts == nil { r := &ReceiptsRequest{Hash: hash, Number: number} if err := odr.Retrieve(ctx, r); err != nil { @@ -141,13 +142,13 @@ func GetBlockReceipts(ctx context.Context, odr OdrBackend, hash common.Hash, num if err != nil { return nil, err } - genesis := core.GetCanonicalHash(odr.Database(), 0) - config, _ := core.GetChainConfig(odr.Database(), genesis) + genesis := rawdb.ReadCanonicalHash(odr.Database(), 0) + config := rawdb.ReadChainConfig(odr.Database(), genesis) if err := core.SetReceiptsData(config, block, receipts); err != nil { return nil, err } - core.WriteBlockReceipts(odr.Database(), hash, number, receipts) + rawdb.WriteReceipts(odr.Database(), hash, number, receipts) } return receipts, nil } @@ -156,7 +157,7 @@ func GetBlockReceipts(ctx context.Context, odr OdrBackend, hash common.Hash, num // block given by its hash. func GetBlockLogs(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64) ([][]*types.Log, error) { // Retrieve the potentially incomplete receipts from disk or network - receipts := core.GetBlockReceipts(odr.Database(), hash, number) + receipts := rawdb.ReadReceipts(odr.Database(), hash, number) if receipts == nil { r := &ReceiptsRequest{Hash: hash, Number: number} if err := odr.Retrieve(ctx, r); err != nil { @@ -187,24 +188,24 @@ func GetBloomBits(ctx context.Context, odr OdrBackend, bitIdx uint, sectionIdxLi ) if odr.BloomTrieIndexer() != nil { bloomTrieCount, sectionHeadNum, sectionHead = odr.BloomTrieIndexer().Sections() - canonicalHash := core.GetCanonicalHash(db, sectionHeadNum) + canonicalHash := rawdb.ReadCanonicalHash(db, sectionHeadNum) // if the BloomTrie was injected as a trusted checkpoint, we have no canonical hash yet so we accept zero hash too for bloomTrieCount > 0 && canonicalHash != sectionHead && canonicalHash != (common.Hash{}) { bloomTrieCount-- if bloomTrieCount > 0 { sectionHeadNum = bloomTrieCount*BloomTrieFrequency - 1 sectionHead = odr.BloomTrieIndexer().SectionHead(bloomTrieCount - 1) - canonicalHash = core.GetCanonicalHash(db, sectionHeadNum) + canonicalHash = rawdb.ReadCanonicalHash(db, sectionHeadNum) } } } for i, sectionIdx := range sectionIdxList { - sectionHead := core.GetCanonicalHash(db, (sectionIdx+1)*BloomTrieFrequency-1) + sectionHead := rawdb.ReadCanonicalHash(db, (sectionIdx+1)*BloomTrieFrequency-1) // if we don't have the canonical hash stored for this section head number, we'll still look for // an entry with a zero sectionHead (we store it with zero section head too if we don't know it // at the time of the retrieval) - bloomBits, err := core.GetBloomBits(db, bitIdx, sectionIdx, sectionHead) + bloomBits, err := rawdb.ReadBloomBits(db, bitIdx, sectionIdx, sectionHead) if err == nil { result[i] = bloomBits } else { diff --git a/light/postprocess.go b/light/postprocess.go index 702eb53c86..8b9a029112 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/bitutil" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" @@ -161,7 +162,7 @@ func (c *ChtIndexerBackend) Process(header *types.Header) { hash, num := header.Hash(), header.Number.Uint64() c.lastHash = hash - td := core.GetTd(c.diskdb, hash, num) + td := rawdb.ReadTd(c.diskdb, hash, num) if td == nil { panic(nil) } @@ -272,7 +273,7 @@ func (b *BloomTrieIndexerBackend) Commit() error { binary.BigEndian.PutUint64(encKey[2:10], b.section) var decomp []byte for j := uint64(0); j < b.bloomTrieRatio; j++ { - data, err := core.GetBloomBits(b.diskdb, i, b.section*b.bloomTrieRatio+j, b.sectionHeads[j]) + data, err := rawdb.ReadBloomBits(b.diskdb, i, b.section*b.bloomTrieRatio+j, b.sectionHeads[j]) if err != nil { return err } diff --git a/light/trie_test.go b/light/trie_test.go index 0d6b2cc1d8..84c6f162fb 100644 --- a/light/trie_test.go +++ b/light/trie_test.go @@ -34,10 +34,10 @@ import ( func TestNodeIterator(t *testing.T) { var ( - fulldb, _ = ethdb.NewMemDatabase() - lightdb, _ = ethdb.NewMemDatabase() - gspec = core.Genesis{Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}} - genesis = gspec.MustCommit(fulldb) + fulldb = ethdb.NewMemDatabase() + lightdb = ethdb.NewMemDatabase() + gspec = core.Genesis{Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}} + genesis = gspec.MustCommit(fulldb) ) gspec.MustCommit(lightdb) blockchain, _ := core.NewBlockChain(fulldb, nil, params.TestChainConfig, ethash.NewFullFaker(), vm.Config{}) diff --git a/light/txpool.go b/light/txpool.go index ca41490bdc..94c8139cbd 100644 --- a/light/txpool.go +++ b/light/txpool.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" @@ -183,9 +184,8 @@ func (pool *TxPool) checkMinedTxs(ctx context.Context, hash common.Hash, number if _, err := GetBlockReceipts(ctx, pool.odr, hash, number); err != nil { // ODR caches, ignore results return err } - if err := core.WriteTxLookupEntries(pool.chainDb, block); err != nil { - return err - } + rawdb.WriteTxLookupEntries(pool.chainDb, block) + // Update the transaction pool's state for _, tx := range list { delete(pool.pending, tx.Hash()) @@ -202,7 +202,7 @@ func (pool *TxPool) rollbackTxs(hash common.Hash, txc txStateChanges) { if list, ok := pool.mined[hash]; ok { for _, tx := range list { txHash := tx.Hash() - core.DeleteTxLookupEntry(pool.chainDb, txHash) + rawdb.DeleteTxLookupEntry(pool.chainDb, txHash) pool.pending[txHash] = tx txc.setState(txHash, false) } @@ -258,7 +258,7 @@ func (pool *TxPool) reorgOnNewHead(ctx context.Context, newHeader *types.Header) idx2 := idx - txPermanent if len(pool.mined) > 0 { for i := pool.clearIdx; i < idx2; i++ { - hash := core.GetCanonicalHash(pool.chainDb, i) + hash := rawdb.ReadCanonicalHash(pool.chainDb, i) if list, ok := pool.mined[hash]; ok { hashes := make([]common.Hash, len(list)) for i, tx := range list { diff --git a/light/txpool_test.go b/light/txpool_test.go index 13d7d3cebb..ccbd83a943 100644 --- a/light/txpool_test.go +++ b/light/txpool_test.go @@ -81,8 +81,8 @@ func TestTxPool(t *testing.T) { } var ( - sdb, _ = ethdb.NewMemDatabase() - ldb, _ = ethdb.NewMemDatabase() + sdb = ethdb.NewMemDatabase() + ldb = ethdb.NewMemDatabase() gspec = core.Genesis{Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}} genesis = gspec.MustCommit(sdb) ) diff --git a/log/handler.go b/log/handler.go index d5594b853d..41d5718ddb 100644 --- a/log/handler.go +++ b/log/handler.go @@ -234,9 +234,8 @@ func FailoverHandler(hs ...Handler) Handler { err = h.Log(r) if err == nil { return nil - } else { - r.Ctx = append(r.Ctx, fmt.Sprintf("failover_err_%d", i), err) } + r.Ctx = append(r.Ctx, fmt.Sprintf("failover_err_%d", i), err) } return err @@ -320,13 +319,12 @@ func evaluateLazy(lz Lazy) (interface{}, error) { results := value.Call([]reflect.Value{}) if len(results) == 1 { return results[0].Interface(), nil - } else { - values := make([]interface{}, len(results)) - for i, v := range results { - values[i] = v.Interface() - } - return values, nil } + values := make([]interface{}, len(results)) + for i, v := range results { + values[i] = v.Interface() + } + return values, nil } // DiscardHandler reports success for all writes but does nothing. diff --git a/metrics/librato/client.go b/metrics/librato/client.go index 8c0c850e38..1f8920cb1a 100644 --- a/metrics/librato/client.go +++ b/metrics/librato/client.go @@ -65,7 +65,7 @@ type Batch struct { Source string `json:"source"` } -func (self *LibratoClient) PostMetrics(batch Batch) (err error) { +func (c *LibratoClient) PostMetrics(batch Batch) (err error) { var ( js []byte req *http.Request @@ -85,7 +85,7 @@ func (self *LibratoClient) PostMetrics(batch Batch) (err error) { } req.Header.Set("Content-Type", "application/json") - req.SetBasicAuth(self.Email, self.Token) + req.SetBasicAuth(c.Email, c.Token) if resp, err = http.DefaultClient.Do(req); err != nil { return diff --git a/metrics/librato/librato.go b/metrics/librato/librato.go index f8c8c9ecb7..2138e01ae8 100644 --- a/metrics/librato/librato.go +++ b/metrics/librato/librato.go @@ -40,14 +40,14 @@ func Librato(r metrics.Registry, d time.Duration, e string, t string, s string, NewReporter(r, d, e, t, s, p, u).Run() } -func (self *Reporter) Run() { +func (rep *Reporter) Run() { log.Printf("WARNING: This client has been DEPRECATED! It has been moved to https://github.com/mihasya/go-metrics-librato and will be removed from rcrowley/go-metrics on August 5th 2015") - ticker := time.Tick(self.Interval) - metricsApi := &LibratoClient{self.Email, self.Token} + ticker := time.Tick(rep.Interval) + metricsApi := &LibratoClient{rep.Email, rep.Token} for now := range ticker { var metrics Batch var err error - if metrics, err = self.BuildRequest(now, self.Registry); err != nil { + if metrics, err = rep.BuildRequest(now, rep.Registry); err != nil { log.Printf("ERROR constructing librato request body %s", err) continue } @@ -79,21 +79,21 @@ func sumSquaresTimer(t metrics.Timer) float64 { return sumSquares } -func (self *Reporter) BuildRequest(now time.Time, r metrics.Registry) (snapshot Batch, err error) { +func (rep *Reporter) BuildRequest(now time.Time, r metrics.Registry) (snapshot Batch, err error) { snapshot = Batch{ // coerce timestamps to a stepping fn so that they line up in Librato graphs - MeasureTime: (now.Unix() / self.intervalSec) * self.intervalSec, - Source: self.Source, + MeasureTime: (now.Unix() / rep.intervalSec) * rep.intervalSec, + Source: rep.Source, } snapshot.Gauges = make([]Measurement, 0) snapshot.Counters = make([]Measurement, 0) - histogramGaugeCount := 1 + len(self.Percentiles) + histogramGaugeCount := 1 + len(rep.Percentiles) r.Each(func(name string, metric interface{}) { - if self.Namespace != "" { - name = fmt.Sprintf("%s.%s", self.Namespace, name) + if rep.Namespace != "" { + name = fmt.Sprintf("%s.%s", rep.Namespace, name) } measurement := Measurement{} - measurement[Period] = self.Interval.Seconds() + measurement[Period] = rep.Interval.Seconds() switch m := metric.(type) { case metrics.Counter: if m.Count() > 0 { @@ -125,7 +125,7 @@ func (self *Reporter) BuildRequest(now time.Time, r metrics.Registry) (snapshot measurement[Sum] = float64(s.Sum()) measurement[SumSquares] = sumSquares(s) gauges[0] = measurement - for i, p := range self.Percentiles { + for i, p := range rep.Percentiles { gauges[i+1] = Measurement{ Name: fmt.Sprintf("%s.%.2f", measurement[Name], p), Value: s.Percentile(p), @@ -142,7 +142,7 @@ func (self *Reporter) BuildRequest(now time.Time, r metrics.Registry) (snapshot Measurement{ Name: fmt.Sprintf("%s.%s", name, "1min"), Value: m.Rate1(), - Period: int64(self.Interval.Seconds()), + Period: int64(rep.Interval.Seconds()), Attributes: map[string]interface{}{ DisplayUnitsLong: Operations, DisplayUnitsShort: OperationsShort, @@ -152,7 +152,7 @@ func (self *Reporter) BuildRequest(now time.Time, r metrics.Registry) (snapshot Measurement{ Name: fmt.Sprintf("%s.%s", name, "5min"), Value: m.Rate5(), - Period: int64(self.Interval.Seconds()), + Period: int64(rep.Interval.Seconds()), Attributes: map[string]interface{}{ DisplayUnitsLong: Operations, DisplayUnitsShort: OperationsShort, @@ -162,7 +162,7 @@ func (self *Reporter) BuildRequest(now time.Time, r metrics.Registry) (snapshot Measurement{ Name: fmt.Sprintf("%s.%s", name, "15min"), Value: m.Rate15(), - Period: int64(self.Interval.Seconds()), + Period: int64(rep.Interval.Seconds()), Attributes: map[string]interface{}{ DisplayUnitsLong: Operations, DisplayUnitsShort: OperationsShort, @@ -184,15 +184,15 @@ func (self *Reporter) BuildRequest(now time.Time, r metrics.Registry) (snapshot Max: float64(m.Max()), Min: float64(m.Min()), SumSquares: sumSquaresTimer(m), - Period: int64(self.Interval.Seconds()), - Attributes: self.TimerAttributes, + Period: int64(rep.Interval.Seconds()), + Attributes: rep.TimerAttributes, } - for i, p := range self.Percentiles { + for i, p := range rep.Percentiles { gauges[i+1] = Measurement{ Name: fmt.Sprintf("%s.timer.%2.0f", name, p*100), Value: m.Percentile(p), - Period: int64(self.Interval.Seconds()), - Attributes: self.TimerAttributes, + Period: int64(rep.Interval.Seconds()), + Attributes: rep.TimerAttributes, } } snapshot.Gauges = append(snapshot.Gauges, gauges...) @@ -200,7 +200,7 @@ func (self *Reporter) BuildRequest(now time.Time, r metrics.Registry) (snapshot Measurement{ Name: fmt.Sprintf("%s.%s", name, "rate.1min"), Value: m.Rate1(), - Period: int64(self.Interval.Seconds()), + Period: int64(rep.Interval.Seconds()), Attributes: map[string]interface{}{ DisplayUnitsLong: Operations, DisplayUnitsShort: OperationsShort, @@ -210,7 +210,7 @@ func (self *Reporter) BuildRequest(now time.Time, r metrics.Registry) (snapshot Measurement{ Name: fmt.Sprintf("%s.%s", name, "rate.5min"), Value: m.Rate5(), - Period: int64(self.Interval.Seconds()), + Period: int64(rep.Interval.Seconds()), Attributes: map[string]interface{}{ DisplayUnitsLong: Operations, DisplayUnitsShort: OperationsShort, @@ -220,7 +220,7 @@ func (self *Reporter) BuildRequest(now time.Time, r metrics.Registry) (snapshot Measurement{ Name: fmt.Sprintf("%s.%s", name, "rate.15min"), Value: m.Rate15(), - Period: int64(self.Interval.Seconds()), + Period: int64(rep.Interval.Seconds()), Attributes: map[string]interface{}{ DisplayUnitsLong: Operations, DisplayUnitsShort: OperationsShort, diff --git a/mobile/types.go b/mobile/types.go index 196c7839f5..f32b4918f3 100644 --- a/mobile/types.go +++ b/mobile/types.go @@ -329,6 +329,7 @@ func (r *Receipt) EncodeJSON() (string, error) { return string(data), err } +func (r *Receipt) GetStatus() int { return int(r.receipt.Status) } func (r *Receipt) GetPostState() []byte { return r.receipt.PostState } func (r *Receipt) GetCumulativeGasUsed() int64 { return int64(r.receipt.CumulativeGasUsed) } func (r *Receipt) GetBloom() *Bloom { return &Bloom{r.receipt.Bloom} } diff --git a/node/node.go b/node/node.go index 83b6c4c07e..c4368189f7 100644 --- a/node/node.go +++ b/node/node.go @@ -568,7 +568,7 @@ func (n *Node) EventMux() *event.TypeMux { // ephemeral, a memory database is returned. func (n *Node) OpenDatabase(name string, cache, handles int) (ethdb.Database, error) { if n.config.DataDir == "" { - return ethdb.NewMemDatabase() + return ethdb.NewMemDatabase(), nil } return ethdb.NewLDBDatabase(n.config.resolvePath(name), cache, handles) } diff --git a/node/service.go b/node/service.go index 55062a5004..afc43e8486 100644 --- a/node/service.go +++ b/node/service.go @@ -41,7 +41,7 @@ type ServiceContext struct { // node is an ephemeral one, a memory database is returned. func (ctx *ServiceContext) OpenDatabase(name string, cache int, handles int) (ethdb.Database, error) { if ctx.config.DataDir == "" { - return ethdb.NewMemDatabase() + return ethdb.NewMemDatabase(), nil } db, err := ethdb.NewLDBDatabase(ctx.config.resolvePath(name), cache, handles) if err != nil { diff --git a/p2p/discover/table_test.go b/p2p/discover/table_test.go index 3ce48d2995..f2d3f9a2ad 100644 --- a/p2p/discover/table_test.go +++ b/p2p/discover/table_test.go @@ -582,26 +582,26 @@ func (*preminedTestnet) ping(toid NodeID, toaddr *net.UDPAddr) error { return ni // mine generates a testnet struct literal with nodes at // various distances to the given target. -func (n *preminedTestnet) mine(target NodeID) { - n.target = target - n.targetSha = crypto.Keccak256Hash(n.target[:]) +func (tn *preminedTestnet) mine(target NodeID) { + tn.target = target + tn.targetSha = crypto.Keccak256Hash(tn.target[:]) found := 0 for found < bucketSize*10 { k := newkey() id := PubkeyID(&k.PublicKey) sha := crypto.Keccak256Hash(id[:]) - ld := logdist(n.targetSha, sha) - if len(n.dists[ld]) < bucketSize { - n.dists[ld] = append(n.dists[ld], id) + ld := logdist(tn.targetSha, sha) + if len(tn.dists[ld]) < bucketSize { + tn.dists[ld] = append(tn.dists[ld], id) fmt.Println("found ID with ld", ld) found++ } } fmt.Println("&preminedTestnet{") - fmt.Printf(" target: %#v,\n", n.target) - fmt.Printf(" targetSha: %#v,\n", n.targetSha) - fmt.Printf(" dists: [%d][]NodeID{\n", len(n.dists)) - for ld, ns := range n.dists { + fmt.Printf(" target: %#v,\n", tn.target) + fmt.Printf(" targetSha: %#v,\n", tn.targetSha) + fmt.Printf(" dists: [%d][]NodeID{\n", len(tn.dists)) + for ld, ns := range tn.dists { if len(ns) == 0 { continue } diff --git a/p2p/discv5/net_test.go b/p2p/discv5/net_test.go index 369282ca9c..001d193cc9 100644 --- a/p2p/discv5/net_test.go +++ b/p2p/discv5/net_test.go @@ -336,26 +336,26 @@ func (*preminedTestnet) localAddr() *net.UDPAddr { // mine generates a testnet struct literal with nodes at // various distances to the given target. -func (n *preminedTestnet) mine(target NodeID) { - n.target = target - n.targetSha = crypto.Keccak256Hash(n.target[:]) +func (tn *preminedTestnet) mine(target NodeID) { + tn.target = target + tn.targetSha = crypto.Keccak256Hash(tn.target[:]) found := 0 for found < bucketSize*10 { k := newkey() id := PubkeyID(&k.PublicKey) sha := crypto.Keccak256Hash(id[:]) - ld := logdist(n.targetSha, sha) - if len(n.dists[ld]) < bucketSize { - n.dists[ld] = append(n.dists[ld], id) + ld := logdist(tn.targetSha, sha) + if len(tn.dists[ld]) < bucketSize { + tn.dists[ld] = append(tn.dists[ld], id) fmt.Println("found ID with ld", ld) found++ } } fmt.Println("&preminedTestnet{") - fmt.Printf(" target: %#v,\n", n.target) - fmt.Printf(" targetSha: %#v,\n", n.targetSha) - fmt.Printf(" dists: [%d][]NodeID{\n", len(n.dists)) - for ld, ns := range n.dists { + fmt.Printf(" target: %#v,\n", tn.target) + fmt.Printf(" targetSha: %#v,\n", tn.targetSha) + fmt.Printf(" dists: [%d][]NodeID{\n", len(tn.dists)) + for ld, ns := range tn.dists { if len(ns) == 0 { continue } diff --git a/p2p/discv5/node.go b/p2p/discv5/node.go index fd88a55b15..3d47485122 100644 --- a/p2p/discv5/node.go +++ b/p2p/discv5/node.go @@ -315,11 +315,11 @@ func PubkeyID(pub *ecdsa.PublicKey) NodeID { // Pubkey returns the public key represented by the node ID. // It returns an error if the ID is not a point on the curve. -func (id NodeID) Pubkey() (*ecdsa.PublicKey, error) { +func (n NodeID) Pubkey() (*ecdsa.PublicKey, error) { p := &ecdsa.PublicKey{Curve: crypto.S256(), X: new(big.Int), Y: new(big.Int)} - half := len(id) / 2 - p.X.SetBytes(id[:half]) - p.Y.SetBytes(id[half:]) + half := len(n) / 2 + p.X.SetBytes(n[:half]) + p.Y.SetBytes(n[half:]) if !p.Curve.IsOnCurve(p.X, p.Y) { return nil, errors.New("id is invalid secp256k1 curve point") } diff --git a/p2p/discv5/ticket.go b/p2p/discv5/ticket.go index b3d1ac4baf..ae4b18e7cd 100644 --- a/p2p/discv5/ticket.go +++ b/p2p/discv5/ticket.go @@ -304,8 +304,8 @@ func (s ticketRefByWaitTime) Len() int { return len(s) } -func (r ticketRef) waitTime() mclock.AbsTime { - return r.t.regTime[r.idx] - r.t.issueTime +func (ref ticketRef) waitTime() mclock.AbsTime { + return ref.t.regTime[ref.idx] - ref.t.issueTime } // Less reports whether the element with diff --git a/p2p/discv5/topic.go b/p2p/discv5/topic.go index e7a7f8e02a..609a41297f 100644 --- a/p2p/discv5/topic.go +++ b/p2p/discv5/topic.go @@ -271,15 +271,15 @@ func (t *topicTable) useTicket(node *Node, serialNo uint32, topics []Topic, idx return false } -func (topictab *topicTable) getTicket(node *Node, topics []Topic) *ticket { - topictab.collectGarbage() +func (t *topicTable) getTicket(node *Node, topics []Topic) *ticket { + t.collectGarbage() now := mclock.Now() - n := topictab.getOrNewNode(node) + n := t.getOrNewNode(node) n.lastIssuedTicket++ - topictab.storeTicketCounters(node) + t.storeTicketCounters(node) - t := &ticket{ + tic := &ticket{ issueTime: now, topics: topics, serial: n.lastIssuedTicket, @@ -287,15 +287,15 @@ func (topictab *topicTable) getTicket(node *Node, topics []Topic) *ticket { } for i, topic := range topics { var waitPeriod time.Duration - if topic := topictab.topics[topic]; topic != nil { + if topic := t.topics[topic]; topic != nil { waitPeriod = topic.wcl.waitPeriod } else { waitPeriod = minWaitPeriod } - t.regTime[i] = now + mclock.AbsTime(waitPeriod) + tic.regTime[i] = now + mclock.AbsTime(waitPeriod) } - return t + return tic } const gcInterval = time.Minute diff --git a/p2p/message.go b/p2p/message.go index 50b419970d..a4eac54d33 100644 --- a/p2p/message.go +++ b/p2p/message.go @@ -229,21 +229,20 @@ func ExpectMsg(r MsgReader, code uint64, content interface{}) error { } if content == nil { return msg.Discard() - } else { - contentEnc, err := rlp.EncodeToBytes(content) - if err != nil { - panic("content encode error: " + err.Error()) - } - if int(msg.Size) != len(contentEnc) { - return fmt.Errorf("message size mismatch: got %d, want %d", msg.Size, len(contentEnc)) - } - actualContent, err := ioutil.ReadAll(msg.Payload) - if err != nil { - return err - } - if !bytes.Equal(actualContent, contentEnc) { - return fmt.Errorf("message payload mismatch:\ngot: %x\nwant: %x", actualContent, contentEnc) - } + } + contentEnc, err := rlp.EncodeToBytes(content) + if err != nil { + panic("content encode error: " + err.Error()) + } + if int(msg.Size) != len(contentEnc) { + return fmt.Errorf("message size mismatch: got %d, want %d", msg.Size, len(contentEnc)) + } + actualContent, err := ioutil.ReadAll(msg.Payload) + if err != nil { + return err + } + if !bytes.Equal(actualContent, contentEnc) { + return fmt.Errorf("message payload mismatch:\ngot: %x\nwant: %x", actualContent, contentEnc) } return nil } @@ -271,15 +270,15 @@ func newMsgEventer(rw MsgReadWriter, feed *event.Feed, peerID discover.NodeID, p // ReadMsg reads a message from the underlying MsgReadWriter and emits a // "message received" event -func (self *msgEventer) ReadMsg() (Msg, error) { - msg, err := self.MsgReadWriter.ReadMsg() +func (ev *msgEventer) ReadMsg() (Msg, error) { + msg, err := ev.MsgReadWriter.ReadMsg() if err != nil { return msg, err } - self.feed.Send(&PeerEvent{ + ev.feed.Send(&PeerEvent{ Type: PeerEventTypeMsgRecv, - Peer: self.peerID, - Protocol: self.Protocol, + Peer: ev.peerID, + Protocol: ev.Protocol, MsgCode: &msg.Code, MsgSize: &msg.Size, }) @@ -288,15 +287,15 @@ func (self *msgEventer) ReadMsg() (Msg, error) { // WriteMsg writes a message to the underlying MsgReadWriter and emits a // "message sent" event -func (self *msgEventer) WriteMsg(msg Msg) error { - err := self.MsgReadWriter.WriteMsg(msg) +func (ev *msgEventer) WriteMsg(msg Msg) error { + err := ev.MsgReadWriter.WriteMsg(msg) if err != nil { return err } - self.feed.Send(&PeerEvent{ + ev.feed.Send(&PeerEvent{ Type: PeerEventTypeMsgSend, - Peer: self.peerID, - Protocol: self.Protocol, + Peer: ev.peerID, + Protocol: ev.Protocol, MsgCode: &msg.Code, MsgSize: &msg.Size, }) @@ -305,8 +304,8 @@ func (self *msgEventer) WriteMsg(msg Msg) error { // Close closes the underlying MsgReadWriter if it implements the io.Closer // interface -func (self *msgEventer) Close() error { - if v, ok := self.MsgReadWriter.(io.Closer); ok { +func (ev *msgEventer) Close() error { + if v, ok := ev.MsgReadWriter.(io.Closer); ok { return v.Close() } return nil diff --git a/p2p/peer.go b/p2p/peer.go index 73e33418ed..c3907349fc 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -220,6 +220,7 @@ loop: reason = discReasonForError(err) break loop case err = <-p.disc: + reason = discReasonForError(err) break loop } } diff --git a/p2p/peer_error.go b/p2p/peer_error.go index a1cddb707b..ab61bfef06 100644 --- a/p2p/peer_error.go +++ b/p2p/peer_error.go @@ -48,8 +48,8 @@ func newPeerError(code int, format string, v ...interface{}) *peerError { return err } -func (self *peerError) Error() string { - return self.message +func (pe *peerError) Error() string { + return pe.message } var errProtocolReturned = errors.New("protocol returned") diff --git a/p2p/simulations/adapters/exec.go b/p2p/simulations/adapters/exec.go index a566fb27d8..f381c11596 100644 --- a/p2p/simulations/adapters/exec.go +++ b/p2p/simulations/adapters/exec.go @@ -17,7 +17,6 @@ package adapters import ( - "bufio" "context" "crypto/ecdsa" "encoding/json" @@ -29,7 +28,6 @@ import ( "os/exec" "os/signal" "path/filepath" - "regexp" "strings" "sync" "syscall" @@ -150,10 +148,6 @@ func (n *ExecNode) Client() (*rpc.Client, error) { return n.client, nil } -// wsAddrPattern is a regex used to read the WebSocket address from the node's -// log -var wsAddrPattern = regexp.MustCompile(`ws://[\d.:]+`) - // Start exec's the node passing the ID and service as command line arguments // and the node config encoded as JSON in the _P2P_NODE_CONFIG environment // variable @@ -196,23 +190,9 @@ func (n *ExecNode) Start(snapshots map[string][]byte) (err error) { n.Cmd = cmd // read the WebSocket address from the stderr logs - var wsAddr string - wsAddrC := make(chan string) - go func() { - s := bufio.NewScanner(stderrR) - for s.Scan() { - if strings.Contains(s.Text(), "WebSocket endpoint opened:") { - wsAddrC <- wsAddrPattern.FindString(s.Text()) - } - } - }() - select { - case wsAddr = <-wsAddrC: - if wsAddr == "" { - return errors.New("failed to read WebSocket address from stderr") - } - case <-time.After(10 * time.Second): - return errors.New("timed out waiting for WebSocket address on stderr") + wsAddr, err := findWSAddr(stderrR, 10*time.Second) + if err != nil { + return fmt.Errorf("error getting WebSocket address: %s", err) } // create the RPC client and load the node info diff --git a/p2p/simulations/adapters/inproc.go b/p2p/simulations/adapters/inproc.go index 48d7c17301..6d90b4a9fc 100644 --- a/p2p/simulations/adapters/inproc.go +++ b/p2p/simulations/adapters/inproc.go @@ -154,30 +154,30 @@ type SimNode struct { } // Addr returns the node's discovery address -func (self *SimNode) Addr() []byte { - return []byte(self.Node().String()) +func (sn *SimNode) Addr() []byte { + return []byte(sn.Node().String()) } // Node returns a discover.Node representing the SimNode -func (self *SimNode) Node() *discover.Node { - return discover.NewNode(self.ID, net.IP{127, 0, 0, 1}, 30303, 30303) +func (sn *SimNode) Node() *discover.Node { + return discover.NewNode(sn.ID, net.IP{127, 0, 0, 1}, 30303, 30303) } // Client returns an rpc.Client which can be used to communicate with the // underlying services (it is set once the node has started) -func (self *SimNode) Client() (*rpc.Client, error) { - self.lock.RLock() - defer self.lock.RUnlock() - if self.client == nil { +func (sn *SimNode) Client() (*rpc.Client, error) { + sn.lock.RLock() + defer sn.lock.RUnlock() + if sn.client == nil { return nil, errors.New("node not started") } - return self.client, nil + return sn.client, nil } // ServeRPC serves RPC requests over the given connection by creating an // in-memory client to the node's RPC server -func (self *SimNode) ServeRPC(conn net.Conn) error { - handler, err := self.node.RPCHandler() +func (sn *SimNode) ServeRPC(conn net.Conn) error { + handler, err := sn.node.RPCHandler() if err != nil { return err } @@ -187,13 +187,13 @@ func (self *SimNode) ServeRPC(conn net.Conn) error { // Snapshots creates snapshots of the services by calling the // simulation_snapshot RPC method -func (self *SimNode) Snapshots() (map[string][]byte, error) { - self.lock.RLock() - services := make(map[string]node.Service, len(self.running)) - for name, service := range self.running { +func (sn *SimNode) Snapshots() (map[string][]byte, error) { + sn.lock.RLock() + services := make(map[string]node.Service, len(sn.running)) + for name, service := range sn.running { services[name] = service } - self.lock.RUnlock() + sn.lock.RUnlock() if len(services) == 0 { return nil, errors.New("no running services") } @@ -213,23 +213,23 @@ func (self *SimNode) Snapshots() (map[string][]byte, error) { } // Start registers the services and starts the underlying devp2p node -func (self *SimNode) Start(snapshots map[string][]byte) error { +func (sn *SimNode) Start(snapshots map[string][]byte) error { newService := func(name string) func(ctx *node.ServiceContext) (node.Service, error) { return func(nodeCtx *node.ServiceContext) (node.Service, error) { ctx := &ServiceContext{ - RPCDialer: self.adapter, + RPCDialer: sn.adapter, NodeContext: nodeCtx, - Config: self.config, + Config: sn.config, } if snapshots != nil { ctx.Snapshot = snapshots[name] } - serviceFunc := self.adapter.services[name] + serviceFunc := sn.adapter.services[name] service, err := serviceFunc(ctx) if err != nil { return nil, err } - self.running[name] = service + sn.running[name] = service return service, nil } } @@ -237,9 +237,9 @@ func (self *SimNode) Start(snapshots map[string][]byte) error { // ensure we only register the services once in the case of the node // being stopped and then started again var regErr error - self.registerOnce.Do(func() { - for _, name := range self.config.Services { - if err := self.node.Register(newService(name)); err != nil { + sn.registerOnce.Do(func() { + for _, name := range sn.config.Services { + if err := sn.node.Register(newService(name)); err != nil { regErr = err return } @@ -249,54 +249,54 @@ func (self *SimNode) Start(snapshots map[string][]byte) error { return regErr } - if err := self.node.Start(); err != nil { + if err := sn.node.Start(); err != nil { return err } // create an in-process RPC client - handler, err := self.node.RPCHandler() + handler, err := sn.node.RPCHandler() if err != nil { return err } - self.lock.Lock() - self.client = rpc.DialInProc(handler) - self.lock.Unlock() + sn.lock.Lock() + sn.client = rpc.DialInProc(handler) + sn.lock.Unlock() return nil } // Stop closes the RPC client and stops the underlying devp2p node -func (self *SimNode) Stop() error { - self.lock.Lock() - if self.client != nil { - self.client.Close() - self.client = nil +func (sn *SimNode) Stop() error { + sn.lock.Lock() + if sn.client != nil { + sn.client.Close() + sn.client = nil } - self.lock.Unlock() - return self.node.Stop() + sn.lock.Unlock() + return sn.node.Stop() } // Services returns a copy of the underlying services -func (self *SimNode) Services() []node.Service { - self.lock.RLock() - defer self.lock.RUnlock() - services := make([]node.Service, 0, len(self.running)) - for _, service := range self.running { +func (sn *SimNode) Services() []node.Service { + sn.lock.RLock() + defer sn.lock.RUnlock() + services := make([]node.Service, 0, len(sn.running)) + for _, service := range sn.running { services = append(services, service) } return services } // Server returns the underlying p2p.Server -func (self *SimNode) Server() *p2p.Server { - return self.node.Server() +func (sn *SimNode) Server() *p2p.Server { + return sn.node.Server() } // SubscribeEvents subscribes the given channel to peer events from the // underlying p2p.Server -func (self *SimNode) SubscribeEvents(ch chan *p2p.PeerEvent) event.Subscription { - srv := self.Server() +func (sn *SimNode) SubscribeEvents(ch chan *p2p.PeerEvent) event.Subscription { + srv := sn.Server() if srv == nil { panic("node not running") } @@ -304,12 +304,12 @@ func (self *SimNode) SubscribeEvents(ch chan *p2p.PeerEvent) event.Subscription } // NodeInfo returns information about the node -func (self *SimNode) NodeInfo() *p2p.NodeInfo { - server := self.Server() +func (sn *SimNode) NodeInfo() *p2p.NodeInfo { + server := sn.Server() if server == nil { return &p2p.NodeInfo{ - ID: self.ID.String(), - Enode: self.Node().String(), + ID: sn.ID.String(), + Enode: sn.Node().String(), } } return server.NodeInfo() diff --git a/p2p/simulations/adapters/state.go b/p2p/simulations/adapters/state.go index 0d4ecfb0ff..78dfb11f95 100644 --- a/p2p/simulations/adapters/state.go +++ b/p2p/simulations/adapters/state.go @@ -20,12 +20,12 @@ type SimStateStore struct { m map[string][]byte } -func (self *SimStateStore) Load(s string) ([]byte, error) { - return self.m[s], nil +func (st *SimStateStore) Load(s string) ([]byte, error) { + return st.m[s], nil } -func (self *SimStateStore) Save(s string, data []byte) error { - self.m[s] = data +func (st *SimStateStore) Save(s string, data []byte) error { + st.m[s] = data return nil } diff --git a/p2p/simulations/adapters/ws.go b/p2p/simulations/adapters/ws.go new file mode 100644 index 0000000000..979a21709e --- /dev/null +++ b/p2p/simulations/adapters/ws.go @@ -0,0 +1,51 @@ +package adapters + +import ( + "bufio" + "errors" + "io" + "regexp" + "strings" + "time" +) + +// wsAddrPattern is a regex used to read the WebSocket address from the node's +// log +var wsAddrPattern = regexp.MustCompile(`ws://[\d.:]+`) + +func matchWSAddr(str string) (string, bool) { + if !strings.Contains(str, "WebSocket endpoint opened") { + return "", false + } + + return wsAddrPattern.FindString(str), true +} + +// findWSAddr scans through reader r, looking for the log entry with +// WebSocket address information. +func findWSAddr(r io.Reader, timeout time.Duration) (string, error) { + ch := make(chan string) + + go func() { + s := bufio.NewScanner(r) + for s.Scan() { + addr, ok := matchWSAddr(s.Text()) + if ok { + ch <- addr + } + } + close(ch) + }() + + var wsAddr string + select { + case wsAddr = <-ch: + if wsAddr == "" { + return "", errors.New("empty result") + } + case <-time.After(timeout): + return "", errors.New("timed out") + } + + return wsAddr, nil +} diff --git a/p2p/simulations/adapters/ws_test.go b/p2p/simulations/adapters/ws_test.go new file mode 100644 index 0000000000..0bb9ed2b2b --- /dev/null +++ b/p2p/simulations/adapters/ws_test.go @@ -0,0 +1,21 @@ +package adapters + +import ( + "bytes" + "testing" + "time" +) + +func TestFindWSAddr(t *testing.T) { + line := `t=2018-05-02T19:00:45+0200 lvl=info msg="WebSocket endpoint opened" node.id=26c65a606d1125a44695bc08573190d047152b6b9a776ccbbe593e90f91444d9c1ebdadac6a775ad9fdd0923468a1d698ed3a842c1fb89c1bc0f9d4801f8c39c url=ws://127.0.0.1:59975` + buf := bytes.NewBufferString(line) + got, err := findWSAddr(buf, 10*time.Second) + if err != nil { + t.Fatalf("Failed to find addr: %v", err) + } + expected := `ws://127.0.0.1:59975` + + if got != expected { + t.Fatalf("Expected to get '%s', but got '%s'", expected, got) + } +} diff --git a/p2p/simulations/network.go b/p2p/simulations/network.go index caf428ece1..1a2c1e8ff4 100644 --- a/p2p/simulations/network.go +++ b/p2p/simulations/network.go @@ -74,22 +74,22 @@ func NewNetwork(nodeAdapter adapters.NodeAdapter, conf *NetworkConfig) *Network } // Events returns the output event feed of the Network. -func (self *Network) Events() *event.Feed { - return &self.events +func (net *Network) Events() *event.Feed { + return &net.events } // NewNode adds a new node to the network with a random ID -func (self *Network) NewNode() (*Node, error) { +func (net *Network) NewNode() (*Node, error) { conf := adapters.RandomNodeConfig() - conf.Services = []string{self.DefaultService} - return self.NewNodeWithConfig(conf) + conf.Services = []string{net.DefaultService} + return net.NewNodeWithConfig(conf) } // NewNodeWithConfig adds a new node to the network with the given config, // returning an error if a node with the same ID or name already exists -func (self *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error) { - self.lock.Lock() - defer self.lock.Unlock() +func (net *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error) { + net.lock.Lock() + defer net.lock.Unlock() // create a random ID and PrivateKey if not set if conf.ID == (discover.NodeID{}) { @@ -100,31 +100,31 @@ func (self *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error) id := conf.ID if conf.Reachable == nil { conf.Reachable = func(otherID discover.NodeID) bool { - _, err := self.InitConn(conf.ID, otherID) + _, err := net.InitConn(conf.ID, otherID) return err == nil } } // assign a name to the node if not set if conf.Name == "" { - conf.Name = fmt.Sprintf("node%02d", len(self.Nodes)+1) + conf.Name = fmt.Sprintf("node%02d", len(net.Nodes)+1) } // check the node doesn't already exist - if node := self.getNode(id); node != nil { + if node := net.getNode(id); node != nil { return nil, fmt.Errorf("node with ID %q already exists", id) } - if node := self.getNodeByName(conf.Name); node != nil { + if node := net.getNodeByName(conf.Name); node != nil { return nil, fmt.Errorf("node with name %q already exists", conf.Name) } // if no services are configured, use the default service if len(conf.Services) == 0 { - conf.Services = []string{self.DefaultService} + conf.Services = []string{net.DefaultService} } // use the NodeAdapter to create the node - adapterNode, err := self.nodeAdapter.NewNode(conf) + adapterNode, err := net.nodeAdapter.NewNode(conf) if err != nil { return nil, err } @@ -133,27 +133,27 @@ func (self *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error) Config: conf, } log.Trace(fmt.Sprintf("node %v created", id)) - self.nodeMap[id] = len(self.Nodes) - self.Nodes = append(self.Nodes, node) + net.nodeMap[id] = len(net.Nodes) + net.Nodes = append(net.Nodes, node) // emit a "control" event - self.events.Send(ControlEvent(node)) + net.events.Send(ControlEvent(node)) return node, nil } // Config returns the network configuration -func (self *Network) Config() *NetworkConfig { - return &self.NetworkConfig +func (net *Network) Config() *NetworkConfig { + return &net.NetworkConfig } // StartAll starts all nodes in the network -func (self *Network) StartAll() error { - for _, node := range self.Nodes { +func (net *Network) StartAll() error { + for _, node := range net.Nodes { if node.Up { continue } - if err := self.Start(node.ID()); err != nil { + if err := net.Start(node.ID()); err != nil { return err } } @@ -161,12 +161,12 @@ func (self *Network) StartAll() error { } // StopAll stops all nodes in the network -func (self *Network) StopAll() error { - for _, node := range self.Nodes { +func (net *Network) StopAll() error { + for _, node := range net.Nodes { if !node.Up { continue } - if err := self.Stop(node.ID()); err != nil { + if err := net.Stop(node.ID()); err != nil { return err } } @@ -174,21 +174,21 @@ func (self *Network) StopAll() error { } // Start starts the node with the given ID -func (self *Network) Start(id discover.NodeID) error { - return self.startWithSnapshots(id, nil) +func (net *Network) Start(id discover.NodeID) error { + return net.startWithSnapshots(id, nil) } // startWithSnapshots starts the node with the given ID using the give // snapshots -func (self *Network) startWithSnapshots(id discover.NodeID, snapshots map[string][]byte) error { - node := self.GetNode(id) +func (net *Network) startWithSnapshots(id discover.NodeID, snapshots map[string][]byte) error { + node := net.GetNode(id) if node == nil { return fmt.Errorf("node %v does not exist", id) } if node.Up { return fmt.Errorf("node %v already up", id) } - log.Trace(fmt.Sprintf("starting node %v: %v using %v", id, node.Up, self.nodeAdapter.Name())) + log.Trace(fmt.Sprintf("starting node %v: %v using %v", id, node.Up, net.nodeAdapter.Name())) if err := node.Start(snapshots); err != nil { log.Warn(fmt.Sprintf("start up failed: %v", err)) return err @@ -196,7 +196,7 @@ func (self *Network) startWithSnapshots(id discover.NodeID, snapshots map[string node.Up = true log.Info(fmt.Sprintf("started node %v: %v", id, node.Up)) - self.events.Send(NewEvent(node)) + net.events.Send(NewEvent(node)) // subscribe to peer events client, err := node.Client() @@ -208,22 +208,22 @@ func (self *Network) startWithSnapshots(id discover.NodeID, snapshots map[string if err != nil { return fmt.Errorf("error getting peer events for node %v: %s", id, err) } - go self.watchPeerEvents(id, events, sub) + go net.watchPeerEvents(id, events, sub) return nil } // watchPeerEvents reads peer events from the given channel and emits // corresponding network events -func (self *Network) watchPeerEvents(id discover.NodeID, events chan *p2p.PeerEvent, sub event.Subscription) { +func (net *Network) watchPeerEvents(id discover.NodeID, events chan *p2p.PeerEvent, sub event.Subscription) { defer func() { sub.Unsubscribe() // assume the node is now down - self.lock.Lock() - node := self.getNode(id) + net.lock.Lock() + node := net.getNode(id) node.Up = false - self.lock.Unlock() - self.events.Send(NewEvent(node)) + net.lock.Unlock() + net.events.Send(NewEvent(node)) }() for { select { @@ -235,16 +235,16 @@ func (self *Network) watchPeerEvents(id discover.NodeID, events chan *p2p.PeerEv switch event.Type { case p2p.PeerEventTypeAdd: - self.DidConnect(id, peer) + net.DidConnect(id, peer) case p2p.PeerEventTypeDrop: - self.DidDisconnect(id, peer) + net.DidDisconnect(id, peer) case p2p.PeerEventTypeMsgSend: - self.DidSend(id, peer, event.Protocol, *event.MsgCode) + net.DidSend(id, peer, event.Protocol, *event.MsgCode) case p2p.PeerEventTypeMsgRecv: - self.DidReceive(peer, id, event.Protocol, *event.MsgCode) + net.DidReceive(peer, id, event.Protocol, *event.MsgCode) } @@ -258,8 +258,8 @@ func (self *Network) watchPeerEvents(id discover.NodeID, events chan *p2p.PeerEv } // Stop stops the node with the given ID -func (self *Network) Stop(id discover.NodeID) error { - node := self.GetNode(id) +func (net *Network) Stop(id discover.NodeID) error { + node := net.GetNode(id) if node == nil { return fmt.Errorf("node %v does not exist", id) } @@ -272,15 +272,15 @@ func (self *Network) Stop(id discover.NodeID) error { node.Up = false log.Info(fmt.Sprintf("stop node %v: %v", id, node.Up)) - self.events.Send(ControlEvent(node)) + net.events.Send(ControlEvent(node)) return nil } // Connect connects two nodes together by calling the "admin_addPeer" RPC // method on the "one" node so that it connects to the "other" node -func (self *Network) Connect(oneID, otherID discover.NodeID) error { +func (net *Network) Connect(oneID, otherID discover.NodeID) error { log.Debug(fmt.Sprintf("connecting %s to %s", oneID, otherID)) - conn, err := self.InitConn(oneID, otherID) + conn, err := net.InitConn(oneID, otherID) if err != nil { return err } @@ -288,14 +288,14 @@ func (self *Network) Connect(oneID, otherID discover.NodeID) error { if err != nil { return err } - self.events.Send(ControlEvent(conn)) + net.events.Send(ControlEvent(conn)) return client.Call(nil, "admin_addPeer", string(conn.other.Addr())) } // Disconnect disconnects two nodes by calling the "admin_removePeer" RPC // method on the "one" node so that it disconnects from the "other" node -func (self *Network) Disconnect(oneID, otherID discover.NodeID) error { - conn := self.GetConn(oneID, otherID) +func (net *Network) Disconnect(oneID, otherID discover.NodeID) error { + conn := net.GetConn(oneID, otherID) if conn == nil { return fmt.Errorf("connection between %v and %v does not exist", oneID, otherID) } @@ -306,13 +306,13 @@ func (self *Network) Disconnect(oneID, otherID discover.NodeID) error { if err != nil { return err } - self.events.Send(ControlEvent(conn)) + net.events.Send(ControlEvent(conn)) return client.Call(nil, "admin_removePeer", string(conn.other.Addr())) } // DidConnect tracks the fact that the "one" node connected to the "other" node -func (self *Network) DidConnect(one, other discover.NodeID) error { - conn, err := self.GetOrCreateConn(one, other) +func (net *Network) DidConnect(one, other discover.NodeID) error { + conn, err := net.GetOrCreateConn(one, other) if err != nil { return fmt.Errorf("connection between %v and %v does not exist", one, other) } @@ -320,14 +320,14 @@ func (self *Network) DidConnect(one, other discover.NodeID) error { return fmt.Errorf("%v and %v already connected", one, other) } conn.Up = true - self.events.Send(NewEvent(conn)) + net.events.Send(NewEvent(conn)) return nil } // DidDisconnect tracks the fact that the "one" node disconnected from the // "other" node -func (self *Network) DidDisconnect(one, other discover.NodeID) error { - conn := self.GetConn(one, other) +func (net *Network) DidDisconnect(one, other discover.NodeID) error { + conn := net.GetConn(one, other) if conn == nil { return fmt.Errorf("connection between %v and %v does not exist", one, other) } @@ -336,12 +336,12 @@ func (self *Network) DidDisconnect(one, other discover.NodeID) error { } conn.Up = false conn.initiated = time.Now().Add(-dialBanTimeout) - self.events.Send(NewEvent(conn)) + net.events.Send(NewEvent(conn)) return nil } // DidSend tracks the fact that "sender" sent a message to "receiver" -func (self *Network) DidSend(sender, receiver discover.NodeID, proto string, code uint64) error { +func (net *Network) DidSend(sender, receiver discover.NodeID, proto string, code uint64) error { msg := &Msg{ One: sender, Other: receiver, @@ -349,12 +349,12 @@ func (self *Network) DidSend(sender, receiver discover.NodeID, proto string, cod Code: code, Received: false, } - self.events.Send(NewEvent(msg)) + net.events.Send(NewEvent(msg)) return nil } // DidReceive tracks the fact that "receiver" received a message from "sender" -func (self *Network) DidReceive(sender, receiver discover.NodeID, proto string, code uint64) error { +func (net *Network) DidReceive(sender, receiver discover.NodeID, proto string, code uint64) error { msg := &Msg{ One: sender, Other: receiver, @@ -362,36 +362,36 @@ func (self *Network) DidReceive(sender, receiver discover.NodeID, proto string, Code: code, Received: true, } - self.events.Send(NewEvent(msg)) + net.events.Send(NewEvent(msg)) return nil } // GetNode gets the node with the given ID, returning nil if the node does not // exist -func (self *Network) GetNode(id discover.NodeID) *Node { - self.lock.Lock() - defer self.lock.Unlock() - return self.getNode(id) +func (net *Network) GetNode(id discover.NodeID) *Node { + net.lock.Lock() + defer net.lock.Unlock() + return net.getNode(id) } // GetNode gets the node with the given name, returning nil if the node does // not exist -func (self *Network) GetNodeByName(name string) *Node { - self.lock.Lock() - defer self.lock.Unlock() - return self.getNodeByName(name) +func (net *Network) GetNodeByName(name string) *Node { + net.lock.Lock() + defer net.lock.Unlock() + return net.getNodeByName(name) } -func (self *Network) getNode(id discover.NodeID) *Node { - i, found := self.nodeMap[id] +func (net *Network) getNode(id discover.NodeID) *Node { + i, found := net.nodeMap[id] if !found { return nil } - return self.Nodes[i] + return net.Nodes[i] } -func (self *Network) getNodeByName(name string) *Node { - for _, node := range self.Nodes { +func (net *Network) getNodeByName(name string) *Node { + for _, node := range net.Nodes { if node.Config.Name == name { return node } @@ -400,40 +400,40 @@ func (self *Network) getNodeByName(name string) *Node { } // GetNodes returns the existing nodes -func (self *Network) GetNodes() (nodes []*Node) { - self.lock.Lock() - defer self.lock.Unlock() +func (net *Network) GetNodes() (nodes []*Node) { + net.lock.Lock() + defer net.lock.Unlock() - nodes = append(nodes, self.Nodes...) + nodes = append(nodes, net.Nodes...) return nodes } // GetConn returns the connection which exists between "one" and "other" // regardless of which node initiated the connection -func (self *Network) GetConn(oneID, otherID discover.NodeID) *Conn { - self.lock.Lock() - defer self.lock.Unlock() - return self.getConn(oneID, otherID) +func (net *Network) GetConn(oneID, otherID discover.NodeID) *Conn { + net.lock.Lock() + defer net.lock.Unlock() + return net.getConn(oneID, otherID) } // GetOrCreateConn is like GetConn but creates the connection if it doesn't // already exist -func (self *Network) GetOrCreateConn(oneID, otherID discover.NodeID) (*Conn, error) { - self.lock.Lock() - defer self.lock.Unlock() - return self.getOrCreateConn(oneID, otherID) +func (net *Network) GetOrCreateConn(oneID, otherID discover.NodeID) (*Conn, error) { + net.lock.Lock() + defer net.lock.Unlock() + return net.getOrCreateConn(oneID, otherID) } -func (self *Network) getOrCreateConn(oneID, otherID discover.NodeID) (*Conn, error) { - if conn := self.getConn(oneID, otherID); conn != nil { +func (net *Network) getOrCreateConn(oneID, otherID discover.NodeID) (*Conn, error) { + if conn := net.getConn(oneID, otherID); conn != nil { return conn, nil } - one := self.getNode(oneID) + one := net.getNode(oneID) if one == nil { return nil, fmt.Errorf("node %v does not exist", oneID) } - other := self.getNode(otherID) + other := net.getNode(otherID) if other == nil { return nil, fmt.Errorf("node %v does not exist", otherID) } @@ -444,18 +444,18 @@ func (self *Network) getOrCreateConn(oneID, otherID discover.NodeID) (*Conn, err other: other, } label := ConnLabel(oneID, otherID) - self.connMap[label] = len(self.Conns) - self.Conns = append(self.Conns, conn) + net.connMap[label] = len(net.Conns) + net.Conns = append(net.Conns, conn) return conn, nil } -func (self *Network) getConn(oneID, otherID discover.NodeID) *Conn { +func (net *Network) getConn(oneID, otherID discover.NodeID) *Conn { label := ConnLabel(oneID, otherID) - i, found := self.connMap[label] + i, found := net.connMap[label] if !found { return nil } - return self.Conns[i] + return net.Conns[i] } // InitConn(one, other) retrieves the connectiton model for the connection between @@ -466,13 +466,13 @@ func (self *Network) getConn(oneID, otherID discover.NodeID) *Conn { // it also checks whether there has been recent attempt to connect the peers // this is cheating as the simulation is used as an oracle and know about // remote peers attempt to connect to a node which will then not initiate the connection -func (self *Network) InitConn(oneID, otherID discover.NodeID) (*Conn, error) { - self.lock.Lock() - defer self.lock.Unlock() +func (net *Network) InitConn(oneID, otherID discover.NodeID) (*Conn, error) { + net.lock.Lock() + defer net.lock.Unlock() if oneID == otherID { return nil, fmt.Errorf("refusing to connect to self %v", oneID) } - conn, err := self.getOrCreateConn(oneID, otherID) + conn, err := net.getOrCreateConn(oneID, otherID) if err != nil { return nil, err } @@ -491,28 +491,28 @@ func (self *Network) InitConn(oneID, otherID discover.NodeID) (*Conn, error) { } // Shutdown stops all nodes in the network and closes the quit channel -func (self *Network) Shutdown() { - for _, node := range self.Nodes { +func (net *Network) Shutdown() { + for _, node := range net.Nodes { log.Debug(fmt.Sprintf("stopping node %s", node.ID().TerminalString())) if err := node.Stop(); err != nil { log.Warn(fmt.Sprintf("error stopping node %s", node.ID().TerminalString()), "err", err) } } - close(self.quitc) + close(net.quitc) } //Reset resets all network properties: //emtpies the nodes and the connection list -func (self *Network) Reset() { - self.lock.Lock() - defer self.lock.Unlock() +func (net *Network) Reset() { + net.lock.Lock() + defer net.lock.Unlock() //re-initialize the maps - self.connMap = make(map[string]int) - self.nodeMap = make(map[discover.NodeID]int) + net.connMap = make(map[string]int) + net.nodeMap = make(map[discover.NodeID]int) - self.Nodes = nil - self.Conns = nil + net.Nodes = nil + net.Conns = nil } // Node is a wrapper around adapters.Node which is used to track the status @@ -528,37 +528,37 @@ type Node struct { } // ID returns the ID of the node -func (self *Node) ID() discover.NodeID { - return self.Config.ID +func (n *Node) ID() discover.NodeID { + return n.Config.ID } // String returns a log-friendly string -func (self *Node) String() string { - return fmt.Sprintf("Node %v", self.ID().TerminalString()) +func (n *Node) String() string { + return fmt.Sprintf("Node %v", n.ID().TerminalString()) } // NodeInfo returns information about the node -func (self *Node) NodeInfo() *p2p.NodeInfo { +func (n *Node) NodeInfo() *p2p.NodeInfo { // avoid a panic if the node is not started yet - if self.Node == nil { + if n.Node == nil { return nil } - info := self.Node.NodeInfo() - info.Name = self.Config.Name + info := n.Node.NodeInfo() + info.Name = n.Config.Name return info } // MarshalJSON implements the json.Marshaler interface so that the encoded // JSON includes the NodeInfo -func (self *Node) MarshalJSON() ([]byte, error) { +func (n *Node) MarshalJSON() ([]byte, error) { return json.Marshal(struct { Info *p2p.NodeInfo `json:"info,omitempty"` Config *adapters.NodeConfig `json:"config,omitempty"` Up bool `json:"up"` }{ - Info: self.NodeInfo(), - Config: self.Config, - Up: self.Up, + Info: n.NodeInfo(), + Config: n.Config, + Up: n.Up, }) } @@ -580,19 +580,19 @@ type Conn struct { } // nodesUp returns whether both nodes are currently up -func (self *Conn) nodesUp() error { - if !self.one.Up { - return fmt.Errorf("one %v is not up", self.One) +func (c *Conn) nodesUp() error { + if !c.one.Up { + return fmt.Errorf("one %v is not up", c.One) } - if !self.other.Up { - return fmt.Errorf("other %v is not up", self.Other) + if !c.other.Up { + return fmt.Errorf("other %v is not up", c.Other) } return nil } // String returns a log-friendly string -func (self *Conn) String() string { - return fmt.Sprintf("Conn %v->%v", self.One.TerminalString(), self.Other.TerminalString()) +func (c *Conn) String() string { + return fmt.Sprintf("Conn %v->%v", c.One.TerminalString(), c.Other.TerminalString()) } // Msg represents a p2p message sent between two nodes in the network @@ -605,8 +605,8 @@ type Msg struct { } // String returns a log-friendly string -func (self *Msg) String() string { - return fmt.Sprintf("Msg(%d) %v->%v", self.Code, self.One.TerminalString(), self.Other.TerminalString()) +func (m *Msg) String() string { + return fmt.Sprintf("Msg(%d) %v->%v", m.Code, m.One.TerminalString(), m.Other.TerminalString()) } // ConnLabel generates a deterministic string which represents a connection @@ -640,14 +640,14 @@ type NodeSnapshot struct { } // Snapshot creates a network snapshot -func (self *Network) Snapshot() (*Snapshot, error) { - self.lock.Lock() - defer self.lock.Unlock() +func (net *Network) Snapshot() (*Snapshot, error) { + net.lock.Lock() + defer net.lock.Unlock() snap := &Snapshot{ - Nodes: make([]NodeSnapshot, len(self.Nodes)), - Conns: make([]Conn, len(self.Conns)), + Nodes: make([]NodeSnapshot, len(net.Nodes)), + Conns: make([]Conn, len(net.Conns)), } - for i, node := range self.Nodes { + for i, node := range net.Nodes { snap.Nodes[i] = NodeSnapshot{Node: *node} if !node.Up { continue @@ -658,33 +658,33 @@ func (self *Network) Snapshot() (*Snapshot, error) { } snap.Nodes[i].Snapshots = snapshots } - for i, conn := range self.Conns { + for i, conn := range net.Conns { snap.Conns[i] = *conn } return snap, nil } // Load loads a network snapshot -func (self *Network) Load(snap *Snapshot) error { +func (net *Network) Load(snap *Snapshot) error { for _, n := range snap.Nodes { - if _, err := self.NewNodeWithConfig(n.Node.Config); err != nil { + if _, err := net.NewNodeWithConfig(n.Node.Config); err != nil { return err } if !n.Node.Up { continue } - if err := self.startWithSnapshots(n.Node.Config.ID, n.Snapshots); err != nil { + if err := net.startWithSnapshots(n.Node.Config.ID, n.Snapshots); err != nil { return err } } for _, conn := range snap.Conns { - if !self.GetNode(conn.One).Up || !self.GetNode(conn.Other).Up { + if !net.GetNode(conn.One).Up || !net.GetNode(conn.Other).Up { //in this case, at least one of the nodes of a connection is not up, //so it would result in the snapshot `Load` to fail continue } - if err := self.Connect(conn.One, conn.Other); err != nil { + if err := net.Connect(conn.One, conn.Other); err != nil { return err } } @@ -692,7 +692,7 @@ func (self *Network) Load(snap *Snapshot) error { } // Subscribe reads control events from a channel and executes them -func (self *Network) Subscribe(events chan *Event) { +func (net *Network) Subscribe(events chan *Event) { for { select { case event, ok := <-events: @@ -700,23 +700,23 @@ func (self *Network) Subscribe(events chan *Event) { return } if event.Control { - self.executeControlEvent(event) + net.executeControlEvent(event) } - case <-self.quitc: + case <-net.quitc: return } } } -func (self *Network) executeControlEvent(event *Event) { +func (net *Network) executeControlEvent(event *Event) { log.Trace("execute control event", "type", event.Type, "event", event) switch event.Type { case EventTypeNode: - if err := self.executeNodeEvent(event); err != nil { + if err := net.executeNodeEvent(event); err != nil { log.Error("error executing node event", "event", event, "err", err) } case EventTypeConn: - if err := self.executeConnEvent(event); err != nil { + if err := net.executeConnEvent(event); err != nil { log.Error("error executing conn event", "event", event, "err", err) } case EventTypeMsg: @@ -724,21 +724,21 @@ func (self *Network) executeControlEvent(event *Event) { } } -func (self *Network) executeNodeEvent(e *Event) error { +func (net *Network) executeNodeEvent(e *Event) error { if !e.Node.Up { - return self.Stop(e.Node.ID()) + return net.Stop(e.Node.ID()) } - if _, err := self.NewNodeWithConfig(e.Node.Config); err != nil { + if _, err := net.NewNodeWithConfig(e.Node.Config); err != nil { return err } - return self.Start(e.Node.ID()) + return net.Start(e.Node.ID()) } -func (self *Network) executeConnEvent(e *Event) error { +func (net *Network) executeConnEvent(e *Event) error { if e.Conn.Up { - return self.Connect(e.Conn.One, e.Conn.Other) + return net.Connect(e.Conn.One, e.Conn.Other) } else { - return self.Disconnect(e.Conn.One, e.Conn.Other) + return net.Disconnect(e.Conn.One, e.Conn.Other) } } diff --git a/p2p/testing/peerpool.go b/p2p/testing/peerpool.go index 45c6e61425..ed00396e23 100644 --- a/p2p/testing/peerpool.go +++ b/p2p/testing/peerpool.go @@ -39,29 +39,29 @@ func NewTestPeerPool() *TestPeerPool { return &TestPeerPool{peers: make(map[discover.NodeID]TestPeer)} } -func (self *TestPeerPool) Add(p TestPeer) { - self.lock.Lock() - defer self.lock.Unlock() - log.Trace(fmt.Sprintf("pp add peer %v", p.ID())) - self.peers[p.ID()] = p +func (p *TestPeerPool) Add(peer TestPeer) { + p.lock.Lock() + defer p.lock.Unlock() + log.Trace(fmt.Sprintf("pp add peer %v", peer.ID())) + p.peers[peer.ID()] = peer } -func (self *TestPeerPool) Remove(p TestPeer) { - self.lock.Lock() - defer self.lock.Unlock() - delete(self.peers, p.ID()) +func (p *TestPeerPool) Remove(peer TestPeer) { + p.lock.Lock() + defer p.lock.Unlock() + delete(p.peers, peer.ID()) } -func (self *TestPeerPool) Has(id discover.NodeID) bool { - self.lock.Lock() - defer self.lock.Unlock() - _, ok := self.peers[id] +func (p *TestPeerPool) Has(id discover.NodeID) bool { + p.lock.Lock() + defer p.lock.Unlock() + _, ok := p.peers[id] return ok } -func (self *TestPeerPool) Get(id discover.NodeID) TestPeer { - self.lock.Lock() - defer self.lock.Unlock() - return self.peers[id] +func (p *TestPeerPool) Get(id discover.NodeID) TestPeer { + p.lock.Lock() + defer p.lock.Unlock() + return p.peers[id] } diff --git a/p2p/testing/protocolsession.go b/p2p/testing/protocolsession.go index 361285f06e..8f73bfa03e 100644 --- a/p2p/testing/protocolsession.go +++ b/p2p/testing/protocolsession.go @@ -78,10 +78,10 @@ type Disconnect struct { } // trigger sends messages from peers -func (self *ProtocolSession) trigger(trig Trigger) error { - simNode, ok := self.adapter.GetNode(trig.Peer) +func (s *ProtocolSession) trigger(trig Trigger) error { + simNode, ok := s.adapter.GetNode(trig.Peer) if !ok { - return fmt.Errorf("trigger: peer %v does not exist (1- %v)", trig.Peer, len(self.IDs)) + return fmt.Errorf("trigger: peer %v does not exist (1- %v)", trig.Peer, len(s.IDs)) } mockNode, ok := simNode.Services()[0].(*mockNode) if !ok { @@ -107,7 +107,7 @@ func (self *ProtocolSession) trigger(trig Trigger) error { } // expect checks an expectation of a message sent out by the pivot node -func (self *ProtocolSession) expect(exps []Expect) error { +func (s *ProtocolSession) expect(exps []Expect) error { // construct a map of expectations for each node peerExpects := make(map[discover.NodeID][]Expect) for _, exp := range exps { @@ -120,9 +120,9 @@ func (self *ProtocolSession) expect(exps []Expect) error { // construct a map of mockNodes for each node mockNodes := make(map[discover.NodeID]*mockNode) for nodeID := range peerExpects { - simNode, ok := self.adapter.GetNode(nodeID) + simNode, ok := s.adapter.GetNode(nodeID) if !ok { - return fmt.Errorf("trigger: peer %v does not exist (1- %v)", nodeID, len(self.IDs)) + return fmt.Errorf("trigger: peer %v does not exist (1- %v)", nodeID, len(s.IDs)) } mockNode, ok := simNode.Services()[0].(*mockNode) if !ok { @@ -202,9 +202,9 @@ func (self *ProtocolSession) expect(exps []Expect) error { } // TestExchanges tests a series of exchanges against the session -func (self *ProtocolSession) TestExchanges(exchanges ...Exchange) error { +func (s *ProtocolSession) TestExchanges(exchanges ...Exchange) error { for i, e := range exchanges { - if err := self.testExchange(e); err != nil { + if err := s.testExchange(e); err != nil { return fmt.Errorf("exchange #%d %q: %v", i, e.Label, err) } log.Trace(fmt.Sprintf("exchange #%d %q: run successfully", i, e.Label)) @@ -214,14 +214,14 @@ func (self *ProtocolSession) TestExchanges(exchanges ...Exchange) error { // testExchange tests a single Exchange. // Default timeout value is 2 seconds. -func (self *ProtocolSession) testExchange(e Exchange) error { +func (s *ProtocolSession) testExchange(e Exchange) error { errc := make(chan error) done := make(chan struct{}) defer close(done) go func() { for _, trig := range e.Triggers { - err := self.trigger(trig) + err := s.trigger(trig) if err != nil { errc <- err return @@ -229,7 +229,7 @@ func (self *ProtocolSession) testExchange(e Exchange) error { } select { - case errc <- self.expect(e.Expects): + case errc <- s.expect(e.Expects): case <-done: } }() @@ -250,7 +250,7 @@ func (self *ProtocolSession) testExchange(e Exchange) error { // TestDisconnected tests the disconnections given as arguments // the disconnect structs describe what disconnect error is expected on which peer -func (self *ProtocolSession) TestDisconnected(disconnects ...*Disconnect) error { +func (s *ProtocolSession) TestDisconnected(disconnects ...*Disconnect) error { expects := make(map[discover.NodeID]error) for _, disconnect := range disconnects { expects[disconnect.Peer] = disconnect.Error @@ -259,7 +259,7 @@ func (self *ProtocolSession) TestDisconnected(disconnects ...*Disconnect) error timeout := time.After(time.Second) for len(expects) > 0 { select { - case event := <-self.events: + case event := <-s.events: if event.Type != p2p.PeerEventTypeDrop { continue } diff --git a/p2p/testing/protocoltester.go b/p2p/testing/protocoltester.go index a797412d60..636613c57a 100644 --- a/p2p/testing/protocoltester.go +++ b/p2p/testing/protocoltester.go @@ -101,24 +101,24 @@ func NewProtocolTester(t *testing.T, id discover.NodeID, n int, run func(*p2p.Pe } // Stop stops the p2p server -func (self *ProtocolTester) Stop() error { - self.Server.Stop() +func (t *ProtocolTester) Stop() error { + t.Server.Stop() return nil } // Connect brings up the remote peer node and connects it using the // p2p/simulations network connection with the in memory network adapter -func (self *ProtocolTester) Connect(selfID discover.NodeID, peers ...*adapters.NodeConfig) { +func (t *ProtocolTester) Connect(selfID discover.NodeID, peers ...*adapters.NodeConfig) { for _, peer := range peers { log.Trace(fmt.Sprintf("start node %v", peer.ID)) - if _, err := self.network.NewNodeWithConfig(peer); err != nil { + if _, err := t.network.NewNodeWithConfig(peer); err != nil { panic(fmt.Sprintf("error starting peer %v: %v", peer.ID, err)) } - if err := self.network.Start(peer.ID); err != nil { + if err := t.network.Start(peer.ID); err != nil { panic(fmt.Sprintf("error starting peer %v: %v", peer.ID, err)) } log.Trace(fmt.Sprintf("connect to %v", peer.ID)) - if err := self.network.Connect(selfID, peer.ID); err != nil { + if err := t.network.Connect(selfID, peer.ID); err != nil { panic(fmt.Sprintf("error connecting to peer %v: %v", peer.ID, err)) } } diff --git a/params/version.go b/params/version.go index fae1d5a88f..8b96f25167 100644 --- a/params/version.go +++ b/params/version.go @@ -23,7 +23,7 @@ import ( const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 8 // Minor version component of the current release - VersionPatch = 7 // Patch version component of the current release + VersionPatch = 8 // Patch version component of the current release VersionMeta = "unstable" // Version metadata to append to the version string ) diff --git a/rlp/decode.go b/rlp/decode.go index 60d9dab2b5..dbbe599597 100644 --- a/rlp/decode.go +++ b/rlp/decode.go @@ -29,6 +29,23 @@ import ( ) var ( + // EOL is returned when the end of the current list + // has been reached during streaming. + EOL = errors.New("rlp: end of list") + + // Actual Errors + ErrExpectedString = errors.New("rlp: expected String or Byte") + ErrExpectedList = errors.New("rlp: expected List") + ErrCanonInt = errors.New("rlp: non-canonical integer format") + ErrCanonSize = errors.New("rlp: non-canonical size information") + ErrElemTooLarge = errors.New("rlp: element is larger than containing list") + ErrValueTooLarge = errors.New("rlp: value size exceeds available input length") + ErrMoreThanOneValue = errors.New("rlp: input contains more than one value") + + // internal errors + errNotInList = errors.New("rlp: call of ListEnd outside of any list") + errNotAtEOL = errors.New("rlp: call of ListEnd not positioned at EOL") + errUintOverflow = errors.New("rlp: uint overflow") errNoPointer = errors.New("rlp: interface given to Decode must be a pointer") errDecodeIntoNil = errors.New("rlp: pointer given to Decode must not be nil") ) @@ -274,9 +291,8 @@ func makeListDecoder(typ reflect.Type, tag tags) (decoder, error) { if etype.Kind() == reflect.Uint8 && !reflect.PtrTo(etype).Implements(decoderInterface) { if typ.Kind() == reflect.Array { return decodeByteArray, nil - } else { - return decodeByteSlice, nil } + return decodeByteSlice, nil } etypeinfo, err := cachedTypeInfo1(etype, tags{}) if err != nil { @@ -555,29 +571,6 @@ func (k Kind) String() string { } } -var ( - // EOL is returned when the end of the current list - // has been reached during streaming. - EOL = errors.New("rlp: end of list") - - // Actual Errors - ErrExpectedString = errors.New("rlp: expected String or Byte") - ErrExpectedList = errors.New("rlp: expected List") - ErrCanonInt = errors.New("rlp: non-canonical integer format") - ErrCanonSize = errors.New("rlp: non-canonical size information") - ErrElemTooLarge = errors.New("rlp: element is larger than containing list") - ErrValueTooLarge = errors.New("rlp: value size exceeds available input length") - - // This error is reported by DecodeBytes if the slice contains - // additional data after the first RLP value. - ErrMoreThanOneValue = errors.New("rlp: input contains more than one value") - - // internal errors - errNotInList = errors.New("rlp: call of ListEnd outside of any list") - errNotAtEOL = errors.New("rlp: call of ListEnd not positioned at EOL") - errUintOverflow = errors.New("rlp: uint overflow") -) - // ByteReader must be implemented by any input reader for a Stream. It // is implemented by e.g. bufio.Reader and bytes.Reader. type ByteReader interface { diff --git a/rlp/encode.go b/rlp/encode.go index 44592c2f53..445b4b5b21 100644 --- a/rlp/encode.go +++ b/rlp/encode.go @@ -92,7 +92,7 @@ func Encode(w io.Writer, val interface{}) error { return eb.toWriter(w) } -// EncodeBytes returns the RLP encoding of val. +// EncodeToBytes returns the RLP encoding of val. // Please see the documentation of Encode for the encoding rules. func EncodeToBytes(val interface{}) ([]byte, error) { eb := encbufPool.Get().(*encbuf) @@ -104,7 +104,7 @@ func EncodeToBytes(val interface{}) ([]byte, error) { return eb.toBytes(), nil } -// EncodeReader returns a reader from which the RLP encoding of val +// EncodeToReader returns a reader from which the RLP encoding of val // can be read. The returned size is the total size of the encoded // data. // @@ -151,11 +151,10 @@ func puthead(buf []byte, smalltag, largetag byte, size uint64) int { if size < 56 { buf[0] = smalltag + byte(size) return 1 - } else { - sizesize := putint(buf[1:], size) - buf[0] = largetag + byte(sizesize) - return sizesize + 1 } + sizesize := putint(buf[1:], size) + buf[0] = largetag + byte(sizesize) + return sizesize + 1 } // encbufs are pooled. @@ -218,7 +217,7 @@ func (w *encbuf) list() *listhead { func (w *encbuf) listEnd(lh *listhead) { lh.size = w.size() - lh.offset - lh.size if lh.size < 56 { - w.lhsize += 1 // length encoded into kind tag + w.lhsize++ // length encoded into kind tag } else { w.lhsize += 1 + intsize(uint64(lh.size)) } @@ -322,10 +321,9 @@ func (r *encReader) next() []byte { p := r.buf.str[r.strpos:head.offset] r.strpos += sizebefore return p - } else { - r.lhpos++ - return head.encode(r.buf.sizebuf) } + r.lhpos++ + return head.encode(r.buf.sizebuf) case r.strpos < len(r.buf.str): // String data at the end, after all list headers. @@ -576,9 +574,8 @@ func makePtrWriter(typ reflect.Type) (writer, error) { writer := func(val reflect.Value, w *encbuf) error { if val.IsNil() { return nilfunc(w) - } else { - return etypeinfo.writer(val.Elem(), w) } + return etypeinfo.writer(val.Elem(), w) } return writer, err } diff --git a/rpc/http.go b/rpc/http.go index 0c2e060fb4..feaa7348c4 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -191,7 +191,7 @@ func (srv *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { defer codec.Close() w.Header().Set("content-type", contentType) - srv.ServeSingleRequest(codec, OptionMethodInvocation, ctx) + srv.ServeSingleRequest(ctx, codec, OptionMethodInvocation) } // validateRequest returns a non-zero response code and error message if the diff --git a/rpc/inproc.go b/rpc/inproc.go index 595a7ca651..cbe65d10e7 100644 --- a/rpc/inproc.go +++ b/rpc/inproc.go @@ -21,7 +21,7 @@ import ( "net" ) -// NewInProcClient attaches an in-process connection to the given RPC server. +// DialInProc attaches an in-process connection to the given RPC server. func DialInProc(handler *Server) *Client { initctx := context.Background() c, _ := newClient(initctx, func(context.Context) (net.Conn, error) { diff --git a/rpc/server.go b/rpc/server.go index 3ca4bcf708..f469f63937 100644 --- a/rpc/server.go +++ b/rpc/server.go @@ -125,7 +125,7 @@ func (s *Server) RegisterName(name string, rcvr interface{}) error { // If singleShot is true it will process a single request, otherwise it will handle // requests until the codec returns an error when reading a request (in most cases // an EOF). It executes requests in parallel when singleShot is false. -func (s *Server) serveRequest(codec ServerCodec, singleShot bool, options CodecOption, ctx context.Context) error { +func (s *Server) serveRequest(ctx context.Context, codec ServerCodec, singleShot bool, options CodecOption) error { var pend sync.WaitGroup defer func() { @@ -216,14 +216,14 @@ func (s *Server) serveRequest(codec ServerCodec, singleShot bool, options CodecO // stopped. In either case the codec is closed. func (s *Server) ServeCodec(codec ServerCodec, options CodecOption) { defer codec.Close() - s.serveRequest(codec, false, options, context.Background()) + s.serveRequest(context.Background(), codec, false, options) } // ServeSingleRequest reads and processes a single RPC request from the given codec. It will not // close the codec unless a non-recoverable error has occurred. Note, this method will return after // a single request has been processed! -func (s *Server) ServeSingleRequest(codec ServerCodec, options CodecOption, ctx context.Context) { - s.serveRequest(codec, true, options, ctx) +func (s *Server) ServeSingleRequest(ctx context.Context, codec ServerCodec, options CodecOption) { + s.serveRequest(ctx, codec, true, options) } // Stop will stop reading new requests, wait for stopPendingRequestTimeout to allow pending requests to finish, diff --git a/signer/core/abihelper.go b/signer/core/abihelper.go index 2674c7346a..1d4fbc7dc2 100644 --- a/signer/core/abihelper.go +++ b/signer/core/abihelper.go @@ -94,13 +94,12 @@ func parseCallData(calldata []byte, abidata string) (*decodedCallData, error) { for n, argument := range method.Inputs { if err != nil { return nil, fmt.Errorf("Failed to decode argument %d (signature %v): %v", n, method.Sig(), err) - } else { - decodedArg := decodedArgument{ - soltype: argument, - value: v[n], - } - decoded.inputs = append(decoded.inputs, decodedArg) } + decodedArg := decodedArgument{ + soltype: argument, + value: v[n], + } + decoded.inputs = append(decoded.inputs, decodedArg) } // We're finished decoding the data. At this point, we encode the decoded data to see if it matches with the @@ -240,7 +239,7 @@ func (db *AbiDb) saveCustomAbi(selector, signature string) error { return err } -// Adds a signature to the database, if custom database saving is enabled. +// AddSignature to the database, if custom database saving is enabled. // OBS: This method does _not_ validate the correctness of the data, // it is assumed that the caller has already done so func (db *AbiDb) AddSignature(selector string, data []byte) error { diff --git a/signer/core/api.go b/signer/core/api.go index 1387041cc3..45933284bf 100644 --- a/signer/core/api.go +++ b/signer/core/api.go @@ -474,7 +474,7 @@ func (api *SignerAPI) Export(ctx context.Context, addr common.Address) (json.Raw return ioutil.ReadFile(wallet.URL().Path) } -// Imports tries to import the given keyJSON in the local keystore. The keyJSON data is expected to be +// Import tries to import the given keyJSON in the local keystore. The keyJSON data is expected to be // in web3 keystore format. It will decrypt the keyJSON with the given passphrase and on successful // decryption it will encrypt the key with the given newPassphrase and store it in the keystore. func (api *SignerAPI) Import(ctx context.Context, keyJSON json.RawMessage) (Account, error) { diff --git a/signer/core/cliui.go b/signer/core/cliui.go index 0d9b5f3d36..2f969669c2 100644 --- a/signer/core/cliui.go +++ b/signer/core/cliui.go @@ -13,6 +13,7 @@ // // You should have received a copy of the GNU General Public License // along with go-ethereum. If not, see . + package core import ( diff --git a/signer/core/types.go b/signer/core/types.go index 8386bd44e7..2acc0a4f4a 100644 --- a/signer/core/types.go +++ b/signer/core/types.go @@ -73,8 +73,8 @@ type SendTxArgs struct { Input *hexutil.Bytes `json:"input"` } -func (t SendTxArgs) String() string { - s, err := json.Marshal(t) +func (args SendTxArgs) String() string { + s, err := json.Marshal(args) if err == nil { return string(s) } diff --git a/signer/core/validation.go b/signer/core/validation.go index 97bb3b6850..288456df87 100644 --- a/signer/core/validation.go +++ b/signer/core/validation.go @@ -128,7 +128,7 @@ func (v *Validator) validate(msgs *ValidationMessages, txargs *SendTxArgs, metho if len(data) == 0 { if txargs.Value.ToInt().Cmp(big.NewInt(0)) > 0 { // Sending ether into black hole - return errors.New(`Tx will create contract with value but empty code!`) + return errors.New("Tx will create contract with value but empty code!") } // No value submitted at least msgs.crit("Tx will create contract with empty code!") diff --git a/signer/rules/rules.go b/signer/rules/rules.go index fa270436df..711e2dddec 100644 --- a/signer/rules/rules.go +++ b/signer/rules/rules.go @@ -46,17 +46,17 @@ func consoleOutput(call otto.FunctionCall) otto.Value { return otto.Value{} } -// rulesetUi provides an implementation of SignerUI that evaluates a javascript +// rulesetUI provides an implementation of SignerUI that evaluates a javascript // file for each defined UI-method -type rulesetUi struct { +type rulesetUI struct { next core.SignerUI // The next handler, for manual processing storage storage.Storage credentials storage.Storage jsRules string // The rules to use } -func NewRuleEvaluator(next core.SignerUI, jsbackend, credentialsBackend storage.Storage) (*rulesetUi, error) { - c := &rulesetUi{ +func NewRuleEvaluator(next core.SignerUI, jsbackend, credentialsBackend storage.Storage) (*rulesetUI, error) { + c := &rulesetUI{ next: next, storage: jsbackend, credentials: credentialsBackend, @@ -66,11 +66,11 @@ func NewRuleEvaluator(next core.SignerUI, jsbackend, credentialsBackend storage. return c, nil } -func (r *rulesetUi) Init(javascriptRules string) error { +func (r *rulesetUI) Init(javascriptRules string) error { r.jsRules = javascriptRules return nil } -func (r *rulesetUi) execute(jsfunc string, jsarg interface{}) (otto.Value, error) { +func (r *rulesetUI) execute(jsfunc string, jsarg interface{}) (otto.Value, error) { // Instantiate a fresh vm engine every time vm := otto.New() @@ -115,7 +115,7 @@ func (r *rulesetUi) execute(jsfunc string, jsarg interface{}) (otto.Value, error return vm.Run(call) } -func (r *rulesetUi) checkApproval(jsfunc string, jsarg []byte, err error) (bool, error) { +func (r *rulesetUI) checkApproval(jsfunc string, jsarg []byte, err error) (bool, error) { if err != nil { return false, err } @@ -139,7 +139,7 @@ func (r *rulesetUi) checkApproval(jsfunc string, jsarg []byte, err error) (bool, return false, fmt.Errorf("Unknown response") } -func (r *rulesetUi) ApproveTx(request *core.SignTxRequest) (core.SignTxResponse, error) { +func (r *rulesetUI) ApproveTx(request *core.SignTxRequest) (core.SignTxResponse, error) { jsonreq, err := json.Marshal(request) approved, err := r.checkApproval("ApproveTx", jsonreq, err) if err != nil { @@ -158,11 +158,11 @@ func (r *rulesetUi) ApproveTx(request *core.SignTxRequest) (core.SignTxResponse, return core.SignTxResponse{Approved: false}, err } -func (r *rulesetUi) lookupPassword(address common.Address) string { +func (r *rulesetUI) lookupPassword(address common.Address) string { return r.credentials.Get(strings.ToLower(address.String())) } -func (r *rulesetUi) ApproveSignData(request *core.SignDataRequest) (core.SignDataResponse, error) { +func (r *rulesetUI) ApproveSignData(request *core.SignDataRequest) (core.SignDataResponse, error) { jsonreq, err := json.Marshal(request) approved, err := r.checkApproval("ApproveSignData", jsonreq, err) if err != nil { @@ -175,7 +175,7 @@ func (r *rulesetUi) ApproveSignData(request *core.SignDataRequest) (core.SignDat return core.SignDataResponse{Approved: false, Password: ""}, err } -func (r *rulesetUi) ApproveExport(request *core.ExportRequest) (core.ExportResponse, error) { +func (r *rulesetUI) ApproveExport(request *core.ExportRequest) (core.ExportResponse, error) { jsonreq, err := json.Marshal(request) approved, err := r.checkApproval("ApproveExport", jsonreq, err) if err != nil { @@ -188,13 +188,13 @@ func (r *rulesetUi) ApproveExport(request *core.ExportRequest) (core.ExportRespo return core.ExportResponse{Approved: false}, err } -func (r *rulesetUi) ApproveImport(request *core.ImportRequest) (core.ImportResponse, error) { +func (r *rulesetUI) ApproveImport(request *core.ImportRequest) (core.ImportResponse, error) { // This cannot be handled by rules, requires setting a password // dispatch to next return r.next.ApproveImport(request) } -func (r *rulesetUi) ApproveListing(request *core.ListRequest) (core.ListResponse, error) { +func (r *rulesetUI) ApproveListing(request *core.ListRequest) (core.ListResponse, error) { jsonreq, err := json.Marshal(request) approved, err := r.checkApproval("ApproveListing", jsonreq, err) if err != nil { @@ -207,22 +207,22 @@ func (r *rulesetUi) ApproveListing(request *core.ListRequest) (core.ListResponse return core.ListResponse{}, err } -func (r *rulesetUi) ApproveNewAccount(request *core.NewAccountRequest) (core.NewAccountResponse, error) { +func (r *rulesetUI) ApproveNewAccount(request *core.NewAccountRequest) (core.NewAccountResponse, error) { // This cannot be handled by rules, requires setting a password // dispatch to next return r.next.ApproveNewAccount(request) } -func (r *rulesetUi) ShowError(message string) { +func (r *rulesetUI) ShowError(message string) { log.Error(message) r.next.ShowError(message) } -func (r *rulesetUi) ShowInfo(message string) { +func (r *rulesetUI) ShowInfo(message string) { log.Info(message) r.next.ShowInfo(message) } -func (r *rulesetUi) OnSignerStartup(info core.StartupInfo) { +func (r *rulesetUI) OnSignerStartup(info core.StartupInfo) { jsonInfo, err := json.Marshal(info) if err != nil { log.Warn("failed marshalling data", "data", info) @@ -235,7 +235,7 @@ func (r *rulesetUi) OnSignerStartup(info core.StartupInfo) { } } -func (r *rulesetUi) OnApprovedTx(tx ethapi.SignTransactionResult) { +func (r *rulesetUI) OnApprovedTx(tx ethapi.SignTransactionResult) { jsonTx, err := json.Marshal(tx) if err != nil { log.Warn("failed marshalling transaction", "tx", tx) diff --git a/signer/rules/rules_test.go b/signer/rules/rules_test.go index 6a04035004..b6060eba73 100644 --- a/signer/rules/rules_test.go +++ b/signer/rules/rules_test.go @@ -33,18 +33,18 @@ import ( const JS = ` /** -This is an example implementation of a Javascript rule file. +This is an example implementation of a Javascript rule file. -When the signer receives a request over the external API, the corresponding method is evaluated. -Three things can happen: +When the signer receives a request over the external API, the corresponding method is evaluated. +Three things can happen: -1. The method returns "Approve". This means the operation is permitted. -2. The method returns "Reject". This means the operation is rejected. +1. The method returns "Approve". This means the operation is permitted. +2. The method returns "Reject". This means the operation is rejected. 3. Anything else; other return values [*], method not implemented or exception occurred during processing. This means -that the operation will continue to manual processing, via the regular UI method chosen by the user. +that the operation will continue to manual processing, via the regular UI method chosen by the user. -[*] Note: Future version of the ruleset may use more complex json-based returnvalues, making it possible to not -only respond Approve/Reject/Manual, but also modify responses. For example, choose to list only one, but not all +[*] Note: Future version of the ruleset may use more complex json-based returnvalues, making it possible to not +only respond Approve/Reject/Manual, but also modify responses. For example, choose to list only one, but not all accounts in a list-request. The points above will continue to hold for non-json based responses ("Approve"/"Reject"). **/ @@ -72,49 +72,49 @@ func mixAddr(a string) (*common.MixedcaseAddress, error) { return common.NewMixedcaseAddressFromString(a) } -type alwaysDenyUi struct{} +type alwaysDenyUI struct{} -func (alwaysDenyUi) OnSignerStartup(info core.StartupInfo) { +func (alwaysDenyUI) OnSignerStartup(info core.StartupInfo) { } -func (alwaysDenyUi) ApproveTx(request *core.SignTxRequest) (core.SignTxResponse, error) { +func (alwaysDenyUI) ApproveTx(request *core.SignTxRequest) (core.SignTxResponse, error) { return core.SignTxResponse{Transaction: request.Transaction, Approved: false, Password: ""}, nil } -func (alwaysDenyUi) ApproveSignData(request *core.SignDataRequest) (core.SignDataResponse, error) { +func (alwaysDenyUI) ApproveSignData(request *core.SignDataRequest) (core.SignDataResponse, error) { return core.SignDataResponse{Approved: false, Password: ""}, nil } -func (alwaysDenyUi) ApproveExport(request *core.ExportRequest) (core.ExportResponse, error) { +func (alwaysDenyUI) ApproveExport(request *core.ExportRequest) (core.ExportResponse, error) { return core.ExportResponse{Approved: false}, nil } -func (alwaysDenyUi) ApproveImport(request *core.ImportRequest) (core.ImportResponse, error) { +func (alwaysDenyUI) ApproveImport(request *core.ImportRequest) (core.ImportResponse, error) { return core.ImportResponse{Approved: false, OldPassword: "", NewPassword: ""}, nil } -func (alwaysDenyUi) ApproveListing(request *core.ListRequest) (core.ListResponse, error) { +func (alwaysDenyUI) ApproveListing(request *core.ListRequest) (core.ListResponse, error) { return core.ListResponse{Accounts: nil}, nil } -func (alwaysDenyUi) ApproveNewAccount(request *core.NewAccountRequest) (core.NewAccountResponse, error) { +func (alwaysDenyUI) ApproveNewAccount(request *core.NewAccountRequest) (core.NewAccountResponse, error) { return core.NewAccountResponse{Approved: false, Password: ""}, nil } -func (alwaysDenyUi) ShowError(message string) { +func (alwaysDenyUI) ShowError(message string) { panic("implement me") } -func (alwaysDenyUi) ShowInfo(message string) { +func (alwaysDenyUI) ShowInfo(message string) { panic("implement me") } -func (alwaysDenyUi) OnApprovedTx(tx ethapi.SignTransactionResult) { +func (alwaysDenyUI) OnApprovedTx(tx ethapi.SignTransactionResult) { panic("implement me") } -func initRuleEngine(js string) (*rulesetUi, error) { - r, err := NewRuleEvaluator(&alwaysDenyUi{}, storage.NewEphemeralStorage(), storage.NewEphemeralStorage()) +func initRuleEngine(js string) (*rulesetUI, error) { + r, err := NewRuleEvaluator(&alwaysDenyUI{}, storage.NewEphemeralStorage(), storage.NewEphemeralStorage()) if err != nil { return nil, fmt.Errorf("failed to create js engine: %v", err) } @@ -196,59 +196,59 @@ func TestSignTxRequest(t *testing.T) { } } -type dummyUi struct { +type dummyUI struct { calls []string } -func (d *dummyUi) ApproveTx(request *core.SignTxRequest) (core.SignTxResponse, error) { +func (d *dummyUI) ApproveTx(request *core.SignTxRequest) (core.SignTxResponse, error) { d.calls = append(d.calls, "ApproveTx") return core.SignTxResponse{}, core.ErrRequestDenied } -func (d *dummyUi) ApproveSignData(request *core.SignDataRequest) (core.SignDataResponse, error) { +func (d *dummyUI) ApproveSignData(request *core.SignDataRequest) (core.SignDataResponse, error) { d.calls = append(d.calls, "ApproveSignData") return core.SignDataResponse{}, core.ErrRequestDenied } -func (d *dummyUi) ApproveExport(request *core.ExportRequest) (core.ExportResponse, error) { +func (d *dummyUI) ApproveExport(request *core.ExportRequest) (core.ExportResponse, error) { d.calls = append(d.calls, "ApproveExport") return core.ExportResponse{}, core.ErrRequestDenied } -func (d *dummyUi) ApproveImport(request *core.ImportRequest) (core.ImportResponse, error) { +func (d *dummyUI) ApproveImport(request *core.ImportRequest) (core.ImportResponse, error) { d.calls = append(d.calls, "ApproveImport") return core.ImportResponse{}, core.ErrRequestDenied } -func (d *dummyUi) ApproveListing(request *core.ListRequest) (core.ListResponse, error) { +func (d *dummyUI) ApproveListing(request *core.ListRequest) (core.ListResponse, error) { d.calls = append(d.calls, "ApproveListing") return core.ListResponse{}, core.ErrRequestDenied } -func (d *dummyUi) ApproveNewAccount(request *core.NewAccountRequest) (core.NewAccountResponse, error) { +func (d *dummyUI) ApproveNewAccount(request *core.NewAccountRequest) (core.NewAccountResponse, error) { d.calls = append(d.calls, "ApproveNewAccount") return core.NewAccountResponse{}, core.ErrRequestDenied } -func (d *dummyUi) ShowError(message string) { +func (d *dummyUI) ShowError(message string) { d.calls = append(d.calls, "ShowError") } -func (d *dummyUi) ShowInfo(message string) { +func (d *dummyUI) ShowInfo(message string) { d.calls = append(d.calls, "ShowInfo") } -func (d *dummyUi) OnApprovedTx(tx ethapi.SignTransactionResult) { +func (d *dummyUI) OnApprovedTx(tx ethapi.SignTransactionResult) { d.calls = append(d.calls, "OnApprovedTx") } -func (d *dummyUi) OnSignerStartup(info core.StartupInfo) { +func (d *dummyUI) OnSignerStartup(info core.StartupInfo) { } //TestForwarding tests that the rule-engine correctly dispatches requests to the next caller func TestForwarding(t *testing.T) { js := "" - ui := &dummyUi{make([]string, 0)} + ui := &dummyUI{make([]string, 0)} jsBackend := storage.NewEphemeralStorage() credBackend := storage.NewEphemeralStorage() r, err := NewRuleEvaluator(ui, jsBackend, credBackend) @@ -308,22 +308,22 @@ func TestStorage(t *testing.T) { function testStorage(){ storage.Put("mykey", "myvalue") a = storage.Get("mykey") - + storage.Put("mykey", ["a", "list"]) // Should result in "a,list" a += storage.Get("mykey") - + storage.Put("mykey", {"an": "object"}) // Should result in "[object Object]" a += storage.Get("mykey") - + storage.Put("mykey", JSON.stringify({"an": "object"})) // Should result in '{"an":"object"}' a += storage.Get("mykey") a += storage.Get("missingkey") //Missing keys should result in empty string storage.Put("","missing key==noop") // Can't store with 0-length key a += storage.Get("") // Should result in '' - + var b = new BigNumber(2) var c = new BigNumber(16)//"0xf0",16) var d = b.plus(c) @@ -361,7 +361,7 @@ const ExampleTxWindow = ` if(str.slice(0,2) == "0x"){ return new BigNumber(str.slice(2),16)} return new BigNumber(str) } - + // Time window: 1 week var window = 1000* 3600*24*7; @@ -370,7 +370,7 @@ const ExampleTxWindow = ` function isLimitOk(transaction){ var value = big(transaction.value) - // Start of our window function + // Start of our window function var windowstart = new Date().getTime() - window; var txs = []; @@ -382,17 +382,17 @@ const ExampleTxWindow = ` // First, remove all that have passed out of the time-window var newtxs = txs.filter(function(tx){return tx.tstamp > windowstart}); console.log(txs, newtxs.length); - + // Secondly, aggregate the current sum sum = new BigNumber(0) sum = newtxs.reduce(function(agg, tx){ return big(tx.value).plus(agg)}, sum); console.log("ApproveTx > Sum so far", sum); console.log("ApproveTx > Requested", value.toNumber()); - + // Would we exceed weekly limit ? return sum.plus(value).lt(limit) - + } function ApproveTx(r){ console.log(r) @@ -405,14 +405,14 @@ const ExampleTxWindow = ` /** * OnApprovedTx(str) is called when a transaction has been approved and signed. The parameter - * 'response_str' contains the return value that will be sent to the external caller. - * The return value from this method is ignore - the reason for having this callback is to allow the - * ruleset to keep track of approved transactions. + * 'response_str' contains the return value that will be sent to the external caller. + * The return value from this method is ignore - the reason for having this callback is to allow the + * ruleset to keep track of approved transactions. * - * When implementing rate-limited rules, this callback should be used. + * When implementing rate-limited rules, this callback should be used. * If a rule responds with neither 'Approve' nor 'Reject' - the tx goes to manual processing. If the user * then accepts the transaction, this method will be called. - * + * * TLDR; Use this method to keep track of signed transactions, instead of using the data in ApproveTx. */ function OnApprovedTx(resp){ diff --git a/signer/storage/aes_gcm_storage.go b/signer/storage/aes_gcm_storage.go index 1ac3475588..225276667f 100644 --- a/signer/storage/aes_gcm_storage.go +++ b/signer/storage/aes_gcm_storage.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU General Public License // along with go-ethereum. If not, see . // + package storage import ( @@ -106,10 +107,8 @@ func (s *AESEncryptedStorage) readEncryptedStorage() (map[string]storedCredentia if os.IsNotExist(err) { // Doesn't exist yet return creds, nil - - } else { - log.Warn("Failed to read encrypted storage", "err", err, "file", s.filename) } + log.Warn("Failed to read encrypted storage", "err", err, "file", s.filename) } if err = json.Unmarshal(raw, &creds); err != nil { log.Warn("Failed to unmarshal encrypted storage", "err", err, "file", s.filename) diff --git a/tests/block_test_util.go b/tests/block_test_util.go index beba484833..a72799f6e2 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -98,13 +98,13 @@ func (t *BlockTest) Run() error { } // import pre accounts & construct test genesis block & state root - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() gblock, err := t.genesis(config).Commit(db) if err != nil { return err } if gblock.Hash() != t.json.Genesis.Hash { - return fmt.Errorf("genesis block hash doesn't match test: computed=%x, test=%x\n", gblock.Hash().Bytes()[:6], t.json.Genesis.Hash[:6]) + return fmt.Errorf("genesis block hash doesn't match test: computed=%x, test=%x", gblock.Hash().Bytes()[:6], t.json.Genesis.Hash[:6]) } if gblock.Root() != t.json.Genesis.StateRoot { return fmt.Errorf("genesis block state root does not match test: computed=%x, test=%x", gblock.Root().Bytes()[:6], t.json.Genesis.StateRoot[:6]) diff --git a/tests/init.go b/tests/init.go index ff8ee7da18..0bea5ccd63 100644 --- a/tests/init.go +++ b/tests/init.go @@ -23,7 +23,7 @@ import ( "github.com/ethereum/go-ethereum/params" ) -// This table defines supported forks and their chain config. +// Forks table defines supported forks and their chain config. var Forks = map[string]*params.ChainConfig{ "Frontier": { ChainId: big.NewInt(1), diff --git a/tests/init_test.go b/tests/init_test.go index fbb214b08c..26e919d24b 100644 --- a/tests/init_test.go +++ b/tests/init_test.go @@ -42,7 +42,7 @@ var ( difficultyTestDir = filepath.Join(baseDir, "BasicTests") ) -func readJson(reader io.Reader, value interface{}) error { +func readJSON(reader io.Reader, value interface{}) error { data, err := ioutil.ReadAll(reader) if err != nil { return fmt.Errorf("error reading JSON file: %v", err) @@ -57,14 +57,14 @@ func readJson(reader io.Reader, value interface{}) error { return nil } -func readJsonFile(fn string, value interface{}) error { +func readJSONFile(fn string, value interface{}) error { file, err := os.Open(fn) if err != nil { return err } defer file.Close() - err = readJson(file, value) + err = readJSON(file, value) if err != nil { return fmt.Errorf("%s in file %s", err.Error(), fn) } @@ -169,9 +169,8 @@ func (tm *testMatcher) checkFailure(t *testing.T, name string, err error) error if err != nil { t.Logf("error: %v", err) return nil - } else { - return fmt.Errorf("test succeeded unexpectedly") } + return fmt.Errorf("test succeeded unexpectedly") } return err } @@ -213,7 +212,7 @@ func (tm *testMatcher) runTestFile(t *testing.T, path, name string, runTest inte // Load the file as map[string]. m := makeMapFromTestFunc(runTest) - if err := readJsonFile(path, m.Addr().Interface()); err != nil { + if err := readJSONFile(path, m.Addr().Interface()); err != nil { t.Fatal(err) } diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 3b761bd771..84581fae18 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -126,8 +126,7 @@ func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config) (*state.StateD return nil, UnsupportedForkError{subtest.Fork} } block := t.genesis(config).ToBlock(nil) - db, _ := ethdb.NewMemDatabase() - statedb := MakePreState(db, t.json.Pre) + statedb := MakePreState(ethdb.NewMemDatabase(), t.json.Pre) post := t.json.Post[subtest.Fork][subtest.Index] msg, err := t.json.Tx.toMessage(post) diff --git a/tests/transaction_test_util.go b/tests/transaction_test_util.go index 2028d2a278..8c3dac088c 100644 --- a/tests/transaction_test_util.go +++ b/tests/transaction_test_util.go @@ -72,9 +72,8 @@ func (tt *TransactionTest) Run(config *params.ChainConfig) error { if err := rlp.DecodeBytes(tt.json.RLP, tx); err != nil { if tt.json.Transaction == nil { return nil - } else { - return fmt.Errorf("RLP decoding failed: %v", err) } + return fmt.Errorf("RLP decoding failed: %v", err) } // Check sender derivation. signer := types.MakeSigner(config, new(big.Int).SetUint64(uint64(tt.json.BlockNumber))) diff --git a/tests/vm_test_util.go b/tests/vm_test_util.go index b365167a69..cb81c5b94e 100644 --- a/tests/vm_test_util.go +++ b/tests/vm_test_util.go @@ -79,8 +79,7 @@ type vmExecMarshaling struct { } func (t *VMTest) Run(vmconfig vm.Config) error { - db, _ := ethdb.NewMemDatabase() - statedb := MakePreState(db, t.json.Pre) + statedb := MakePreState(ethdb.NewMemDatabase(), t.json.Pre) ret, gasRemaining, err := t.exec(statedb, vmconfig) if t.json.GasRemaining == nil { diff --git a/trie/iterator.go b/trie/iterator.go index 76146c0d64..3bae8e186b 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -303,7 +303,7 @@ func (it *nodeIterator) push(state *nodeIteratorState, parentIndex *int, path [] it.path = path it.stack = append(it.stack, state) if parentIndex != nil { - *parentIndex += 1 + *parentIndex++ } } @@ -380,7 +380,7 @@ func (it *differenceIterator) Next(bool) bool { if !it.b.Next(true) { return false } - it.count += 1 + it.count++ if it.eof { // a has reached eof, so we just return all elements from b @@ -395,7 +395,7 @@ func (it *differenceIterator) Next(bool) bool { it.eof = true return true } - it.count += 1 + it.count++ case 1: // b is before a return true @@ -405,12 +405,12 @@ func (it *differenceIterator) Next(bool) bool { if !it.b.Next(hasHash) { return false } - it.count += 1 + it.count++ if !it.a.Next(hasHash) { it.eof = true return true } - it.count += 1 + it.count++ } } } @@ -504,14 +504,14 @@ func (it *unionIterator) Next(descend bool) bool { skipped := heap.Pop(it.items).(NodeIterator) // Skip the whole subtree if the nodes have hashes; otherwise just skip this node if skipped.Next(skipped.Hash() == common.Hash{}) { - it.count += 1 + it.count++ // If there are more elements, push the iterator back on the heap heap.Push(it.items, skipped) } } if least.Next(descend) { - it.count += 1 + it.count++ heap.Push(it.items, least) } diff --git a/trie/iterator_test.go b/trie/iterator_test.go index dce1c78b5d..2a510b1c2d 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -289,7 +289,7 @@ func TestIteratorContinueAfterErrorDisk(t *testing.T) { testIteratorContinueA func TestIteratorContinueAfterErrorMemonly(t *testing.T) { testIteratorContinueAfterError(t, true) } func testIteratorContinueAfterError(t *testing.T, memonly bool) { - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) tr, _ := New(common.Hash{}, triedb) @@ -376,7 +376,7 @@ func TestIteratorContinueAfterSeekErrorMemonly(t *testing.T) { func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) { // Commit test trie to db, then remove the node containing "bars". - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) ctr, _ := New(common.Hash{}, triedb) diff --git a/trie/node.go b/trie/node.go index a7697fc0c6..02815042c6 100644 --- a/trie/node.go +++ b/trie/node.go @@ -123,17 +123,17 @@ func decodeNode(hash, buf []byte, cachegen uint16) (node, error) { } switch c, _ := rlp.CountValues(elems); c { case 2: - n, err := decodeShort(hash, buf, elems, cachegen) + n, err := decodeShort(hash, elems, cachegen) return n, wrapError(err, "short") case 17: - n, err := decodeFull(hash, buf, elems, cachegen) + n, err := decodeFull(hash, elems, cachegen) return n, wrapError(err, "full") default: return nil, fmt.Errorf("invalid number of list elements: %v", c) } } -func decodeShort(hash, buf, elems []byte, cachegen uint16) (node, error) { +func decodeShort(hash, elems []byte, cachegen uint16) (node, error) { kbuf, rest, err := rlp.SplitString(elems) if err != nil { return nil, err @@ -155,7 +155,7 @@ func decodeShort(hash, buf, elems []byte, cachegen uint16) (node, error) { return &shortNode{key, r, flag}, nil } -func decodeFull(hash, buf, elems []byte, cachegen uint16) (*fullNode, error) { +func decodeFull(hash, elems []byte, cachegen uint16) (*fullNode, error) { n := &fullNode{flags: nodeFlag{hash: hash, gen: cachegen}} for i := 0; i < 16; i++ { cld, rest, err := decodeRef(elems, cachegen) diff --git a/trie/proof_test.go b/trie/proof_test.go index fff313d7fd..a3537787cc 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -36,7 +36,7 @@ func TestProof(t *testing.T) { trie, vals := randomTrie(500) root := trie.Hash() for _, kv := range vals { - proofs, _ := ethdb.NewMemDatabase() + proofs := ethdb.NewMemDatabase() if trie.Prove(kv.k, 0, proofs) != nil { t.Fatalf("missing key %x while constructing proof", kv.k) } @@ -53,7 +53,7 @@ func TestProof(t *testing.T) { func TestOneElementProof(t *testing.T) { trie := new(Trie) updateString(trie, "k", "v") - proofs, _ := ethdb.NewMemDatabase() + proofs := ethdb.NewMemDatabase() trie.Prove([]byte("k"), 0, proofs) if len(proofs.Keys()) != 1 { t.Error("proof should have one element") @@ -71,7 +71,7 @@ func TestVerifyBadProof(t *testing.T) { trie, vals := randomTrie(800) root := trie.Hash() for _, kv := range vals { - proofs, _ := ethdb.NewMemDatabase() + proofs := ethdb.NewMemDatabase() trie.Prove(kv.k, 0, proofs) if len(proofs.Keys()) == 0 { t.Fatal("zero length proof") @@ -109,7 +109,7 @@ func BenchmarkProve(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { kv := vals[keys[i%len(keys)]] - proofs, _ := ethdb.NewMemDatabase() + proofs := ethdb.NewMemDatabase() if trie.Prove(kv.k, 0, proofs); len(proofs.Keys()) == 0 { b.Fatalf("zero length proof for %x", kv.k) } @@ -123,7 +123,7 @@ func BenchmarkVerifyProof(b *testing.B) { var proofs []*ethdb.MemDatabase for k := range vals { keys = append(keys, k) - proof, _ := ethdb.NewMemDatabase() + proof := ethdb.NewMemDatabase() trie.Prove([]byte(k), 0, proof) proofs = append(proofs, proof) } diff --git a/trie/secure_trie_test.go b/trie/secure_trie_test.go index aedf5a1cde..d16d999684 100644 --- a/trie/secure_trie_test.go +++ b/trie/secure_trie_test.go @@ -28,18 +28,14 @@ import ( ) func newEmptySecure() *SecureTrie { - diskdb, _ := ethdb.NewMemDatabase() - triedb := NewDatabase(diskdb) - - trie, _ := NewSecure(common.Hash{}, triedb, 0) + trie, _ := NewSecure(common.Hash{}, NewDatabase(ethdb.NewMemDatabase()), 0) return trie } // makeTestSecureTrie creates a large enough secure trie for testing. func makeTestSecureTrie() (*Database, *SecureTrie, map[string][]byte) { // Create an empty trie - diskdb, _ := ethdb.NewMemDatabase() - triedb := NewDatabase(diskdb) + triedb := NewDatabase(ethdb.NewMemDatabase()) trie, _ := NewSecure(common.Hash{}, triedb, 0) diff --git a/trie/sync_test.go b/trie/sync_test.go index 4a720612b6..142a6f5b1a 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -27,8 +27,7 @@ import ( // makeTestTrie create a sample test trie to test node-wise reconstruction. func makeTestTrie() (*Database, *Trie, map[string][]byte) { // Create an empty trie - diskdb, _ := ethdb.NewMemDatabase() - triedb := NewDatabase(diskdb) + triedb := NewDatabase(ethdb.NewMemDatabase()) trie, _ := New(common.Hash{}, triedb) // Fill it with some arbitrary data @@ -89,18 +88,13 @@ func checkTrieConsistency(db *Database, root common.Hash) error { // Tests that an empty trie is not scheduled for syncing. func TestEmptyTrieSync(t *testing.T) { - diskdbA, _ := ethdb.NewMemDatabase() - triedbA := NewDatabase(diskdbA) - - diskdbB, _ := ethdb.NewMemDatabase() - triedbB := NewDatabase(diskdbB) - - emptyA, _ := New(common.Hash{}, triedbA) - emptyB, _ := New(emptyRoot, triedbB) + dbA := NewDatabase(ethdb.NewMemDatabase()) + dbB := NewDatabase(ethdb.NewMemDatabase()) + emptyA, _ := New(common.Hash{}, dbA) + emptyB, _ := New(emptyRoot, dbB) for i, trie := range []*Trie{emptyA, emptyB} { - diskdb, _ := ethdb.NewMemDatabase() - if req := NewTrieSync(trie.Hash(), diskdb, nil).Missing(1); len(req) != 0 { + if req := NewTrieSync(trie.Hash(), ethdb.NewMemDatabase(), nil).Missing(1); len(req) != 0 { t.Errorf("test %d: content requested for empty trie: %v", i, req) } } @@ -116,7 +110,7 @@ func testIterativeTrieSync(t *testing.T, batch int) { srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) sched := NewTrieSync(srcTrie.Hash(), diskdb, nil) @@ -149,7 +143,7 @@ func TestIterativeDelayedTrieSync(t *testing.T) { srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) sched := NewTrieSync(srcTrie.Hash(), diskdb, nil) @@ -187,7 +181,7 @@ func testIterativeRandomTrieSync(t *testing.T, batch int) { srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) sched := NewTrieSync(srcTrie.Hash(), diskdb, nil) @@ -228,7 +222,7 @@ func TestIterativeRandomDelayedTrieSync(t *testing.T) { srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) sched := NewTrieSync(srcTrie.Hash(), diskdb, nil) @@ -275,7 +269,7 @@ func TestDuplicateAvoidanceTrieSync(t *testing.T) { srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) sched := NewTrieSync(srcTrie.Hash(), diskdb, nil) @@ -315,7 +309,7 @@ func TestIncompleteTrieSync(t *testing.T) { srcDb, srcTrie, _ := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) sched := NewTrieSync(srcTrie.Hash(), diskdb, nil) diff --git a/trie/trie_test.go b/trie/trie_test.go index 9972226288..f8e5fd12a1 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -43,8 +43,7 @@ func init() { // Used for testing func newEmpty() *Trie { - diskdb, _ := ethdb.NewMemDatabase() - trie, _ := New(common.Hash{}, NewDatabase(diskdb)) + trie, _ := New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase())) return trie } @@ -68,8 +67,7 @@ func TestNull(t *testing.T) { } func TestMissingRoot(t *testing.T) { - diskdb, _ := ethdb.NewMemDatabase() - trie, err := New(common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), NewDatabase(diskdb)) + trie, err := New(common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), NewDatabase(ethdb.NewMemDatabase())) if trie != nil { t.Error("New returned non-nil trie for invalid root") } @@ -82,7 +80,7 @@ func TestMissingNodeDisk(t *testing.T) { testMissingNode(t, false) } func TestMissingNodeMemonly(t *testing.T) { testMissingNode(t, true) } func testMissingNode(t *testing.T, memonly bool) { - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) trie, _ := New(common.Hash{}, triedb) @@ -413,8 +411,7 @@ func (randTest) Generate(r *rand.Rand, size int) reflect.Value { } func runRandTest(rt randTest) bool { - diskdb, _ := ethdb.NewMemDatabase() - triedb := NewDatabase(diskdb) + triedb := NewDatabase(ethdb.NewMemDatabase()) tr, _ := New(common.Hash{}, triedb) values := make(map[string]string) // tracks content of the trie diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/storage/mem_storage.go b/vendor/github.com/syndtr/goleveldb/leveldb/storage/mem_storage.go index 9b0421f035..838f1bee1b 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/storage/mem_storage.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/storage/mem_storage.go @@ -12,7 +12,11 @@ import ( "sync" ) -const typeShift = 3 +const typeShift = 4 + +// Verify at compile-time that typeShift is large enough to cover all FileType +// values by confirming that 0 == 0. +var _ [0]struct{} = [TypeAll >> typeShift]struct{}{} type memStorageLock struct { ms *memStorage @@ -143,7 +147,7 @@ func (ms *memStorage) Remove(fd FileDesc) error { } func (ms *memStorage) Rename(oldfd, newfd FileDesc) error { - if FileDescOk(oldfd) || FileDescOk(newfd) { + if !FileDescOk(oldfd) || !FileDescOk(newfd) { return ErrInvalidFile } if oldfd == newfd { diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/util.go b/vendor/github.com/syndtr/goleveldb/leveldb/util.go index e572a329e9..0e2b519e5c 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/util.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/util.go @@ -20,7 +20,7 @@ func shorten(str string) string { return str[:3] + ".." + str[len(str)-3:] } -var bunits = [...]string{"", "Ki", "Mi", "Gi"} +var bunits = [...]string{"", "Ki", "Mi", "Gi", "Ti"} func shortenb(bytes int) string { i := 0 diff --git a/vendor/vendor.json b/vendor/vendor.json index e083a363ba..fdc7789364 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -418,76 +418,76 @@ "revisionTime": "2017-07-05T02:17:15Z" }, { - "checksumSHA1": "3QsnhPTXGytTbW3uDvQLgSo9s9M=", + "checksumSHA1": "k13cCuMJO7+KhR8ZXx5oUqDKGQA=", "path": "github.com/syndtr/goleveldb/leveldb", - "revision": "169b1b37be738edb2813dab48c97a549bcf99bb5", - "revisionTime": "2018-03-07T11:33:52Z" + "revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a", + "revisionTime": "2018-05-02T07:23:49Z" }, { "checksumSHA1": "EKIow7XkgNdWvR/982ffIZxKG8Y=", "path": "github.com/syndtr/goleveldb/leveldb/cache", - "revision": "169b1b37be738edb2813dab48c97a549bcf99bb5", - "revisionTime": "2018-03-07T11:33:52Z" + "revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a", + "revisionTime": "2018-05-02T07:23:49Z" }, { "checksumSHA1": "5KPgnvCPlR0ysDAqo6jApzRQ3tw=", "path": "github.com/syndtr/goleveldb/leveldb/comparer", - "revision": "169b1b37be738edb2813dab48c97a549bcf99bb5", - "revisionTime": "2018-03-07T11:33:52Z" + "revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a", + "revisionTime": "2018-05-02T07:23:49Z" }, { "checksumSHA1": "1DRAxdlWzS4U0xKN/yQ/fdNN7f0=", "path": "github.com/syndtr/goleveldb/leveldb/errors", - "revision": "169b1b37be738edb2813dab48c97a549bcf99bb5", - "revisionTime": "2018-03-07T11:33:52Z" + "revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a", + "revisionTime": "2018-05-02T07:23:49Z" }, { "checksumSHA1": "eqKeD6DS7eNCtxVYZEHHRKkyZrw=", "path": "github.com/syndtr/goleveldb/leveldb/filter", - "revision": "169b1b37be738edb2813dab48c97a549bcf99bb5", - "revisionTime": "2018-03-07T11:33:52Z" + "revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a", + "revisionTime": "2018-05-02T07:23:49Z" }, { "checksumSHA1": "weSsccMav4BCerDpSLzh3mMxAYo=", "path": "github.com/syndtr/goleveldb/leveldb/iterator", - "revision": "169b1b37be738edb2813dab48c97a549bcf99bb5", - "revisionTime": "2018-03-07T11:33:52Z" + "revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a", + "revisionTime": "2018-05-02T07:23:49Z" }, { "checksumSHA1": "gJY7bRpELtO0PJpZXgPQ2BYFJ88=", "path": "github.com/syndtr/goleveldb/leveldb/journal", - "revision": "169b1b37be738edb2813dab48c97a549bcf99bb5", - "revisionTime": "2018-03-07T11:33:52Z" + "revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a", + "revisionTime": "2018-05-02T07:23:49Z" }, { "checksumSHA1": "MtYY1b2234y/MlS+djL8tXVAcQs=", "path": "github.com/syndtr/goleveldb/leveldb/memdb", - "revision": "169b1b37be738edb2813dab48c97a549bcf99bb5", - "revisionTime": "2018-03-07T11:33:52Z" + "revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a", + "revisionTime": "2018-05-02T07:23:49Z" }, { "checksumSHA1": "UmQeotV+m8/FduKEfLOhjdp18rs=", "path": "github.com/syndtr/goleveldb/leveldb/opt", - "revision": "169b1b37be738edb2813dab48c97a549bcf99bb5", - "revisionTime": "2018-03-07T11:33:52Z" + "revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a", + "revisionTime": "2018-05-02T07:23:49Z" }, { - "checksumSHA1": "QCSae2ub87f8awH+PKMpd8ZYOtg=", + "checksumSHA1": "7H3fa12T7WoMAeXq1+qG5O7LD0w=", "path": "github.com/syndtr/goleveldb/leveldb/storage", - "revision": "169b1b37be738edb2813dab48c97a549bcf99bb5", - "revisionTime": "2018-03-07T11:33:52Z" + "revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a", + "revisionTime": "2018-05-02T07:23:49Z" }, { "checksumSHA1": "gWFPMz8OQeul0t54RM66yMTX49g=", "path": "github.com/syndtr/goleveldb/leveldb/table", - "revision": "169b1b37be738edb2813dab48c97a549bcf99bb5", - "revisionTime": "2018-03-07T11:33:52Z" + "revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a", + "revisionTime": "2018-05-02T07:23:49Z" }, { "checksumSHA1": "V/Dh7NV0/fy/5jX1KaAjmGcNbzI=", "path": "github.com/syndtr/goleveldb/leveldb/util", - "revision": "169b1b37be738edb2813dab48c97a549bcf99bb5", - "revisionTime": "2018-03-07T11:33:52Z" + "revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a", + "revisionTime": "2018-05-02T07:23:49Z" }, { "checksumSHA1": "TT1rac6kpQp2vz24m5yDGUNQ/QQ=", diff --git a/whisper/mailserver/mailserver.go b/whisper/mailserver/mailserver.go index 57e6505ad1..d32eaddec3 100644 --- a/whisper/mailserver/mailserver.go +++ b/whisper/mailserver/mailserver.go @@ -20,7 +20,6 @@ import ( "encoding/binary" "fmt" - "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" @@ -54,19 +53,19 @@ func NewDbKey(t uint32, h common.Hash) *DBKey { return &k } -func (s *WMailServer) Init(shh *whisper.Whisper, path string, password string, pow float64) { +func (s *WMailServer) Init(shh *whisper.Whisper, path string, password string, pow float64) error { var err error if len(path) == 0 { - utils.Fatalf("DB file is not specified") + return fmt.Errorf("DB file is not specified") } if len(password) == 0 { - utils.Fatalf("Password is not specified for MailServer") + return fmt.Errorf("password is not specified") } s.db, err = leveldb.OpenFile(path, nil) if err != nil { - utils.Fatalf("Failed to open DB file: %s", err) + return fmt.Errorf("open DB file: %s", err) } s.w = shh @@ -74,12 +73,13 @@ func (s *WMailServer) Init(shh *whisper.Whisper, path string, password string, p MailServerKeyID, err := s.w.AddSymKeyFromPassword(password) if err != nil { - utils.Fatalf("Failed to create symmetric key for MailServer: %s", err) + return fmt.Errorf("create symmetric key: %s", err) } s.key, err = s.w.GetSymKey(MailServerKeyID) if err != nil { - utils.Fatalf("Failed to save symmetric key for MailServer") + return fmt.Errorf("save symmetric key: %s", err) } + return nil } func (s *WMailServer) Close() { diff --git a/whisper/mailserver/server_test.go b/whisper/mailserver/server_test.go index d5b993afb9..edb817cc75 100644 --- a/whisper/mailserver/server_test.go +++ b/whisper/mailserver/server_test.go @@ -92,7 +92,10 @@ func TestMailServer(t *testing.T) { shh = whisper.New(&whisper.DefaultConfig) shh.RegisterServer(&server) - server.Init(shh, dir, password, powRequirement) + err = server.Init(shh, dir, password, powRequirement) + if err != nil { + t.Fatal(err) + } defer server.Close() keyID, err = shh.AddSymKeyFromPassword(password) diff --git a/whisper/shhclient/client.go b/whisper/shhclient/client.go index bbe694baaa..8e7085a0a6 100644 --- a/whisper/shhclient/client.go +++ b/whisper/shhclient/client.go @@ -67,7 +67,6 @@ func (sc *Client) SetMaxMessageSize(ctx context.Context, size uint32) error { } // SetMinimumPoW (experimental) sets the minimal PoW required by this node. - // This experimental function was introduced for the future dynamic adjustment of // PoW requirement. If the node is overwhelmed with messages, it should raise the // PoW requirement and notify the peers. The new value should be set relative to @@ -77,7 +76,7 @@ func (sc *Client) SetMinimumPoW(ctx context.Context, pow float64) error { return sc.c.CallContext(ctx, &ignored, "shh_setMinPoW", pow) } -// Marks specific peer trusted, which will allow it to send historic (expired) messages. +// MarkTrustedPeer marks specific peer trusted, which will allow it to send historic (expired) messages. // Note This function is not adding new nodes, the node needs to exists as a peer. func (sc *Client) MarkTrustedPeer(ctx context.Context, enode string) error { var ignored bool @@ -136,9 +135,9 @@ func (sc *Client) AddSymmetricKey(ctx context.Context, key []byte) (string, erro } // GenerateSymmetricKeyFromPassword generates the key from password, stores it, and returns its identifier. -func (sc *Client) GenerateSymmetricKeyFromPassword(ctx context.Context, passwd []byte) (string, error) { +func (sc *Client) GenerateSymmetricKeyFromPassword(ctx context.Context, passwd string) (string, error) { var id string - return id, sc.c.CallContext(ctx, &id, "shh_generateSymKeyFromPassword", hexutil.Bytes(passwd)) + return id, sc.c.CallContext(ctx, &id, "shh_generateSymKeyFromPassword", passwd) } // HasSymmetricKey returns an indication if the key associated with the given id is stored in the node. diff --git a/whisper/whisperv5/api.go b/whisper/whisperv5/api.go index 9fb22aa752..c56d139499 100644 --- a/whisper/whisperv5/api.go +++ b/whisper/whisperv5/api.go @@ -89,7 +89,7 @@ func (api *PublicWhisperAPI) SetMaxMessageSize(ctx context.Context, size uint32) return true, api.w.SetMaxMessageSize(size) } -// SetMinPow sets the minimum PoW for a message before it is accepted. +// SetMinPoW sets the minimum PoW for a message before it is accepted. func (api *PublicWhisperAPI) SetMinPoW(ctx context.Context, pow float64) (bool, error) { return true, api.w.SetMinimumPoW(pow) } @@ -142,7 +142,7 @@ func (api *PublicWhisperAPI) GetPublicKey(ctx context.Context, id string) (hexut return crypto.FromECDSAPub(&key.PublicKey), nil } -// GetPublicKey returns the private key associated with the given key. The key is the hex +// GetPrivateKey returns the private key associated with the given key. The key is the hex // encoded representation of a key in the form specified in section 4.3.6 of ANSI X9.62. func (api *PublicWhisperAPI) GetPrivateKey(ctx context.Context, id string) (hexutil.Bytes, error) { key, err := api.w.GetPrivateKey(id) diff --git a/whisper/whisperv5/doc.go b/whisper/whisperv5/doc.go index 7a57488bd7..8161db8ed6 100644 --- a/whisper/whisperv5/doc.go +++ b/whisper/whisperv5/doc.go @@ -15,7 +15,7 @@ // along with the go-ethereum library. If not, see . /* -Package whisper implements the Whisper protocol (version 5). +Package whisperv5 implements the Whisper protocol (version 5). Whisper combines aspects of both DHTs and datagram messaging systems (e.g. UDP). As such it may be likened and compared to both, not dissimilar to the diff --git a/whisper/whisperv5/message.go b/whisper/whisperv5/message.go index 34ce52e64f..35711d724b 100644 --- a/whisper/whisperv5/message.go +++ b/whisper/whisperv5/message.go @@ -33,7 +33,7 @@ import ( "github.com/ethereum/go-ethereum/log" ) -// Options specifies the exact way a message should be wrapped into an Envelope. +// MessageParams specifies the exact way a message should be wrapped into an Envelope. type MessageParams struct { TTL uint32 Src *ecdsa.PrivateKey @@ -86,7 +86,7 @@ func (msg *ReceivedMessage) isAsymmetricEncryption() bool { return msg.Dst != nil } -// NewMessage creates and initializes a non-signed, non-encrypted Whisper message. +// NewSentMessage creates and initializes a non-signed, non-encrypted Whisper message. func NewSentMessage(params *MessageParams) (*sentMessage, error) { msg := sentMessage{} msg.Raw = make([]byte, 1, len(params.Payload)+len(params.Padding)+signatureLength+padSizeLimit) @@ -330,7 +330,7 @@ func (msg *ReceivedMessage) extractPadding(end int) (int, bool) { return paddingSize, true } -// Recover retrieves the public key of the message signer. +// SigToPubKey retrieves the public key of the message signer. func (msg *ReceivedMessage) SigToPubKey() *ecdsa.PublicKey { defer func() { recover() }() // in case of invalid signature diff --git a/whisper/whisperv5/peer.go b/whisper/whisperv5/peer.go index 179c931795..da07631992 100644 --- a/whisper/whisperv5/peer.go +++ b/whisper/whisperv5/peer.go @@ -27,7 +27,7 @@ import ( set "gopkg.in/fatih/set.v0" ) -// peer represents a whisper protocol peer connection. +// Peer represents a whisper protocol peer connection. type Peer struct { host *Whisper peer *p2p.Peer @@ -53,51 +53,51 @@ func newPeer(host *Whisper, remote *p2p.Peer, rw p2p.MsgReadWriter) *Peer { // start initiates the peer updater, periodically broadcasting the whisper packets // into the network. -func (p *Peer) start() { - go p.update() - log.Trace("start", "peer", p.ID()) +func (peer *Peer) start() { + go peer.update() + log.Trace("start", "peer", peer.ID()) } // stop terminates the peer updater, stopping message forwarding to it. -func (p *Peer) stop() { - close(p.quit) - log.Trace("stop", "peer", p.ID()) +func (peer *Peer) stop() { + close(peer.quit) + log.Trace("stop", "peer", peer.ID()) } // handshake sends the protocol initiation status message to the remote peer and // verifies the remote status too. -func (p *Peer) handshake() error { +func (peer *Peer) handshake() error { // Send the handshake status message asynchronously errc := make(chan error, 1) go func() { - errc <- p2p.Send(p.ws, statusCode, ProtocolVersion) + errc <- p2p.Send(peer.ws, statusCode, ProtocolVersion) }() // Fetch the remote status packet and verify protocol match - packet, err := p.ws.ReadMsg() + packet, err := peer.ws.ReadMsg() if err != nil { return err } if packet.Code != statusCode { - return fmt.Errorf("peer [%x] sent packet %x before status packet", p.ID(), packet.Code) + return fmt.Errorf("peer [%x] sent packet %x before status packet", peer.ID(), packet.Code) } s := rlp.NewStream(packet.Payload, uint64(packet.Size)) peerVersion, err := s.Uint() if err != nil { - return fmt.Errorf("peer [%x] sent bad status message: %v", p.ID(), err) + return fmt.Errorf("peer [%x] sent bad status message: %v", peer.ID(), err) } if peerVersion != ProtocolVersion { - return fmt.Errorf("peer [%x]: protocol version mismatch %d != %d", p.ID(), peerVersion, ProtocolVersion) + return fmt.Errorf("peer [%x]: protocol version mismatch %d != %d", peer.ID(), peerVersion, ProtocolVersion) } // Wait until out own status is consumed too if err := <-errc; err != nil { - return fmt.Errorf("peer [%x] failed to send status packet: %v", p.ID(), err) + return fmt.Errorf("peer [%x] failed to send status packet: %v", peer.ID(), err) } return nil } // update executes periodic operations on the peer, including message transmission // and expiration. -func (p *Peer) update() { +func (peer *Peer) update() { // Start the tickers for the updates expire := time.NewTicker(expirationCycle) transmit := time.NewTicker(transmissionCycle) @@ -106,15 +106,15 @@ func (p *Peer) update() { for { select { case <-expire.C: - p.expire() + peer.expire() case <-transmit.C: - if err := p.broadcast(); err != nil { - log.Trace("broadcast failed", "reason", err, "peer", p.ID()) + if err := peer.broadcast(); err != nil { + log.Trace("broadcast failed", "reason", err, "peer", peer.ID()) return } - case <-p.quit: + case <-peer.quit: return } } @@ -148,16 +148,16 @@ func (peer *Peer) expire() { // broadcast iterates over the collection of envelopes and transmits yet unknown // ones over the network. -func (p *Peer) broadcast() error { +func (peer *Peer) broadcast() error { var cnt int - envelopes := p.host.Envelopes() + envelopes := peer.host.Envelopes() for _, envelope := range envelopes { - if !p.marked(envelope) { - err := p2p.Send(p.ws, messagesCode, envelope) + if !peer.marked(envelope) { + err := p2p.Send(peer.ws, messagesCode, envelope) if err != nil { return err } else { - p.mark(envelope) + peer.mark(envelope) cnt++ } } @@ -168,7 +168,7 @@ func (p *Peer) broadcast() error { return nil } -func (p *Peer) ID() []byte { - id := p.peer.ID() +func (peer *Peer) ID() []byte { + id := peer.peer.ID() return id[:] } diff --git a/whisper/whisperv5/peer_test.go b/whisper/whisperv5/peer_test.go index bae2adb6f5..051b52dcf8 100644 --- a/whisper/whisperv5/peer_test.go +++ b/whisper/whisperv5/peer_test.go @@ -32,7 +32,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/nat" ) -var keys []string = []string{ +var keys = []string{ "d49dcf37238dc8a7aac57dc61b9fee68f0a97f062968978b9fafa7d1033d03a9", "73fd6143c48e80ed3c56ea159fe7494a0b6b393a392227b422f4c3e8f1b54f98", "119dd32adb1daa7a4c7bf77f847fb28730785aa92947edf42fdd997b54de40dc", @@ -84,9 +84,9 @@ type TestNode struct { var result TestData var nodes [NumNodes]*TestNode -var sharedKey []byte = []byte("some arbitrary data here") +var sharedKey = []byte("some arbitrary data here") var sharedTopic TopicType = TopicType{0xF, 0x1, 0x2, 0} -var expectedMessage []byte = []byte("per rectum ad astra") +var expectedMessage = []byte("per rectum ad astra") // This test does the following: // 1. creates a chain of whisper nodes, diff --git a/whisper/whisperv5/topic.go b/whisper/whisperv5/topic.go index c4ea67eefa..c4eda1db42 100644 --- a/whisper/whisperv5/topic.go +++ b/whisper/whisperv5/topic.go @@ -23,7 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" ) -// Topic represents a cryptographically secure, probabilistic partial +// TopicType represents a cryptographically secure, probabilistic partial // classifications of a message, determined as the first (left) 4 bytes of the // SHA3 hash of some arbitrary data given by the original author of the message. type TopicType [TopicLength]byte diff --git a/whisper/whisperv5/whisper.go b/whisper/whisperv5/whisper.go index 85849ccce4..62bd1ce179 100644 --- a/whisper/whisperv5/whisper.go +++ b/whisper/whisperv5/whisper.go @@ -469,18 +469,18 @@ func (w *Whisper) Stop() error { // HandlePeer is called by the underlying P2P layer when the whisper sub-protocol // connection is negotiated. -func (wh *Whisper) HandlePeer(peer *p2p.Peer, rw p2p.MsgReadWriter) error { +func (w *Whisper) HandlePeer(peer *p2p.Peer, rw p2p.MsgReadWriter) error { // Create the new peer and start tracking it - whisperPeer := newPeer(wh, peer, rw) + whisperPeer := newPeer(w, peer, rw) - wh.peerMu.Lock() - wh.peers[whisperPeer] = struct{}{} - wh.peerMu.Unlock() + w.peerMu.Lock() + w.peers[whisperPeer] = struct{}{} + w.peerMu.Unlock() defer func() { - wh.peerMu.Lock() - delete(wh.peers, whisperPeer) - wh.peerMu.Unlock() + w.peerMu.Lock() + delete(w.peers, whisperPeer) + w.peerMu.Unlock() }() // Run the peer handshake and state updates @@ -490,11 +490,11 @@ func (wh *Whisper) HandlePeer(peer *p2p.Peer, rw p2p.MsgReadWriter) error { whisperPeer.start() defer whisperPeer.stop() - return wh.runMessageLoop(whisperPeer, rw) + return w.runMessageLoop(whisperPeer, rw) } // runMessageLoop reads and processes inbound messages directly to merge into client-global state. -func (wh *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error { +func (w *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error { for { // fetch the next packet packet, err := rw.ReadMsg() @@ -502,7 +502,7 @@ func (wh *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error { log.Warn("message loop", "peer", p.peer.ID(), "err", err) return err } - if packet.Size > wh.MaxMessageSize() { + if packet.Size > w.MaxMessageSize() { log.Warn("oversized message received", "peer", p.peer.ID()) return errors.New("oversized message received") } @@ -518,7 +518,7 @@ func (wh *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error { log.Warn("failed to decode envelope, peer will be disconnected", "peer", p.peer.ID(), "err", err) return errors.New("invalid envelope") } - cached, err := wh.add(&envelope) + cached, err := w.add(&envelope) if err != nil { log.Warn("bad envelope received, peer will be disconnected", "peer", p.peer.ID(), "err", err) return errors.New("invalid envelope") @@ -537,17 +537,17 @@ func (wh *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error { log.Warn("failed to decode direct message, peer will be disconnected", "peer", p.peer.ID(), "err", err) return errors.New("invalid direct message") } - wh.postEvent(&envelope, true) + w.postEvent(&envelope, true) } case p2pRequestCode: // Must be processed if mail server is implemented. Otherwise ignore. - if wh.mailServer != nil { + if w.mailServer != nil { var request Envelope if err := packet.Decode(&request); err != nil { log.Warn("failed to decode p2p request message, peer will be disconnected", "peer", p.peer.ID(), "err", err) return errors.New("invalid p2p request") } - wh.mailServer.DeliverMail(p, &request) + w.mailServer.DeliverMail(p, &request) } default: // New message types might be implemented in the future versions of Whisper. @@ -561,29 +561,27 @@ func (wh *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error { // add inserts a new envelope into the message pool to be distributed within the // whisper network. It also inserts the envelope into the expiration pool at the // appropriate time-stamp. In case of error, connection should be dropped. -func (wh *Whisper) add(envelope *Envelope) (bool, error) { +func (w *Whisper) add(envelope *Envelope) (bool, error) { now := uint32(time.Now().Unix()) sent := envelope.Expiry - envelope.TTL if sent > now { if sent-SynchAllowance > now { return false, fmt.Errorf("envelope created in the future [%x]", envelope.Hash()) - } else { - // recalculate PoW, adjusted for the time difference, plus one second for latency - envelope.calculatePoW(sent - now + 1) } + // recalculate PoW, adjusted for the time difference, plus one second for latency + envelope.calculatePoW(sent - now + 1) } if envelope.Expiry < now { if envelope.Expiry+SynchAllowance*2 < now { return false, fmt.Errorf("very old message") - } else { - log.Debug("expired envelope dropped", "hash", envelope.Hash().Hex()) - return false, nil // drop envelope without error } + log.Debug("expired envelope dropped", "hash", envelope.Hash().Hex()) + return false, nil // drop envelope without error } - if uint32(envelope.size()) > wh.MaxMessageSize() { + if uint32(envelope.size()) > w.MaxMessageSize() { return false, fmt.Errorf("huge messages are not allowed [%x]", envelope.Hash()) } @@ -598,36 +596,36 @@ func (wh *Whisper) add(envelope *Envelope) (bool, error) { return false, fmt.Errorf("wrong size of AESNonce: %d bytes [env: %x]", aesNonceSize, envelope.Hash()) } - if envelope.PoW() < wh.MinPow() { + if envelope.PoW() < w.MinPow() { log.Debug("envelope with low PoW dropped", "PoW", envelope.PoW(), "hash", envelope.Hash().Hex()) return false, nil // drop envelope without error } hash := envelope.Hash() - wh.poolMu.Lock() - _, alreadyCached := wh.envelopes[hash] + w.poolMu.Lock() + _, alreadyCached := w.envelopes[hash] if !alreadyCached { - wh.envelopes[hash] = envelope - if wh.expirations[envelope.Expiry] == nil { - wh.expirations[envelope.Expiry] = set.NewNonTS() + w.envelopes[hash] = envelope + if w.expirations[envelope.Expiry] == nil { + w.expirations[envelope.Expiry] = set.NewNonTS() } - if !wh.expirations[envelope.Expiry].Has(hash) { - wh.expirations[envelope.Expiry].Add(hash) + if !w.expirations[envelope.Expiry].Has(hash) { + w.expirations[envelope.Expiry].Add(hash) } } - wh.poolMu.Unlock() + w.poolMu.Unlock() if alreadyCached { log.Trace("whisper envelope already cached", "hash", envelope.Hash().Hex()) } else { log.Trace("cached whisper envelope", "hash", envelope.Hash().Hex()) - wh.statsMu.Lock() - wh.stats.memoryUsed += envelope.size() - wh.statsMu.Unlock() - wh.postEvent(envelope, false) // notify the local node about the new message - if wh.mailServer != nil { - wh.mailServer.Archive(envelope) + w.statsMu.Lock() + w.stats.memoryUsed += envelope.size() + w.statsMu.Unlock() + w.postEvent(envelope, false) // notify the local node about the new message + if w.mailServer != nil { + w.mailServer.Archive(envelope) } } return true, nil @@ -838,9 +836,8 @@ func deriveKeyMaterial(key []byte, version uint64) (derivedKey []byte, err error // because it's a once in a session experience derivedKey := pbkdf2.Key(key, nil, 65356, aesKeyLength, sha256.New) return derivedKey, nil - } else { - return nil, unknownVersionError(version) } + return nil, unknownVersionError(version) } // GenerateRandomID generates a random string, which is then returned to be used as a key id