diff --git a/.travis.yml b/.travis.yml index 9e65f34034..9fcfe6338f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,6 @@ matrix: script: - unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703 - brew update - - brew install caskroom/cask/brew-cask - brew cask install osxfuse - go run build/ci.go install - go run build/ci.go test -coverage $TEST_PACKAGES @@ -127,7 +126,7 @@ matrix: # This builder does the Android Maven and Azure uploads - os: linux - dist: precise # Needed for the android tools + dist: trusty addons: apt: packages: @@ -147,16 +146,16 @@ matrix: git: submodules: false # avoid cloning ethereum/tests before_install: - - curl https://storage.googleapis.com/golang/go1.10.1.linux-amd64.tar.gz | tar -xz + - curl https://storage.googleapis.com/golang/go1.10.2.linux-amd64.tar.gz | tar -xz - export PATH=`pwd`/go/bin:$PATH - export GOROOT=`pwd`/go - export GOPATH=$HOME/go script: # Build the Android archive and upload it to Maven Central and Azure - - curl https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip -o android-ndk-r15c.zip - - unzip -q android-ndk-r15c.zip && rm android-ndk-r15c.zip - - mv android-ndk-r15c $HOME - - export ANDROID_NDK=$HOME/android-ndk-r15c + - curl https://dl.google.com/android/repository/android-ndk-r16b-linux-x86_64.zip -o android-ndk-r16b.zip + - unzip -q android-ndk-r16b.zip && rm android-ndk-r16b.zip + - mv android-ndk-r16b $HOME + - export ANDROID_NDK=$HOME/android-ndk-r16b - mkdir -p $GOPATH/src/github.com/ethereum - ln -s `pwd` $GOPATH/src/github.com/ethereum diff --git a/VERSION b/VERSION index f263cd11b4..53ed4ba51e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.8.6 +1.8.9 diff --git a/accounts/abi/argument.go b/accounts/abi/argument.go index 512d8fdfa7..93b513c346 100644 --- a/accounts/abi/argument.go +++ b/accounts/abi/argument.go @@ -111,9 +111,14 @@ func (arguments Arguments) unpackTuple(v interface{}, marshalledValues []interfa if err := requireUnpackKind(value, typ, kind, arguments); err != nil { return err } - // If the output interface is a struct, make sure names don't collide + + // If the interface is a struct, get of abi->struct_field mapping + + var abi2struct map[string]string if kind == reflect.Struct { - if err := requireUniqueStructFieldNames(arguments); err != nil { + var err error + abi2struct, err = mapAbiToStructFields(arguments, value) + if err != nil { return err } } @@ -123,9 +128,10 @@ func (arguments Arguments) unpackTuple(v interface{}, marshalledValues []interfa switch kind { case reflect.Struct: - err := unpackStruct(value, reflectValue, arg) - if err != nil { - return err + if structField, ok := abi2struct[arg.Name]; ok { + if err := set(value.FieldByName(structField), reflectValue, arg); err != nil { + return err + } } case reflect.Slice, reflect.Array: if value.Len() < i { @@ -151,17 +157,22 @@ func (arguments Arguments) unpackAtomic(v interface{}, marshalledValues []interf if len(marshalledValues) != 1 { return fmt.Errorf("abi: wrong length, expected single value, got %d", len(marshalledValues)) } + elem := reflect.ValueOf(v).Elem() kind := elem.Kind() reflectValue := reflect.ValueOf(marshalledValues[0]) + var abi2struct map[string]string if kind == reflect.Struct { - //make sure names don't collide - if err := requireUniqueStructFieldNames(arguments); err != nil { + var err error + if abi2struct, err = mapAbiToStructFields(arguments, elem); err != nil { return err } - - return unpackStruct(elem, reflectValue, arguments[0]) + arg := arguments.NonIndexed()[0] + if structField, ok := abi2struct[arg.Name]; ok { + return set(elem.FieldByName(structField), reflectValue, arg) + } + return nil } return set(elem, reflectValue, arguments.NonIndexed()[0]) @@ -277,18 +288,3 @@ func capitalise(input string) string { } return strings.ToUpper(input[:1]) + input[1:] } - -//unpackStruct extracts each argument into its corresponding struct field -func unpackStruct(value, reflectValue reflect.Value, arg Argument) error { - name := capitalise(arg.Name) - typ := value.Type() - for j := 0; j < typ.NumField(); j++ { - // TODO read tags: `abi:"fieldName"` - if typ.Field(j).Name == name { - if err := set(value.Field(j), reflectValue, arg); err != nil { - return err - } - } - } - return nil -} diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 0d07b25d1f..c29f5ee16e 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -24,13 +24,14 @@ import ( "sync" "time" - "github.com/EthereumCommonwealth/go-callisto" + ethereum "github.com/EthereumCommonwealth/go-callisto" "github.com/EthereumCommonwealth/go-callisto/accounts/abi/bind" "github.com/EthereumCommonwealth/go-callisto/common" "github.com/EthereumCommonwealth/go-callisto/common/math" "github.com/EthereumCommonwealth/go-callisto/consensus/ethash" "github.com/EthereumCommonwealth/go-callisto/core" "github.com/EthereumCommonwealth/go-callisto/core/bloombits" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/state" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/core/vm" @@ -65,7 +66,7 @@ type SimulatedBackend struct { // NewSimulatedBackend creates a new binding backend using a simulated blockchain // for testing purposes. func NewSimulatedBackend(alloc core.GenesisAlloc) *SimulatedBackend { - database, _ := ethdb.NewMemDatabase() + database := ethdb.NewMemDatabase() genesis := core.Genesis{Config: params.AllEthashProtocolChanges, Alloc: alloc} genesis.MustCommit(database) blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}) @@ -159,7 +160,7 @@ func (b *SimulatedBackend) StorageAt(ctx context.Context, contract common.Addres // TransactionReceipt returns the receipt of a transaction. func (b *SimulatedBackend) TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) { - receipt, _, _, _ := core.GetReceipt(b.database, txHash) + receipt, _, _, _ := rawdb.ReadReceipt(b.database, txHash) return receipt, nil } @@ -430,11 +431,19 @@ func (fb *filterBackend) HeaderByNumber(ctx context.Context, block rpc.BlockNumb } func (fb *filterBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { - return core.GetBlockReceipts(fb.db, hash, core.GetBlockNumber(fb.db, hash)), nil + number := rawdb.ReadHeaderNumber(fb.db, hash) + if number == nil { + return nil, nil + } + return rawdb.ReadReceipts(fb.db, hash, *number), nil } func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { - receipts := core.GetBlockReceipts(fb.db, hash, core.GetBlockNumber(fb.db, hash)) + number := rawdb.ReadHeaderNumber(fb.db, hash) + if number == nil { + return nil, nil + } + receipts := rawdb.ReadReceipts(fb.db, hash, *number) if receipts == nil { return nil, nil } @@ -445,7 +454,7 @@ func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*ty return logs, nil } -func (fb *filterBackend) SubscribeTxPreEvent(ch chan<- core.TxPreEvent) event.Subscription { +func (fb *filterBackend) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subscription { return event.NewSubscription(func(quit <-chan struct{}) error { <-quit return nil diff --git a/accounts/abi/event.go b/accounts/abi/event.go index 337ca18fba..cbfbe3bc47 100644 --- a/accounts/abi/event.go +++ b/accounts/abi/event.go @@ -33,15 +33,15 @@ type Event struct { Inputs Arguments } -func (event Event) String() string { - inputs := make([]string, len(event.Inputs)) - for i, input := range event.Inputs { +func (e Event) String() string { + inputs := make([]string, len(e.Inputs)) + for i, input := range e.Inputs { inputs[i] = fmt.Sprintf("%v %v", input.Name, input.Type) if input.Indexed { inputs[i] = fmt.Sprintf("%v indexed %v", input.Name, input.Type) } } - return fmt.Sprintf("event %v(%v)", event.Name, strings.Join(inputs, ", ")) + return fmt.Sprintf("e %v(%v)", e.Name, strings.Join(inputs, ", ")) } // Id returns the canonical representation of the event's signature used by the diff --git a/accounts/abi/event_test.go b/accounts/abi/event_test.go index 27a240a68c..740dc247a6 100644 --- a/accounts/abi/event_test.go +++ b/accounts/abi/event_test.go @@ -58,12 +58,28 @@ var jsonEventPledge = []byte(`{ "type": "event" }`) +var jsonEventMixedCase = []byte(`{ + "anonymous": false, + "inputs": [{ + "indexed": false, "name": "value", "type": "uint256" + }, { + "indexed": false, "name": "_value", "type": "uint256" + }, { + "indexed": false, "name": "Value", "type": "uint256" + }], + "name": "MixedCase", + "type": "event" + }`) + // 1000000 var transferData1 = "00000000000000000000000000000000000000000000000000000000000f4240" // "0x00Ce0d46d924CC8437c806721496599FC3FFA268", 2218516807680, "usd" var pledgeData1 = "00000000000000000000000000ce0d46d924cc8437c806721496599fc3ffa2680000000000000000000000000000000000000000000000000000020489e800007573640000000000000000000000000000000000000000000000000000000000" +// 1000000,2218516807680,1000001 +var mixedCaseData1 = "00000000000000000000000000000000000000000000000000000000000f42400000000000000000000000000000000000000000000000000000020489e8000000000000000000000000000000000000000000000000000000000000000f4241" + func TestEventId(t *testing.T) { var table = []struct { definition string @@ -121,6 +137,27 @@ func TestEventTupleUnpack(t *testing.T) { Value *big.Int } + type EventTransferWithTag struct { + // this is valid because `value` is not exportable, + // so value is only unmarshalled into `Value1`. + value *big.Int + Value1 *big.Int `abi:"value"` + } + + type BadEventTransferWithSameFieldAndTag struct { + Value *big.Int + Value1 *big.Int `abi:"value"` + } + + type BadEventTransferWithDuplicatedTag struct { + Value1 *big.Int `abi:"value"` + Value2 *big.Int `abi:"value"` + } + + type BadEventTransferWithEmptyTag struct { + Value *big.Int `abi:""` + } + type EventPledge struct { Who common.Address Wad *big.Int @@ -133,9 +170,16 @@ func TestEventTupleUnpack(t *testing.T) { Currency [3]byte } + type EventMixedCase struct { + Value1 *big.Int `abi:"value"` + Value2 *big.Int `abi:"_value"` + Value3 *big.Int `abi:"Value"` + } + bigint := new(big.Int) bigintExpected := big.NewInt(1000000) bigintExpected2 := big.NewInt(2218516807680) + bigintExpected3 := big.NewInt(1000001) addr := common.HexToAddress("0x00Ce0d46d924CC8437c806721496599FC3FFA268") var testCases = []struct { data string @@ -158,6 +202,34 @@ func TestEventTupleUnpack(t *testing.T) { jsonEventTransfer, "", "Can unpack ERC20 Transfer event into slice", + }, { + transferData1, + &EventTransferWithTag{}, + &EventTransferWithTag{Value1: bigintExpected}, + jsonEventTransfer, + "", + "Can unpack ERC20 Transfer event into structure with abi: tag", + }, { + transferData1, + &BadEventTransferWithDuplicatedTag{}, + &BadEventTransferWithDuplicatedTag{}, + jsonEventTransfer, + "struct: abi tag in 'Value2' already mapped", + "Can not unpack ERC20 Transfer event with duplicated abi tag", + }, { + transferData1, + &BadEventTransferWithSameFieldAndTag{}, + &BadEventTransferWithSameFieldAndTag{}, + jsonEventTransfer, + "abi: multiple variables maps to the same abi field 'value'", + "Can not unpack ERC20 Transfer event with a field and a tag mapping to the same abi variable", + }, { + transferData1, + &BadEventTransferWithEmptyTag{}, + &BadEventTransferWithEmptyTag{}, + jsonEventTransfer, + "struct: abi tag in 'Value' is empty", + "Can not unpack ERC20 Transfer event with an empty tag", }, { pledgeData1, &EventPledge{}, @@ -216,6 +288,13 @@ func TestEventTupleUnpack(t *testing.T) { jsonEventPledge, "abi: cannot unmarshal tuple into map[string]interface {}", "Can not unpack Pledge event into map", + }, { + mixedCaseData1, + &EventMixedCase{}, + &EventMixedCase{Value1: bigintExpected, Value2: bigintExpected2, Value3: bigintExpected3}, + jsonEventMixedCase, + "", + "Can unpack abi variables with mixed case", }} for _, tc := range testCases { @@ -227,7 +306,7 @@ func TestEventTupleUnpack(t *testing.T) { assert.Nil(err, "Should be able to unpack event data.") assert.Equal(tc.expected, tc.dest, tc.name) } else { - assert.EqualError(err, tc.error) + assert.EqualError(err, tc.error, tc.name) } }) } diff --git a/accounts/abi/reflect.go b/accounts/abi/reflect.go index 5620a70845..0193517a42 100644 --- a/accounts/abi/reflect.go +++ b/accounts/abi/reflect.go @@ -19,6 +19,7 @@ package abi import ( "fmt" "reflect" + "strings" ) // indirect recursively dereferences the value until it either gets the value @@ -111,18 +112,101 @@ func requireUnpackKind(v reflect.Value, t reflect.Type, k reflect.Kind, return nil } -// requireUniqueStructFieldNames makes sure field names don't collide -func requireUniqueStructFieldNames(args Arguments) error { - exists := make(map[string]bool) - for _, arg := range args { - field := capitalise(arg.Name) - if field == "" { - return fmt.Errorf("abi: purely underscored output cannot unpack to struct") +// mapAbiToStringField maps abi to struct fields. +// first round: for each Exportable field that contains a `abi:""` tag +// and this field name exists in the arguments, pair them together. +// second round: for each argument field that has not been already linked, +// find what variable is expected to be mapped into, if it exists and has not been +// used, pair them. +func mapAbiToStructFields(args Arguments, value reflect.Value) (map[string]string, error) { + + typ := value.Type() + + abi2struct := make(map[string]string) + struct2abi := make(map[string]string) + + // first round ~~~ + for i := 0; i < typ.NumField(); i++ { + structFieldName := typ.Field(i).Name + + // skip private struct fields. + if structFieldName[:1] != strings.ToUpper(structFieldName[:1]) { + continue } - if exists[field] { - return fmt.Errorf("abi: multiple outputs mapping to the same struct field '%s'", field) + + // skip fields that have no abi:"" tag. + var ok bool + var tagName string + if tagName, ok = typ.Field(i).Tag.Lookup("abi"); !ok { + continue } - exists[field] = true + + // check if tag is empty. + if tagName == "" { + return nil, fmt.Errorf("struct: abi tag in '%s' is empty", structFieldName) + } + + // check which argument field matches with the abi tag. + found := false + for _, abiField := range args.NonIndexed() { + if abiField.Name == tagName { + if abi2struct[abiField.Name] != "" { + return nil, fmt.Errorf("struct: abi tag in '%s' already mapped", structFieldName) + } + // pair them + abi2struct[abiField.Name] = structFieldName + struct2abi[structFieldName] = abiField.Name + found = true + } + } + + // check if this tag has been mapped. + if !found { + return nil, fmt.Errorf("struct: abi tag '%s' defined but not found in abi", tagName) + } + } - return nil + + // second round ~~~ + for _, arg := range args { + + abiFieldName := arg.Name + structFieldName := capitalise(abiFieldName) + + if structFieldName == "" { + return nil, fmt.Errorf("abi: purely underscored output cannot unpack to struct") + } + + // this abi has already been paired, skip it... unless there exists another, yet unassigned + // struct field with the same field name. If so, raise an error: + // abi: [ { "name": "value" } ] + // struct { Value *big.Int , Value1 *big.Int `abi:"value"`} + if abi2struct[abiFieldName] != "" { + if abi2struct[abiFieldName] != structFieldName && + struct2abi[structFieldName] == "" && + value.FieldByName(structFieldName).IsValid() { + return nil, fmt.Errorf("abi: multiple variables maps to the same abi field '%s'", abiFieldName) + } + continue + } + + // return an error if this struct field has already been paired. + if struct2abi[structFieldName] != "" { + return nil, fmt.Errorf("abi: multiple outputs mapping to the same struct field '%s'", structFieldName) + } + + if value.FieldByName(structFieldName).IsValid() { + // pair them + abi2struct[abiFieldName] = structFieldName + struct2abi[structFieldName] = abiFieldName + } else { + // not paired, but annotate as used, to detect cases like + // abi : [ { "name": "value" }, { "name": "_value" } ] + // struct { Value *big.Int } + struct2abi[structFieldName] = abiFieldName + } + + } + + return abi2struct, nil } diff --git a/accounts/keystore/keystore_passphrase.go b/accounts/keystore/keystore_passphrase.go index 9e3182f500..c7e4928ae2 100644 --- a/accounts/keystore/keystore_passphrase.go +++ b/accounts/keystore/keystore_passphrase.go @@ -108,9 +108,8 @@ func (ks keyStorePassphrase) StoreKey(filename string, key *Key, auth string) er func (ks keyStorePassphrase) JoinPath(filename string) string { if filepath.IsAbs(filename) { return filename - } else { - return filepath.Join(ks.keysDirPath, filename) } + return filepath.Join(ks.keysDirPath, filename) } // EncryptKey encrypts a key using the specified scrypt parameters into a json diff --git a/accounts/keystore/keystore_plain.go b/accounts/keystore/keystore_plain.go index aa635391b8..cf2b819e91 100644 --- a/accounts/keystore/keystore_plain.go +++ b/accounts/keystore/keystore_plain.go @@ -56,7 +56,6 @@ func (ks keyStorePlain) StoreKey(filename string, key *Key, auth string) error { func (ks keyStorePlain) JoinPath(filename string) string { if filepath.IsAbs(filename) { return filename - } else { - return filepath.Join(ks.keysDirPath, filename) } + return filepath.Join(ks.keysDirPath, filename) } diff --git a/appveyor.yml b/appveyor.yml index 141ef16ff8..f10000af86 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -23,8 +23,8 @@ environment: install: - git submodule update --init - rmdir C:\go /s /q - - appveyor DownloadFile https://storage.googleapis.com/golang/go1.10.1.windows-%GETH_ARCH%.zip - - 7z x go1.10.1.windows-%GETH_ARCH%.zip -y -oC:\ > NUL + - appveyor DownloadFile https://storage.googleapis.com/golang/go1.10.2.windows-%GETH_ARCH%.zip + - 7z x go1.10.2.windows-%GETH_ARCH%.zip -y -oC:\ > NUL - go version - gcc --version diff --git a/bmt/bmt.go b/bmt/bmt.go index 3408758675..c290223452 100644 --- a/bmt/bmt.go +++ b/bmt/bmt.go @@ -150,29 +150,29 @@ func NewTreePool(hasher BaseHasher, segmentCount, capacity int) *TreePool { } // Drain drains the pool until it has no more than n resources -func (self *TreePool) Drain(n int) { - self.lock.Lock() - defer self.lock.Unlock() - for len(self.c) > n { - <-self.c - self.count-- +func (p *TreePool) Drain(n int) { + p.lock.Lock() + defer p.lock.Unlock() + for len(p.c) > n { + <-p.c + p.count-- } } // Reserve is blocking until it returns an available Tree // it reuses free Trees or creates a new one if size is not reached -func (self *TreePool) Reserve() *Tree { - self.lock.Lock() - defer self.lock.Unlock() +func (p *TreePool) Reserve() *Tree { + p.lock.Lock() + defer p.lock.Unlock() var t *Tree - if self.count == self.Capacity { - return <-self.c + if p.count == p.Capacity { + return <-p.c } select { - case t = <-self.c: + case t = <-p.c: default: - t = NewTree(self.hasher, self.SegmentSize, self.SegmentCount) - self.count++ + t = NewTree(p.hasher, p.SegmentSize, p.SegmentCount) + p.count++ } return t } @@ -180,8 +180,8 @@ func (self *TreePool) Reserve() *Tree { // Release gives back a Tree to the pool. // This Tree is guaranteed to be in reusable state // does not need locking -func (self *TreePool) Release(t *Tree) { - self.c <- t // can never fail but... +func (p *TreePool) Release(t *Tree) { + p.c <- t // can never fail but... } // Tree is a reusable control structure representing a BMT @@ -193,17 +193,17 @@ type Tree struct { } // Draw draws the BMT (badly) -func (self *Tree) Draw(hash []byte, d int) string { +func (t *Tree) Draw(hash []byte, d int) string { var left, right []string var anc []*Node - for i, n := range self.leaves { + for i, n := range t.leaves { left = append(left, fmt.Sprintf("%v", hashstr(n.left))) if i%2 == 0 { anc = append(anc, n.parent) } right = append(right, fmt.Sprintf("%v", hashstr(n.right))) } - anc = self.leaves + anc = t.leaves var hashes [][]string for l := 0; len(anc) > 0; l++ { var nodes []*Node @@ -277,42 +277,42 @@ func NewTree(hasher BaseHasher, segmentSize, segmentCount int) *Tree { // methods needed by hash.Hash // Size returns the size -func (self *Hasher) Size() int { - return self.size +func (h *Hasher) Size() int { + return h.size } // BlockSize returns the block size -func (self *Hasher) BlockSize() int { - return self.blocksize +func (h *Hasher) BlockSize() int { + return h.blocksize } // Sum returns the hash of the buffer // hash.Hash interface Sum method appends the byte slice to the underlying // data before it calculates and returns the hash of the chunk -func (self *Hasher) Sum(b []byte) (r []byte) { - t := self.bmt - i := self.cur +func (h *Hasher) Sum(b []byte) (r []byte) { + t := h.bmt + i := h.cur n := t.leaves[i] j := i // must run strictly before all nodes calculate // datanodes are guaranteed to have a parent - if len(self.segment) > self.size && i > 0 && n.parent != nil { + if len(h.segment) > h.size && i > 0 && n.parent != nil { n = n.parent } else { i *= 2 } - d := self.finalise(n, i) - self.writeSegment(j, self.segment, d) - c := <-self.result - self.releaseTree() + d := h.finalise(n, i) + h.writeSegment(j, h.segment, d) + c := <-h.result + h.releaseTree() // sha3(length + BMT(pure_chunk)) - if self.blockLength == nil { + if h.blockLength == nil { return c } - res := self.pool.hasher() + res := h.pool.hasher() res.Reset() - res.Write(self.blockLength) + res.Write(h.blockLength) res.Write(c) return res.Sum(nil) } @@ -321,8 +321,8 @@ func (self *Hasher) Sum(b []byte) (r []byte) { // Hash waits for the hasher result and returns it // caller must call this on a BMT Hasher being written to -func (self *Hasher) Hash() []byte { - return <-self.result +func (h *Hasher) Hash() []byte { + return <-h.result } // Hasher implements the io.Writer interface @@ -330,16 +330,16 @@ func (self *Hasher) Hash() []byte { // Write fills the buffer to hash // with every full segment complete launches a hasher go routine // that shoots up the BMT -func (self *Hasher) Write(b []byte) (int, error) { +func (h *Hasher) Write(b []byte) (int, error) { l := len(b) if l <= 0 { return 0, nil } - s := self.segment - i := self.cur - count := (self.count + 1) / 2 - need := self.count*self.size - self.cur*2*self.size - size := self.size + s := h.segment + i := h.cur + count := (h.count + 1) / 2 + need := h.count*h.size - h.cur*2*h.size + size := h.size if need > size { size *= 2 } @@ -356,7 +356,7 @@ func (self *Hasher) Write(b []byte) (int, error) { // read full segments and the last possibly partial segment for need > 0 && i < count-1 { // push all finished chunks we read - self.writeSegment(i, s, self.depth) + h.writeSegment(i, s, h.depth) need -= size if need < 0 { size += need @@ -365,8 +365,8 @@ func (self *Hasher) Write(b []byte) (int, error) { rest += size i++ } - self.segment = s - self.cur = i + h.segment = s + h.cur = i // otherwise, we can assume len(s) == 0, so all buffer is read and chunk is not yet full return l, nil } @@ -376,8 +376,8 @@ func (self *Hasher) Write(b []byte) (int, error) { // ReadFrom reads from io.Reader and appends to the data to hash using Write // it reads so that chunk to hash is maximum length or reader reaches EOF // caller must Reset the hasher prior to call -func (self *Hasher) ReadFrom(r io.Reader) (m int64, err error) { - bufsize := self.size*self.count - self.size*self.cur - len(self.segment) +func (h *Hasher) ReadFrom(r io.Reader) (m int64, err error) { + bufsize := h.size*h.count - h.size*h.cur - len(h.segment) buf := make([]byte, bufsize) var read int for { @@ -385,7 +385,7 @@ func (self *Hasher) ReadFrom(r io.Reader) (m int64, err error) { n, err = r.Read(buf) read += n if err == io.EOF || read == len(buf) { - hash := self.Sum(buf[:n]) + hash := h.Sum(buf[:n]) if read == len(buf) { err = NewEOC(hash) } @@ -394,7 +394,7 @@ func (self *Hasher) ReadFrom(r io.Reader) (m int64, err error) { if err != nil { break } - n, err = self.Write(buf[:n]) + n, err = h.Write(buf[:n]) if err != nil { break } @@ -403,9 +403,9 @@ func (self *Hasher) ReadFrom(r io.Reader) (m int64, err error) { } // Reset needs to be called before writing to the hasher -func (self *Hasher) Reset() { - self.getTree() - self.blockLength = nil +func (h *Hasher) Reset() { + h.getTree() + h.blockLength = nil } // Hasher implements the SwarmHash interface @@ -413,52 +413,52 @@ func (self *Hasher) Reset() { // ResetWithLength needs to be called before writing to the hasher // the argument is supposed to be the byte slice binary representation of // the length of the data subsumed under the hash -func (self *Hasher) ResetWithLength(l []byte) { - self.Reset() - self.blockLength = l +func (h *Hasher) ResetWithLength(l []byte) { + h.Reset() + h.blockLength = l } // Release gives back the Tree to the pool whereby it unlocks // it resets tree, segment and index -func (self *Hasher) releaseTree() { - if self.bmt != nil { - n := self.bmt.leaves[self.cur] +func (h *Hasher) releaseTree() { + if h.bmt != nil { + n := h.bmt.leaves[h.cur] for ; n != nil; n = n.parent { n.unbalanced = false if n.parent != nil { n.root = false } } - self.pool.Release(self.bmt) - self.bmt = nil + h.pool.Release(h.bmt) + h.bmt = nil } - self.cur = 0 - self.segment = nil + h.cur = 0 + h.segment = nil } -func (self *Hasher) writeSegment(i int, s []byte, d int) { - h := self.pool.hasher() - n := self.bmt.leaves[i] +func (h *Hasher) writeSegment(i int, s []byte, d int) { + hash := h.pool.hasher() + n := h.bmt.leaves[i] - if len(s) > self.size && n.parent != nil { + if len(s) > h.size && n.parent != nil { go func() { - h.Reset() - h.Write(s) - s = h.Sum(nil) + hash.Reset() + hash.Write(s) + s = hash.Sum(nil) if n.root { - self.result <- s + h.result <- s return } - self.run(n.parent, h, d, n.index, s) + h.run(n.parent, hash, d, n.index, s) }() return } - go self.run(n, h, d, i*2, s) + go h.run(n, hash, d, i*2, s) } -func (self *Hasher) run(n *Node, h hash.Hash, d int, i int, s []byte) { +func (h *Hasher) run(n *Node, hash hash.Hash, d int, i int, s []byte) { isLeft := i%2 == 0 for { if isLeft { @@ -470,18 +470,18 @@ func (self *Hasher) run(n *Node, h hash.Hash, d int, i int, s []byte) { return } if !n.unbalanced || !isLeft || i == 0 && d == 0 { - h.Reset() - h.Write(n.left) - h.Write(n.right) - s = h.Sum(nil) + hash.Reset() + hash.Write(n.left) + hash.Write(n.right) + s = hash.Sum(nil) } else { s = append(n.left, n.right...) } - self.hash = s + h.hash = s if n.root { - self.result <- s + h.result <- s return } @@ -492,20 +492,20 @@ func (self *Hasher) run(n *Node, h hash.Hash, d int, i int, s []byte) { } // getTree obtains a BMT resource by reserving one from the pool -func (self *Hasher) getTree() *Tree { - if self.bmt != nil { - return self.bmt +func (h *Hasher) getTree() *Tree { + if h.bmt != nil { + return h.bmt } - t := self.pool.Reserve() - self.bmt = t + t := h.pool.Reserve() + h.bmt = t return t } // atomic bool toggle implementing a concurrent reusable 2-state object // atomic addint with %2 implements atomic bool toggle // it returns true if the toggler just put it in the active/waiting state -func (self *Node) toggle() bool { - return atomic.AddInt32(&self.state, 1)%2 == 1 +func (n *Node) toggle() bool { + return atomic.AddInt32(&n.state, 1)%2 == 1 } func hashstr(b []byte) string { @@ -525,7 +525,7 @@ func depth(n int) (d int) { // finalise is following the zigzags on the tree belonging // to the final datasegment -func (self *Hasher) finalise(n *Node, i int) (d int) { +func (h *Hasher) finalise(n *Node, i int) (d int) { isLeft := i%2 == 0 for { // when the final segment's path is going via left segments @@ -550,8 +550,8 @@ type EOC struct { } // Error returns the error string -func (self *EOC) Error() string { - return fmt.Sprintf("hasher limit reached, chunk hash: %x", self.Hash) +func (e *EOC) Error() string { + return fmt.Sprintf("hasher limit reached, chunk hash: %x", e.Hash) } // NewEOC creates new end of chunk error with the hash diff --git a/build/ci.go b/build/ci.go index 303cf18421..9bd73fad24 100644 --- a/build/ci.go +++ b/build/ci.go @@ -731,7 +731,7 @@ func doAndroidArchive(cmdline []string) { // Build the Android archive and Maven resources build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind")) build.MustRun(gomobileTool("init", "--ndk", os.Getenv("ANDROID_NDK"))) - build.MustRun(gomobileTool("bind", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/EthereumCommonwealth/go-callisto/mobile")) + build.MustRun(gomobileTool("bind", "-ldflags", "-s -w", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/EthereumCommonwealth/go-callisto/mobile")) if *local { // If we're building locally, copy bundle to build dir and skip Maven @@ -755,14 +755,18 @@ func doAndroidArchive(cmdline []string) { os.Rename(archive, meta.Package+".aar") if *signer != "" && *deploy != "" { // Import the signing key into the local GPG instance - if b64key := os.Getenv(*signer); b64key != "" { - key, err := base64.StdEncoding.DecodeString(b64key) - if err != nil { - log.Fatalf("invalid base64 %s", *signer) - } - gpg := exec.Command("gpg", "--import") - gpg.Stdin = bytes.NewReader(key) - build.MustRun(gpg) + b64key := os.Getenv(*signer) + key, err := base64.StdEncoding.DecodeString(b64key) + if err != nil { + log.Fatalf("invalid base64 %s", *signer) + } + gpg := exec.Command("gpg", "--import") + gpg.Stdin = bytes.NewReader(key) + build.MustRun(gpg) + + keyID, err := build.PGPKeyID(string(key)) + if err != nil { + log.Fatal(err) } // Upload the artifacts to Sonatype and/or Maven Central repo := *deploy + "/service/local/staging/deploy/maven2" @@ -771,6 +775,7 @@ func doAndroidArchive(cmdline []string) { } build.MustRunCommand("mvn", "gpg:sign-and-deploy-file", "-e", "-X", "-settings=build/mvn.settings", "-Durl="+repo, "-DrepositoryId=ossrh", + "-Dgpg.keyname="+keyID, "-DpomFile="+meta.Package+".pom", "-Dfile="+meta.Package+".aar") } } @@ -852,7 +857,7 @@ func doXCodeFramework(cmdline []string) { // Build the iOS XCode framework build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind")) build.MustRun(gomobileTool("init")) - bind := gomobileTool("bind", "--target", "ios", "--tags", "ios", "-v", "github.com/EthereumCommonwealth/go-callisto/mobile") + bind := gomobileTool("bind", "-ldflags", "-s -w", "--target", "ios", "--tags", "ios", "-v", "github.com/EthereumCommonwealth/go-callisto/mobile") if *local { // If we're building locally, use the build folder and stop afterwards diff --git a/cmd/clef/README.md b/cmd/clef/README.md index 93799a7610..027c22c98f 100644 --- a/cmd/clef/README.md +++ b/cmd/clef/README.md @@ -12,6 +12,11 @@ synchronised with the chain or a particular Ethereum node that has no built-in ( Clef can run as a daemon on the same machine, or off a usb-stick like [usb armory](https://inversepath.com/usbarmory), or a separate VM in a [QubesOS](https://www.qubes-os.org/) type os setup. +Check out + +* the [tutorial](tutorial.md) for some concrete examples on how the signer works. +* the [setup docs](docs/setup.md) for some information on how to configure it to work on QubesOS or USBArmory. + ## Command line flags Clef accepts the following command line options: @@ -49,7 +54,6 @@ Example: signer -keystore /my/keystore -chainid 4 ``` -Check out the [tutorial](tutorial.md) for some concrete examples on how the signer works. ## Security model @@ -862,3 +866,12 @@ A UI should conform to the following rules. along with the UI. +### UI Implementations + +There are a couple of implementation for a UI. We'll try to keep this list up to date. + +| Name | Repo | UI type| No external resources| Blocky support| Verifies permissions | Hash information | No secondary storage | Statically linked| Can modify parameters| +| ---- | ---- | -------| ---- | ---- | ---- |---- | ---- | ---- | ---- | +| QtSigner| https://github.com/holiman/qtsigner/| Python3/QT-based| :+1:| :+1:| :+1:| :+1:| :+1:| :x: | :+1: (partially)| +| GtkSigner| https://github.com/holiman/gtksigner| Python3/GTK-based| :+1:| :x:| :x:| :+1:| :+1:| :x: | :x: | +| Frame | https://github.com/floating/frame/commits/go-signer| Electron-based| :x:| :x:| :x:| :x:| ?| :x: | :x: | diff --git a/cmd/clef/docs/qubes/clef_qubes_http.png b/cmd/clef/docs/qubes/clef_qubes_http.png new file mode 100644 index 0000000000..a641e1987f Binary files /dev/null and b/cmd/clef/docs/qubes/clef_qubes_http.png differ diff --git a/cmd/clef/docs/qubes/clef_qubes_qrexec.png b/cmd/clef/docs/qubes/clef_qubes_qrexec.png new file mode 100644 index 0000000000..f57fc8933e Binary files /dev/null and b/cmd/clef/docs/qubes/clef_qubes_qrexec.png differ diff --git a/cmd/clef/docs/qubes/qrexec-example.png b/cmd/clef/docs/qubes/qrexec-example.png new file mode 100644 index 0000000000..0d86fde19d Binary files /dev/null and b/cmd/clef/docs/qubes/qrexec-example.png differ diff --git a/cmd/clef/docs/qubes/qubes-client.py b/cmd/clef/docs/qubes/qubes-client.py new file mode 100644 index 0000000000..93a74b899b --- /dev/null +++ b/cmd/clef/docs/qubes/qubes-client.py @@ -0,0 +1,23 @@ +""" +This implements a dispatcher which listens to localhost:8550, and proxies +requests via qrexec to the service qubes.EthSign on a target domain +""" + +import http.server +import socketserver,subprocess + +PORT=8550 +TARGET_DOMAIN= 'debian-work' + +class Dispatcher(http.server.BaseHTTPRequestHandler): + def do_POST(self): + post_data = self.rfile.read(int(self.headers['Content-Length'])) + p = subprocess.Popen(['/usr/bin/qrexec-client-vm',TARGET_DOMAIN,'qubes.Clefsign'],stdin=subprocess.PIPE, stdout=subprocess.PIPE) + output = p.communicate(post_data)[0] + self.wfile.write(output) + + +with socketserver.TCPServer(("",PORT), Dispatcher) as httpd: + print("Serving at port", PORT) + httpd.serve_forever() + diff --git a/cmd/clef/docs/qubes/qubes.Clefsign b/cmd/clef/docs/qubes/qubes.Clefsign new file mode 100644 index 0000000000..9b5af7b4fe --- /dev/null +++ b/cmd/clef/docs/qubes/qubes.Clefsign @@ -0,0 +1,16 @@ +#!/bin/bash + +SIGNER_BIN="/home/user/tools/clef/clef" +SIGNER_CMD="/home/user/tools/gtksigner/gtkui.py -s $SIGNER_BIN" + +# Start clef if not already started +if [ ! -S /home/user/.clef/clef.ipc ]; then + $SIGNER_CMD & + sleep 1 +fi + +# Should be started by now +if [ -S /home/user/.clef/clef.ipc ]; then + # Post incoming request to HTTP channel + curl -H "Content-Type: application/json" -X POST -d @- http://localhost:8550 2>/dev/null +fi diff --git a/cmd/clef/docs/qubes/qubes_newaccount-1.png b/cmd/clef/docs/qubes/qubes_newaccount-1.png new file mode 100644 index 0000000000..598dbbee7a Binary files /dev/null and b/cmd/clef/docs/qubes/qubes_newaccount-1.png differ diff --git a/cmd/clef/docs/qubes/qubes_newaccount-2.png b/cmd/clef/docs/qubes/qubes_newaccount-2.png new file mode 100644 index 0000000000..cd762a1934 Binary files /dev/null and b/cmd/clef/docs/qubes/qubes_newaccount-2.png differ diff --git a/cmd/clef/docs/setup.md b/cmd/clef/docs/setup.md new file mode 100644 index 0000000000..33d2b0381f --- /dev/null +++ b/cmd/clef/docs/setup.md @@ -0,0 +1,198 @@ +# Setting up Clef + +This document describes how Clef can be used in a more secure manner than executing it from your everyday laptop, +in order to ensure that the keys remain safe in the event that your computer should get compromised. + +## Qubes OS + + +### Background + +The Qubes operating system is based around virtual machines (qubes), where a set of virtual machines are configured, typically for +different purposes such as: + +- personal + - Your personal email, browsing etc +- work + - Work email etc +- vault + - a VM without network access, where gpg-keys and/or keepass credentials are stored. + +A couple of dedicated virtual machines handle externalities: + +- sys-net provides networking to all other (network-enabled) machines +- sys-firewall handles firewall rules +- sys-usb handles USB devices, and can map usb-devices to certain qubes. + +The goal of this document is to describe how we can set up clef to provide secure transaction +signing from a `vault` vm, to another networked qube which runs Dapps. + +### Setup + +There are two ways that this can be achieved: integrated via Qubes or integrated via networking. + + +#### 1. Qubes Integrated + +Qubes provdes a facility for inter-qubes communication via `qrexec`. A qube can request to make a cross-qube RPC request +to another qube. The OS then asks the user if the call is permitted. + +![Example](qubes/qrexec-example.png) + +A policy-file can be created to allow such interaction. On the `target` domain, a service is invoked which can read the +`stdin` from the `client` qube. + +This is how [Split GPG](https://www.qubes-os.org/doc/split-gpg/) is implemented. We can set up Clef the same way: + +##### Server + +![Clef via qrexec](qubes/clef_qubes_qrexec.png) + +On the `target` qubes, we need to define the rpc service. + +[qubes.Clefsign](qubes/qubes.Clefsign): + +```bash +#!/bin/bash + +SIGNER_BIN="/home/user/tools/clef/clef" +SIGNER_CMD="/home/user/tools/gtksigner/gtkui.py -s $SIGNER_BIN" + +# Start clef if not already started +if [ ! -S /home/user/.clef/clef.ipc ]; then + $SIGNER_CMD & + sleep 1 +fi + +# Should be started by now +if [ -S /home/user/.clef/clef.ipc ]; then + # Post incoming request to HTTP channel + curl -H "Content-Type: application/json" -X POST -d @- http://localhost:8550 2>/dev/null +fi + +``` +This RPC service is not complete (see notes about HTTP headers below), but works as a proof-of-concept. +It will forward the data received on `stdin` (forwarded by the OS) to Clef's HTTP channel. + +It would have been possible to send data directly to the `/home/user/.clef/.clef.ipc` +socket via e.g `nc -U /home/user/.clef/clef.ipc`, but the reason for sending the request +data over `HTTP` instead of `IPC` is that we want the ability to forward `HTTP` headers. + +To enable the service: + +``` bash +sudo cp qubes.Clefsign /etc/qubes-rpc/ +sudo chmod +x /etc/qubes-rpc/ qubes.Clefsign +``` + +This setup uses [gtksigner](https://github.com/holiman/gtksigner), which is a very minimal GTK-based UI that works well +with minimal requirements. + +##### Client + + +On the `client` qube, we need to create a listener which will receive the request from the Dapp, and proxy it. + + +[qubes-client.py](qubes/client/qubes-client.py): + +```python + +""" +This implements a dispatcher which listens to localhost:8550, and proxies +requests via qrexec to the service qubes.EthSign on a target domain +""" + +import http.server +import socketserver,subprocess + +PORT=8550 +TARGET_DOMAIN= 'debian-work' + +class Dispatcher(http.server.BaseHTTPRequestHandler): + def do_POST(self): + post_data = self.rfile.read(int(self.headers['Content-Length'])) + p = subprocess.Popen(['/usr/bin/qrexec-client-vm',TARGET_DOMAIN,'qubes.Clefsign'],stdin=subprocess.PIPE, stdout=subprocess.PIPE) + output = p.communicate(post_data)[0] + self.wfile.write(output) + + +with socketserver.TCPServer(("",PORT), Dispatcher) as httpd: + print("Serving at port", PORT) + httpd.serve_forever() + + +``` + +#### Testing + +To test the flow, if we have set up `debian-work` as the `target`, we can do + +```bash +$ cat newaccnt.json +{ "id": 0, "jsonrpc": "2.0","method": "account_new","params": []} + +$ cat newaccnt.json| qrexec-client-vm debian-work qubes.Clefsign +``` + +This should pop up first a dialog to allow the IPC call: + +![one](qubes/qubes_newaccount-1.png) + +Followed by a GTK-dialog to approve the operation + +![two](qubes/qubes_newaccount-2.png) + +To test the full flow, we use the client wrapper. Start it on the `client` qube: +``` +[user@work qubes]$ python3 qubes-client.py +``` + +Make the request over http (`client` qube): +``` +[user@work clef]$ cat newaccnt.json | curl -X POST -d @- http://localhost:8550 +``` +And it should show the same popups again. + +##### Pros and cons + +The benefits of this setup are: + +- This is the qubes-os intended model for inter-qube communication, +- and thus benefits from qubes-os dialogs and policies for user approval + +However, it comes with a couple of drawbacks: + +- The `qubes-gpg-client` must forward the http request via RPC to the `target` qube. When doing so, the proxy + will either drop important headers, or replace them. + - The `Host` header is most likely `localhost` + - The `Origin` header must be forwarded + - Information about the remote ip must be added as a `X-Forwarded-For`. However, Clef cannot always trust an `XFF` header, + since malicious clients may lie about `XFF` in order to fool the http server into believing it comes from another address. +- Even with a policy in place to allow rpc-calls between `caller` and `target`, there will be several popups: + - One qubes-specific where the user specifies the `target` vm + - One clef-specific to approve the transaction + + +#### 2. Network integrated + +The second way to set up Clef on a qubes system is to allow networking, and have Clef listen to a port which is accessible +form other qubes. + +![Clef via http](qubes/clef_qubes_http.png) + + + + +## USBArmory + +The [USB armory](https://inversepath.com/usbarmory) is an open source hardware design with an 800 Mhz ARM processor. It is a pocket-size +computer. When inserted into a laptop, it identifies itself as a USB network interface, basically adding another network +to your computer. Over this new network interface, you can SSH into the device. + +Running Clef off a USB armory means that you can use the armory as a very versatile offline computer, which only +ever connects to a local network between your computer and the device itself. + +Needless to say, the while this model should be fairly secure against remote attacks, an attacker with physical access +to the USB Armory would trivially be able to extract the contents of the device filesystem. + diff --git a/cmd/clef/main.go b/cmd/clef/main.go index 6e4c1724aa..48f29d50a0 100644 --- a/cmd/clef/main.go +++ b/cmd/clef/main.go @@ -47,11 +47,11 @@ import ( "gopkg.in/urfave/cli.v1" ) -// ExternalApiVersion -- see extapi_changelog.md -const ExternalApiVersion = "2.0.0" +// ExternalAPIVersion -- see extapi_changelog.md +const ExternalAPIVersion = "2.0.0" -// InternalApiVersion -- see intapi_changelog.md -const InternalApiVersion = "2.0.0" +// InternalAPIVersion -- see intapi_changelog.md +const InternalAPIVersion = "2.0.0" const legalWarning = ` WARNING! @@ -398,10 +398,10 @@ func signer(c *cli.Context) error { } // register signer API with server var ( - extapiUrl = "n/a" - ipcApiUrl = "n/a" + extapiURL = "n/a" + ipcapiURL = "n/a" ) - rpcApi := []rpc.API{ + rpcAPI := []rpc.API{ { Namespace: "account", Public: true, @@ -415,12 +415,12 @@ func signer(c *cli.Context) error { // start http server httpEndpoint := fmt.Sprintf("%s:%d", c.String(utils.RPCListenAddrFlag.Name), c.Int(rpcPortFlag.Name)) - listener, _, err := rpc.StartHTTPEndpoint(httpEndpoint, rpcApi, []string{"account"}, cors, vhosts) + listener, _, err := rpc.StartHTTPEndpoint(httpEndpoint, rpcAPI, []string{"account"}, cors, vhosts) if err != nil { utils.Fatalf("Could not start RPC api: %v", err) } - extapiUrl = fmt.Sprintf("http://%s", httpEndpoint) - log.Info("HTTP endpoint opened", "url", extapiUrl) + extapiURL = fmt.Sprintf("http://%s", httpEndpoint) + log.Info("HTTP endpoint opened", "url", extapiURL) defer func() { listener.Close() @@ -430,19 +430,19 @@ func signer(c *cli.Context) error { } if !c.Bool(utils.IPCDisabledFlag.Name) { if c.IsSet(utils.IPCPathFlag.Name) { - ipcApiUrl = c.String(utils.IPCPathFlag.Name) + ipcapiURL = c.String(utils.IPCPathFlag.Name) } else { - ipcApiUrl = filepath.Join(configDir, "clef.ipc") + ipcapiURL = filepath.Join(configDir, "clef.ipc") } - listener, _, err := rpc.StartIPCEndpoint(ipcApiUrl, rpcApi) + listener, _, err := rpc.StartIPCEndpoint(ipcapiURL, rpcAPI) if err != nil { utils.Fatalf("Could not start IPC api: %v", err) } - log.Info("IPC endpoint opened", "url", ipcApiUrl) + log.Info("IPC endpoint opened", "url", ipcapiURL) defer func() { listener.Close() - log.Info("IPC endpoint closed", "url", ipcApiUrl) + log.Info("IPC endpoint closed", "url", ipcapiURL) }() } @@ -453,10 +453,10 @@ func signer(c *cli.Context) error { } ui.OnSignerStartup(core.StartupInfo{ Info: map[string]interface{}{ - "extapi_version": ExternalApiVersion, - "intapi_version": InternalApiVersion, - "extapi_http": extapiUrl, - "extapi_ipc": ipcApiUrl, + "extapi_version": ExternalAPIVersion, + "intapi_version": InternalAPIVersion, + "extapi_http": extapiURL, + "extapi_ipc": ipcapiURL, }, }) diff --git a/cmd/evm/json_logger.go b/cmd/evm/json_logger.go index 6127514536..521eed2b4b 100644 --- a/cmd/evm/json_logger.go +++ b/cmd/evm/json_logger.go @@ -32,6 +32,8 @@ type JSONLogger struct { cfg *vm.LogConfig } +// NewJSONLogger creates a new EVM tracer that prints execution steps as JSON objects +// into the provided stream. func NewJSONLogger(cfg *vm.LogConfig, writer io.Writer) *JSONLogger { return &JSONLogger{json.NewEncoder(writer), cfg} } diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index aeb7a4e37f..7d9b5020fa 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -21,12 +21,12 @@ import ( "encoding/json" "fmt" "io/ioutil" + "math/big" "os" + goruntime "runtime" "runtime/pprof" "time" - goruntime "runtime" - "github.com/EthereumCommonwealth/go-callisto/cmd/evm/internal/compiler" "github.com/EthereumCommonwealth/go-callisto/cmd/utils" "github.com/EthereumCommonwealth/go-callisto/common" @@ -86,6 +86,7 @@ func runCmd(ctx *cli.Context) error { chainConfig *params.ChainConfig sender = common.BytesToAddress([]byte("sender")) receiver = common.BytesToAddress([]byte("receiver")) + blockNumber uint64 ) if ctx.GlobalBool(MachineFlag.Name) { tracer = NewJSONLogger(logconfig, os.Stdout) @@ -97,13 +98,13 @@ func runCmd(ctx *cli.Context) error { } if ctx.GlobalString(GenesisFlag.Name) != "" { gen := readGenesis(ctx.GlobalString(GenesisFlag.Name)) - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() genesis := gen.ToBlock(db) statedb, _ = state.New(genesis.Root(), state.NewDatabase(db)) chainConfig = gen.Config + blockNumber = gen.Number } else { - db, _ := ethdb.NewMemDatabase() - statedb, _ = state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) } if ctx.GlobalString(SenderFlag.Name) != "" { sender = common.HexToAddress(ctx.GlobalString(SenderFlag.Name)) @@ -156,11 +157,12 @@ func runCmd(ctx *cli.Context) error { initialGas := ctx.GlobalUint64(GasFlag.Name) runtimeConfig := runtime.Config{ - Origin: sender, - State: statedb, - GasLimit: initialGas, - GasPrice: utils.GlobalBig(ctx, PriceFlag.Name), - Value: utils.GlobalBig(ctx, ValueFlag.Name), + Origin: sender, + State: statedb, + GasLimit: initialGas, + GasPrice: utils.GlobalBig(ctx, PriceFlag.Name), + Value: utils.GlobalBig(ctx, ValueFlag.Name), + BlockNumber: new(big.Int).SetUint64(blockNumber), EVMConfig: vm.Config{ Tracer: tracer, Debug: ctx.GlobalBool(DebugFlag.Name) || ctx.GlobalBool(MachineFlag.Name), diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index 734d54a609..5284a30496 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -38,6 +38,8 @@ var stateTestCommand = cli.Command{ ArgsUsage: "", } +// StatetestResult contains the execution status after running a state test, any +// error that might have occurred and a dump of the final state if requested. type StatetestResult struct { Name string `json:"name"` Pass bool `json:"pass"` diff --git a/cmd/geth/accountcmd.go b/cmd/geth/accountcmd.go index bfc45d89c1..3154b43418 100644 --- a/cmd/geth/accountcmd.go +++ b/cmd/geth/accountcmd.go @@ -340,7 +340,7 @@ func importWallet(ctx *cli.Context) error { if len(keyfile) == 0 { utils.Fatalf("keyfile must be given as argument") } - keyJson, err := ioutil.ReadFile(keyfile) + keyJSON, err := ioutil.ReadFile(keyfile) if err != nil { utils.Fatalf("Could not read wallet file: %v", err) } @@ -349,7 +349,7 @@ func importWallet(ctx *cli.Context) error { passphrase := getPassPhrase("", false, 0, utils.MakePasswordList(ctx)) ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore) - acct, err := ks.ImportPreSaleKey(keyJson, passphrase) + acct, err := ks.ImportPreSaleKey(keyJSON, passphrase) if err != nil { utils.Fatalf("%v", err) } diff --git a/cmd/geth/bugcmd.go b/cmd/geth/bugcmd.go index 853e53a7e9..42373f9c9f 100644 --- a/cmd/geth/bugcmd.go +++ b/cmd/geth/bugcmd.go @@ -41,7 +41,7 @@ var bugCommand = cli.Command{ Category: "MISCELLANEOUS COMMANDS", } -const issueUrl = "https://github.com/EthereumCommonwealth/go-callisto/issues/new" +const issueURL = "https://github.com/EthereumCommonwealth/go-callisto/issues/new" // reportBug reports a bug by opening a new URL to the go-ethereum GH issue // tracker and setting default values as the issue body. @@ -58,8 +58,8 @@ func reportBug(ctx *cli.Context) error { fmt.Fprintln(&buff, header) // open a new GH issue - if !browser.Open(issueUrl + "?body=" + url.QueryEscape(buff.String())) { - fmt.Printf("Please file a new issue at %s using this template:\n\n%s", issueUrl, buff.String()) + if !browser.Open(issueURL + "?body=" + url.QueryEscape(buff.String())) { + fmt.Printf("Please file a new issue at %s using this template:\n\n%s", issueURL, buff.String()) } return nil } diff --git a/cmd/geth/dao_test.go b/cmd/geth/dao_test.go index 7366ebf336..1314a8043e 100644 --- a/cmd/geth/dao_test.go +++ b/cmd/geth/dao_test.go @@ -24,9 +24,8 @@ import ( "testing" "github.com/EthereumCommonwealth/go-callisto/common" - "github.com/EthereumCommonwealth/go-callisto/core" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/ethdb" - //"github.com/EthereumCommonwealth/go-callisto/params" ) // Genesis block for nodes which don't care about the DAO fork (i.e. not configured) @@ -131,8 +130,8 @@ func testDAOForkBlockNewChain(t *testing.T, test int, genesis string, expectBloc if genesis != "" { genesisHash = daoGenesisHash } - config, err := core.GetChainConfig(db, genesisHash) - if err != nil { + config := rawdb.ReadChainConfig(db, genesisHash) + if config == nil { t.Errorf("test %d: failed to retrieve chain config: %v", test, err) return // we want to return here, the other checks can't make it past this point (nil panic). } diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 1ec262d208..3ce24720af 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -223,6 +223,8 @@ func geth(ctx *cli.Context) error { // it unlocks any requested accounts, and starts the RPC/IPC interfaces and the // miner. func startNode(ctx *cli.Context, stack *node.Node) { + debug.Memsize.Add("node", stack) + // Start up the node itself utils.StartNode(stack) diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 6ae64586c4..62339ec0e2 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -29,6 +29,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common" "github.com/EthereumCommonwealth/go-callisto/core" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/crypto" "github.com/EthereumCommonwealth/go-callisto/ethdb" @@ -271,15 +272,13 @@ func ImportPreimages(db *ethdb.LDBDatabase, fn string) error { // Accumulate the preimages and flush when enough ws gathered preimages[crypto.Keccak256Hash(blob)] = common.CopyBytes(blob) if len(preimages) > 1024 { - if err := core.WritePreimages(db, 0, preimages); err != nil { - return err - } + rawdb.WritePreimages(db, 0, preimages) preimages = make(map[common.Hash][]byte) } } // Flush the last batch preimage data if len(preimages) > 0 { - return core.WritePreimages(db, 0, preimages) + rawdb.WritePreimages(db, 0, preimages) } return nil } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 8b08b37992..6c3e9cb5b2 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -158,11 +158,11 @@ var ( } FastSyncFlag = cli.BoolFlag{ Name: "fast", - Usage: "Enable fast syncing through state downloads", + Usage: "Enable fast syncing through state downloads (replaced by --syncmode)", } LightModeFlag = cli.BoolFlag{ Name: "light", - Usage: "Enable light client mode", + Usage: "Enable light client mode (replaced by --syncmode)", } defaultSyncMode = eth.DefaultConfig.SyncMode SyncModeFlag = TextMarshalerFlag{ diff --git a/cmd/wnode/main.go b/cmd/wnode/main.go index 02896e7851..0fd3346652 100644 --- a/cmd/wnode/main.go +++ b/cmd/wnode/main.go @@ -271,7 +271,9 @@ func initialize() { if *mailServerMode { shh.RegisterServer(&mailServer) - mailServer.Init(shh, *argDBPath, msPassword, *argServerPoW) + if err := mailServer.Init(shh, *argDBPath, msPassword, *argServerPoW); err != nil { + utils.Fatalf("Failed to init MailServer: %s", err) + } } server = &p2p.Server{ diff --git a/common/bytes.go b/common/bytes.go index ba00e8a4b2..e2adc80059 100644 --- a/common/bytes.go +++ b/common/bytes.go @@ -87,15 +87,13 @@ func Hex2BytesFixed(str string, flen int) []byte { h, _ := hex.DecodeString(str) if len(h) == flen { return h - } else { - if len(h) > flen { - return h[len(h)-flen:] - } else { - hh := make([]byte, flen) - copy(hh[flen-len(h):flen], h[:]) - return hh - } } + if len(h) > flen { + return h[len(h)-flen:] + } + hh := make([]byte, flen) + copy(hh[flen-len(h):flen], h[:]) + return hh } func RightPadBytes(slice []byte, l int) []byte { diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index fae965814a..aa1e0e3548 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -384,7 +384,7 @@ func (c *Clique) snapshot(chain consensus.ChainReader, number uint64, hash commo // If an on-disk checkpoint snapshot can be found, use that if number%checkpointInterval == 0 { if s, err := loadSnapshot(c.config, c.signatures, c.db, hash); err == nil { - log.Trace("Loaded voting snapshot form disk", "number", number, "hash", hash) + log.Trace("Loaded voting snapshot from disk", "number", number, "hash", hash) snap = s break } diff --git a/consensus/clique/snapshot_test.go b/consensus/clique/snapshot_test.go index 1775fa5075..122bc81fa3 100644 --- a/consensus/clique/snapshot_test.go +++ b/consensus/clique/snapshot_test.go @@ -24,6 +24,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common" "github.com/EthereumCommonwealth/go-callisto/core" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/crypto" "github.com/EthereumCommonwealth/go-callisto/ethdb" @@ -81,7 +82,7 @@ func (r *testerChainReader) GetBlock(common.Hash, uint64) *types.Block { panic func (r *testerChainReader) GetHeaderByHash(common.Hash) *types.Header { panic("not supported") } func (r *testerChainReader) GetHeaderByNumber(number uint64) *types.Header { if number == 0 { - return core.GetHeader(r.db, core.GetCanonicalHash(r.db, 0), 0) + return rawdb.ReadHeader(r.db, rawdb.ReadCanonicalHash(r.db, 0), 0) } panic("not supported") } @@ -351,7 +352,7 @@ func TestVoting(t *testing.T) { copy(genesis.ExtraData[extraVanity+j*common.AddressLength:], signer[:]) } // Create a pristine blockchain with the genesis injected - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() genesis.Commit(db) // Assemble a chain of headers from the cast votes diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go index d421a73fe4..bea4dd82c4 100644 --- a/consensus/ethash/ethash.go +++ b/consensus/ethash/ethash.go @@ -156,7 +156,7 @@ type lru struct { futureItem interface{} } -// newlru create a new least-recently-used cache for ither the verification caches +// newlru create a new least-recently-used cache for either the verification caches // or the mining datasets. func newlru(what string, maxItems int, new func(epoch uint64) interface{}) *lru { if maxItems <= 0 { diff --git a/core/bench_test.go b/core/bench_test.go index 77673a0704..8265c9efd3 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -26,6 +26,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common" "github.com/EthereumCommonwealth/go-callisto/common/math" "github.com/EthereumCommonwealth/go-callisto/consensus/ethash" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/core/vm" "github.com/EthereumCommonwealth/go-callisto/crypto" @@ -148,7 +149,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) { // Create the database in memory or in a temporary directory. var db ethdb.Database if !disk { - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() } else { dir, err := ioutil.TempDir("", "eth-core-bench") if err != nil { @@ -234,13 +235,15 @@ func makeChainForBench(db ethdb.Database, full bool, count uint64) { ReceiptHash: types.EmptyRootHash, } hash = header.Hash() - WriteHeader(db, header) - WriteCanonicalHash(db, hash, n) - WriteTd(db, hash, n, big.NewInt(int64(n+1))) + + rawdb.WriteHeader(db, header) + rawdb.WriteCanonicalHash(db, hash, n) + rawdb.WriteTd(db, hash, n, big.NewInt(int64(n+1))) + if full || n == 0 { block := types.NewBlockWithHeader(header) - WriteBody(db, hash, n, block.Body()) - WriteBlockReceipts(db, hash, n, nil) + rawdb.WriteBody(db, hash, n, block.Body()) + rawdb.WriteReceipts(db, hash, n, nil) } } } @@ -292,11 +295,10 @@ func benchReadChain(b *testing.B, full bool, count uint64) { header := chain.GetHeaderByNumber(n) if full { hash := header.Hash() - GetBody(db, hash, n) - GetBlockReceipts(db, hash, n) + rawdb.ReadBody(db, hash, n) + rawdb.ReadReceipts(db, hash, n) } } - chain.Stop() db.Close() } diff --git a/core/block_validator_test.go b/core/block_validator_test.go index 4a1fcce8a4..e6e381a12f 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -32,7 +32,7 @@ import ( func TestHeaderVerification(t *testing.T) { // Create a simple chain to verify var ( - testdb, _ = ethdb.NewMemDatabase() + testdb = ethdb.NewMemDatabase() gspec = &Genesis{Config: params.TestChainConfig} genesis = gspec.MustCommit(testdb) blocks, _ = GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), testdb, 8, nil) @@ -84,7 +84,7 @@ func TestHeaderConcurrentVerification32(t *testing.T) { testHeaderConcurrentVeri func testHeaderConcurrentVerification(t *testing.T, threads int) { // Create a simple chain to verify var ( - testdb, _ = ethdb.NewMemDatabase() + testdb = ethdb.NewMemDatabase() gspec = &Genesis{Config: params.TestChainConfig} genesis = gspec.MustCommit(testdb) blocks, _ = GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), testdb, 8, nil) @@ -156,7 +156,7 @@ func TestHeaderConcurrentAbortion32(t *testing.T) { testHeaderConcurrentAbortion func testHeaderConcurrentAbortion(t *testing.T, threads int) { // Create a simple chain to verify var ( - testdb, _ = ethdb.NewMemDatabase() + testdb = ethdb.NewMemDatabase() gspec = &Genesis{Config: params.TestChainConfig} genesis = gspec.MustCommit(testdb) blocks, _ = GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), testdb, 1024, nil) diff --git a/core/blockchain.go b/core/blockchain.go index f35b26a62a..a07b4d11e4 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -30,6 +30,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common" "github.com/EthereumCommonwealth/go-callisto/common/mclock" "github.com/EthereumCommonwealth/go-callisto/consensus" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/state" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/core/vm" @@ -202,7 +203,7 @@ func (bc *BlockChain) getProcInterrupt() bool { // assumes that the chain manager mutex is held. func (bc *BlockChain) loadLastState() error { // Restore the last known head block - head := GetHeadBlockHash(bc.db) + head := rawdb.ReadHeadBlockHash(bc.db) if head == (common.Hash{}) { // Corrupt or empty database, init from scratch log.Warn("Empty database, resetting chain") @@ -228,7 +229,7 @@ func (bc *BlockChain) loadLastState() error { // Restore the last known head header currentHeader := currentBlock.Header() - if head := GetHeadHeaderHash(bc.db); head != (common.Hash{}) { + if head := rawdb.ReadHeadHeaderHash(bc.db); head != (common.Hash{}) { if header := bc.GetHeaderByHash(head); header != nil { currentHeader = header } @@ -237,7 +238,7 @@ func (bc *BlockChain) loadLastState() error { // Restore the last known head fast block bc.currentFastBlock.Store(currentBlock) - if head := GetHeadFastBlockHash(bc.db); head != (common.Hash{}) { + if head := rawdb.ReadHeadFastBlockHash(bc.db); head != (common.Hash{}) { if block := bc.GetBlockByHash(head); block != nil { bc.currentFastBlock.Store(block) } @@ -269,7 +270,7 @@ func (bc *BlockChain) SetHead(head uint64) error { // Rewind the header chain, deleting all block bodies until then delFn := func(hash common.Hash, num uint64) { - DeleteBody(bc.db, hash, num) + rawdb.DeleteBody(bc.db, hash, num) } bc.hc.SetHead(head, delFn) currentHeader := bc.hc.CurrentHeader() @@ -303,12 +304,10 @@ func (bc *BlockChain) SetHead(head uint64) error { } currentBlock := bc.CurrentBlock() currentFastBlock := bc.CurrentFastBlock() - if err := WriteHeadBlockHash(bc.db, currentBlock.Hash()); err != nil { - log.Crit("Failed to reset head full block", "err", err) - } - if err := WriteHeadFastBlockHash(bc.db, currentFastBlock.Hash()); err != nil { - log.Crit("Failed to reset head fast block", "err", err) - } + + rawdb.WriteHeadBlockHash(bc.db, currentBlock.Hash()) + rawdb.WriteHeadFastBlockHash(bc.db, currentFastBlock.Hash()) + return bc.loadLastState() } @@ -406,9 +405,8 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error { if err := bc.hc.WriteTd(genesis.Hash(), genesis.NumberU64(), genesis.Difficulty()); err != nil { log.Crit("Failed to write genesis block TD", "err", err) } - if err := WriteBlock(bc.db, genesis); err != nil { - log.Crit("Failed to write genesis block", "err", err) - } + rawdb.WriteBlock(bc.db, genesis) + bc.genesisBlock = genesis bc.insert(bc.genesisBlock) bc.currentBlock.Store(bc.genesisBlock) @@ -474,24 +472,19 @@ func (bc *BlockChain) ExportN(w io.Writer, first uint64, last uint64) error { // Note, this function assumes that the `mu` mutex is held! func (bc *BlockChain) insert(block *types.Block) { // If the block is on a side chain or an unknown one, force other heads onto it too - updateHeads := GetCanonicalHash(bc.db, block.NumberU64()) != block.Hash() + updateHeads := rawdb.ReadCanonicalHash(bc.db, block.NumberU64()) != block.Hash() // Add the block to the canonical chain number scheme and mark as the head - if err := WriteCanonicalHash(bc.db, block.Hash(), block.NumberU64()); err != nil { - log.Crit("Failed to insert block number", "err", err) - } - if err := WriteHeadBlockHash(bc.db, block.Hash()); err != nil { - log.Crit("Failed to insert head block hash", "err", err) - } + rawdb.WriteCanonicalHash(bc.db, block.Hash(), block.NumberU64()) + rawdb.WriteHeadBlockHash(bc.db, block.Hash()) + bc.currentBlock.Store(block) // If the block is better than our head or is on a different chain, force update heads if updateHeads { bc.hc.SetCurrentHeader(block.Header()) + rawdb.WriteHeadFastBlockHash(bc.db, block.Hash()) - if err := WriteHeadFastBlockHash(bc.db, block.Hash()); err != nil { - log.Crit("Failed to insert head fast block hash", "err", err) - } bc.currentFastBlock.Store(block) } } @@ -509,7 +502,11 @@ func (bc *BlockChain) GetBody(hash common.Hash) *types.Body { body := cached.(*types.Body) return body } - body := GetBody(bc.db, hash, bc.hc.GetBlockNumber(hash)) + number := bc.hc.GetBlockNumber(hash) + if number == nil { + return nil + } + body := rawdb.ReadBody(bc.db, hash, *number) if body == nil { return nil } @@ -525,7 +522,11 @@ func (bc *BlockChain) GetBodyRLP(hash common.Hash) rlp.RawValue { if cached, ok := bc.bodyRLPCache.Get(hash); ok { return cached.(rlp.RawValue) } - body := GetBodyRLP(bc.db, hash, bc.hc.GetBlockNumber(hash)) + number := bc.hc.GetBlockNumber(hash) + if number == nil { + return nil + } + body := rawdb.ReadBodyRLP(bc.db, hash, *number) if len(body) == 0 { return nil } @@ -539,8 +540,7 @@ func (bc *BlockChain) HasBlock(hash common.Hash, number uint64) bool { if bc.blockCache.Contains(hash) { return true } - ok, _ := bc.db.Has(blockBodyKey(hash, number)) - return ok + return rawdb.HasBody(bc.db, hash, number) } // HasState checks if state trie is fully present in the database or not. @@ -567,7 +567,7 @@ func (bc *BlockChain) GetBlock(hash common.Hash, number uint64) *types.Block { if block, ok := bc.blockCache.Get(hash); ok { return block.(*types.Block) } - block := GetBlock(bc.db, hash, number) + block := rawdb.ReadBlock(bc.db, hash, number) if block == nil { return nil } @@ -578,13 +578,17 @@ func (bc *BlockChain) GetBlock(hash common.Hash, number uint64) *types.Block { // GetBlockByHash retrieves a block from the database by hash, caching it if found. func (bc *BlockChain) GetBlockByHash(hash common.Hash) *types.Block { - return bc.GetBlock(hash, bc.hc.GetBlockNumber(hash)) + number := bc.hc.GetBlockNumber(hash) + if number == nil { + return nil + } + return bc.GetBlock(hash, *number) } // GetBlockByNumber retrieves a block from the database by number, caching it // (associated with its hash) if found. func (bc *BlockChain) GetBlockByNumber(number uint64) *types.Block { - hash := GetCanonicalHash(bc.db, number) + hash := rawdb.ReadCanonicalHash(bc.db, number) if hash == (common.Hash{}) { return nil } @@ -593,21 +597,28 @@ func (bc *BlockChain) GetBlockByNumber(number uint64) *types.Block { // GetReceiptsByHash retrieves the receipts for all transactions in a given block. func (bc *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts { - return GetBlockReceipts(bc.db, hash, GetBlockNumber(bc.db, hash)) + number := rawdb.ReadHeaderNumber(bc.db, hash) + if number == nil { + return nil + } + return rawdb.ReadReceipts(bc.db, hash, *number) } // GetBlocksFromHash returns the block corresponding to hash and up to n-1 ancestors. // [deprecated by eth/62] func (bc *BlockChain) GetBlocksFromHash(hash common.Hash, n int) (blocks []*types.Block) { number := bc.hc.GetBlockNumber(hash) + if number == nil { + return nil + } for i := 0; i < n; i++ { - block := bc.GetBlock(hash, number) + block := bc.GetBlock(hash, *number) if block == nil { break } blocks = append(blocks, block) hash = block.ParentHash() - number-- + *number-- } return } @@ -712,12 +723,12 @@ func (bc *BlockChain) Rollback(chain []common.Hash) { if currentFastBlock := bc.CurrentFastBlock(); currentFastBlock.Hash() == hash { newFastBlock := bc.GetBlock(currentFastBlock.ParentHash(), currentFastBlock.NumberU64()-1) bc.currentFastBlock.Store(newFastBlock) - WriteHeadFastBlockHash(bc.db, newFastBlock.Hash()) + rawdb.WriteHeadFastBlockHash(bc.db, newFastBlock.Hash()) } if currentBlock := bc.CurrentBlock(); currentBlock.Hash() == hash { newBlock := bc.GetBlock(currentBlock.ParentHash(), currentBlock.NumberU64()-1) bc.currentBlock.Store(newBlock) - WriteHeadBlockHash(bc.db, newBlock.Hash()) + rawdb.WriteHeadBlockHash(bc.db, newBlock.Hash()) } } } @@ -802,15 +813,10 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ return i, fmt.Errorf("failed to set receipts data: %v", err) } // Write all the data out into the database - if err := WriteBody(batch, block.Hash(), block.NumberU64(), block.Body()); err != nil { - return i, fmt.Errorf("failed to write block body: %v", err) - } - if err := WriteBlockReceipts(batch, block.Hash(), block.NumberU64(), receipts); err != nil { - return i, fmt.Errorf("failed to write block receipts: %v", err) - } - if err := WriteTxLookupEntries(batch, block); err != nil { - return i, fmt.Errorf("failed to write lookup metadata: %v", err) - } + rawdb.WriteBody(batch, block.Hash(), block.NumberU64(), block.Body()) + rawdb.WriteReceipts(batch, block.Hash(), block.NumberU64(), receipts) + rawdb.WriteTxLookupEntries(batch, block) + stats.processed++ if batch.ValueSize() >= ethdb.IdealBatchSize { @@ -834,9 +840,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ if td := bc.GetTd(head.Hash(), head.NumberU64()); td != nil { // Rewind may have occurred, skip in that case currentFastBlock := bc.CurrentFastBlock() if bc.GetTd(currentFastBlock.Hash(), currentFastBlock.NumberU64()).Cmp(td) < 0 { - if err := WriteHeadFastBlockHash(bc.db, head.Hash()); err != nil { - log.Crit("Failed to update head fast block hash", "err", err) - } + rawdb.WriteHeadFastBlockHash(bc.db, head.Hash()) bc.currentFastBlock.Store(head) } } @@ -864,9 +868,8 @@ func (bc *BlockChain) WriteBlockWithoutState(block *types.Block, td *big.Int) (e if err := bc.hc.WriteTd(block.Hash(), block.NumberU64(), td); err != nil { return err } - if err := WriteBlock(bc.db, block); err != nil { - return err - } + rawdb.WriteBlock(bc.db, block) + return nil } @@ -894,9 +897,8 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. } // Write other block data using a batch. batch := bc.db.NewBatch() - if err := WriteBlock(batch, block); err != nil { - return NonStatTy, err - } + rawdb.WriteBlock(batch, block) + root, err := state.Commit(bc.chainConfig.IsEIP158(block.Number())) if err != nil { return NonStatTy, err @@ -953,9 +955,8 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. } } } - if err := WriteBlockReceipts(batch, block.Hash(), block.NumberU64(), receipts); err != nil { - return NonStatTy, err - } + rawdb.WriteReceipts(batch, block.Hash(), block.NumberU64(), receipts) + // If the total difficulty is higher than our known, add it to the canonical chain // Second clause in the if statement reduces the vulnerability to selfish mining. // Please refer to http://www.cs.cornell.edu/~ie53/publications/btcProcFC.pdf @@ -972,14 +973,10 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. return NonStatTy, err } } - // Write the positional metadata for transaction and receipt lookups - if err := WriteTxLookupEntries(batch, block); err != nil { - return NonStatTy, err - } - // Write hash preimages - if err := WritePreimages(bc.db, block.NumberU64(), state.Preimages()); err != nil { - return NonStatTy, err - } + // Write the positional metadata for transaction/receipt lookups and preimages + rawdb.WriteTxLookupEntries(batch, block) + rawdb.WritePreimages(batch, block.NumberU64(), state.Preimages()) + status = CanonStatTy } else { status = SideStatTy @@ -1256,9 +1253,13 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { // collectLogs collects the logs that were generated during the // processing of the block that corresponds with the given hash. // These logs are later announced as deleted. - collectLogs = func(h common.Hash) { + collectLogs = func(hash common.Hash) { // Coalesce logs and set 'Removed'. - receipts := GetBlockReceipts(bc.db, h, bc.hc.GetBlockNumber(h)) + number := bc.hc.GetBlockNumber(hash) + if number == nil { + return + } + receipts := rawdb.ReadReceipts(bc.db, hash, *number) for _, receipt := range receipts { for _, log := range receipt.Logs { del := *log @@ -1327,9 +1328,7 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { // insert the block in the canonical way, re-writing history bc.insert(newChain[i]) // write lookup entries for hash based transaction/receipt searches - if err := WriteTxLookupEntries(bc.db, newChain[i]); err != nil { - return err - } + rawdb.WriteTxLookupEntries(bc.db, newChain[i]) addedTxs = append(addedTxs, newChain[i].Transactions()...) } // calculate the difference between deleted and added transactions @@ -1337,7 +1336,7 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { // When transactions get deleted from the database that means the // receipts that were created in the fork must also be deleted for _, tx := range diff { - DeleteTxLookupEntry(bc.db, tx.Hash()) + rawdb.DeleteTxLookupEntry(bc.db, tx.Hash()) } if len(deletedLogs) > 0 { go bc.rmLogsFeed.Send(RemovedLogsEvent{deletedLogs}) diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 528aa65942..a6c4a2b9e6 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -26,6 +26,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common" "github.com/EthereumCommonwealth/go-callisto/consensus/ethash" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/state" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/core/vm" @@ -128,8 +129,8 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error { return err } blockchain.mu.Lock() - WriteTd(blockchain.db, block.Hash(), block.NumberU64(), new(big.Int).Add(block.Difficulty(), blockchain.GetTdByHash(block.ParentHash()))) - WriteBlock(blockchain.db, block) + rawdb.WriteTd(blockchain.db, block.Hash(), block.NumberU64(), new(big.Int).Add(block.Difficulty(), blockchain.GetTdByHash(block.ParentHash()))) + rawdb.WriteBlock(blockchain.db, block) statedb.Commit(false) blockchain.mu.Unlock() } @@ -146,8 +147,8 @@ func testHeaderChainImport(chain []*types.Header, blockchain *BlockChain) error } // Manually insert the header into the database, but don't reorganise (allows subsequent testing) blockchain.mu.Lock() - WriteTd(blockchain.db, header.Hash(), header.Number.Uint64(), new(big.Int).Add(header.Difficulty, blockchain.GetTdByHash(header.ParentHash))) - WriteHeader(blockchain.db, header) + rawdb.WriteTd(blockchain.db, header.Hash(), header.Number.Uint64(), new(big.Int).Add(header.Difficulty, blockchain.GetTdByHash(header.ParentHash))) + rawdb.WriteHeader(blockchain.db, header) blockchain.mu.Unlock() } return nil @@ -173,7 +174,7 @@ func TestLastBlock(t *testing.T) { if _, err := blockchain.InsertChain(blocks); err != nil { t.Fatalf("Failed to insert block: %v", err) } - if blocks[len(blocks)-1].Hash() != GetHeadBlockHash(blockchain.db) { + if blocks[len(blocks)-1].Hash() != rawdb.ReadHeadBlockHash(blockchain.db) { t.Fatalf("Write/Get HeadBlockHash failed") } } @@ -568,11 +569,11 @@ func testInsertNonceError(t *testing.T, full bool) { func TestFastVsFullChains(t *testing.T) { // Configure and generate a sample block chain var ( - gendb, _ = ethdb.NewMemDatabase() - key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - address = crypto.PubkeyToAddress(key.PublicKey) - funds = big.NewInt(1000000000) - gspec = &Genesis{ + gendb = ethdb.NewMemDatabase() + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + funds = big.NewInt(1000000000) + gspec = &Genesis{ Config: params.TestChainConfig, Alloc: GenesisAlloc{address: {Balance: funds}}, } @@ -598,7 +599,7 @@ func TestFastVsFullChains(t *testing.T) { } }) // Import the chain as an archive node for the comparison baseline - archiveDb, _ := ethdb.NewMemDatabase() + archiveDb := ethdb.NewMemDatabase() gspec.MustCommit(archiveDb) archive, _ := NewBlockChain(archiveDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) defer archive.Stop() @@ -607,7 +608,7 @@ func TestFastVsFullChains(t *testing.T) { t.Fatalf("failed to process block %d: %v", n, err) } // Fast import the chain as a non-archive node to test - fastDb, _ := ethdb.NewMemDatabase() + fastDb := ethdb.NewMemDatabase() gspec.MustCommit(fastDb) fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) defer fast.Stop() @@ -639,13 +640,13 @@ func TestFastVsFullChains(t *testing.T) { } else if types.CalcUncleHash(fblock.Uncles()) != types.CalcUncleHash(ablock.Uncles()) { t.Errorf("block #%d [%x]: uncles mismatch: have %v, want %v", num, hash, fblock.Uncles(), ablock.Uncles()) } - if freceipts, areceipts := GetBlockReceipts(fastDb, hash, GetBlockNumber(fastDb, hash)), GetBlockReceipts(archiveDb, hash, GetBlockNumber(archiveDb, hash)); types.DeriveSha(freceipts) != types.DeriveSha(areceipts) { + if freceipts, areceipts := rawdb.ReadReceipts(fastDb, hash, *rawdb.ReadHeaderNumber(fastDb, hash)), rawdb.ReadReceipts(archiveDb, hash, *rawdb.ReadHeaderNumber(archiveDb, hash)); types.DeriveSha(freceipts) != types.DeriveSha(areceipts) { t.Errorf("block #%d [%x]: receipts mismatch: have %v, want %v", num, hash, freceipts, areceipts) } } // Check that the canonical chains are the same between the databases for i := 0; i < len(blocks)+1; i++ { - if fhash, ahash := GetCanonicalHash(fastDb, uint64(i)), GetCanonicalHash(archiveDb, uint64(i)); fhash != ahash { + if fhash, ahash := rawdb.ReadCanonicalHash(fastDb, uint64(i)), rawdb.ReadCanonicalHash(archiveDb, uint64(i)); fhash != ahash { t.Errorf("block #%d: canonical hash mismatch: have %v, want %v", i, fhash, ahash) } } @@ -656,12 +657,12 @@ func TestFastVsFullChains(t *testing.T) { func TestLightVsFastVsFullChainHeads(t *testing.T) { // Configure and generate a sample block chain var ( - gendb, _ = ethdb.NewMemDatabase() - key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - address = crypto.PubkeyToAddress(key.PublicKey) - funds = big.NewInt(1000000000) - gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{address: {Balance: funds}}} - genesis = gspec.MustCommit(gendb) + gendb = ethdb.NewMemDatabase() + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + funds = big.NewInt(1000000000) + gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{address: {Balance: funds}}} + genesis = gspec.MustCommit(gendb) ) height := uint64(1024) blocks, receipts := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), gendb, int(height), nil) @@ -684,7 +685,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { } } // Import the chain as an archive node and ensure all pointers are updated - archiveDb, _ := ethdb.NewMemDatabase() + archiveDb := ethdb.NewMemDatabase() gspec.MustCommit(archiveDb) archive, _ := NewBlockChain(archiveDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) @@ -698,7 +699,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { assert(t, "archive", archive, height/2, height/2, height/2) // Import the chain as a non-archive node and ensure all pointers are updated - fastDb, _ := ethdb.NewMemDatabase() + fastDb := ethdb.NewMemDatabase() gspec.MustCommit(fastDb) fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) defer fast.Stop() @@ -718,7 +719,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { assert(t, "fast", fast, height/2, height/2, 0) // Import the chain as a light node and ensure all pointers are updated - lightDb, _ := ethdb.NewMemDatabase() + lightDb := ethdb.NewMemDatabase() gspec.MustCommit(lightDb) light, _ := NewBlockChain(lightDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) @@ -741,7 +742,7 @@ func TestChainTxReorgs(t *testing.T) { addr1 = crypto.PubkeyToAddress(key1.PublicKey) addr2 = crypto.PubkeyToAddress(key2.PublicKey) addr3 = crypto.PubkeyToAddress(key3.PublicKey) - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() gspec = &Genesis{ Config: params.TestChainConfig, GasLimit: 3141592, @@ -821,28 +822,28 @@ func TestChainTxReorgs(t *testing.T) { // removed tx for i, tx := range (types.Transactions{pastDrop, freshDrop}) { - if txn, _, _, _ := GetTransaction(db, tx.Hash()); txn != nil { + if txn, _, _, _ := rawdb.ReadTransaction(db, tx.Hash()); txn != nil { t.Errorf("drop %d: tx %v found while shouldn't have been", i, txn) } - if rcpt, _, _, _ := GetReceipt(db, tx.Hash()); rcpt != nil { + if rcpt, _, _, _ := rawdb.ReadReceipt(db, tx.Hash()); rcpt != nil { t.Errorf("drop %d: receipt %v found while shouldn't have been", i, rcpt) } } // added tx for i, tx := range (types.Transactions{pastAdd, freshAdd, futureAdd}) { - if txn, _, _, _ := GetTransaction(db, tx.Hash()); txn == nil { + if txn, _, _, _ := rawdb.ReadTransaction(db, tx.Hash()); txn == nil { t.Errorf("add %d: expected tx to be found", i) } - if rcpt, _, _, _ := GetReceipt(db, tx.Hash()); rcpt == nil { + if rcpt, _, _, _ := rawdb.ReadReceipt(db, tx.Hash()); rcpt == nil { t.Errorf("add %d: expected receipt to be found", i) } } // shared tx for i, tx := range (types.Transactions{postponed, swapped}) { - if txn, _, _, _ := GetTransaction(db, tx.Hash()); txn == nil { + if txn, _, _, _ := rawdb.ReadTransaction(db, tx.Hash()); txn == nil { t.Errorf("share %d: expected tx to be found", i) } - if rcpt, _, _, _ := GetReceipt(db, tx.Hash()); rcpt == nil { + if rcpt, _, _, _ := rawdb.ReadReceipt(db, tx.Hash()); rcpt == nil { t.Errorf("share %d: expected receipt to be found", i) } } @@ -853,7 +854,7 @@ func TestLogReorgs(t *testing.T) { var ( key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() // this code generates a log code = common.Hex2Bytes("60606040525b7f24ec1d3ff24c2f6ff210738839dbc339cd45a5294d85c79361016243157aae7b60405180905060405180910390a15b600a8060416000396000f360606040526008565b00") gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000)}}} @@ -897,7 +898,7 @@ func TestLogReorgs(t *testing.T) { func TestReorgSideEvent(t *testing.T) { var ( - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) gspec = &Genesis{ @@ -997,14 +998,14 @@ func TestCanonicalBlockRetrieval(t *testing.T) { // try to retrieve a block by its canonical hash and see if the block data can be retrieved. for { - ch := GetCanonicalHash(blockchain.db, block.NumberU64()) + ch := rawdb.ReadCanonicalHash(blockchain.db, block.NumberU64()) if ch == (common.Hash{}) { continue // busy wait for canonical hash to be written } if ch != block.Hash() { t.Fatalf("unknown canonical hash, want %s, got %s", block.Hash().Hex(), ch.Hex()) } - fb := GetBlock(blockchain.db, ch, block.NumberU64()) + fb := rawdb.ReadBlock(blockchain.db, ch, block.NumberU64()) if fb == nil { t.Fatalf("unable to retrieve block %d for canonical hash: %s", block.NumberU64(), ch.Hex()) } @@ -1025,7 +1026,7 @@ func TestCanonicalBlockRetrieval(t *testing.T) { func TestEIP155Transition(t *testing.T) { // Configure and generate a sample block chain var ( - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") address = crypto.PubkeyToAddress(key.PublicKey) funds = big.NewInt(1000000000) @@ -1129,7 +1130,7 @@ func TestEIP155Transition(t *testing.T) { func TestEIP161AccountRemoval(t *testing.T) { // Configure and generate a sample block chain var ( - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") address = crypto.PubkeyToAddress(key.PublicKey) funds = big.NewInt(1000000000) @@ -1201,7 +1202,7 @@ func TestBlockchainHeaderchainReorgConsistency(t *testing.T) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() genesis := new(Genesis).MustCommit(db) blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 64, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) @@ -1217,7 +1218,7 @@ func TestBlockchainHeaderchainReorgConsistency(t *testing.T) { } // Import the canonical and fork chain side by side, verifying the current block // and current header consistency - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() new(Genesis).MustCommit(diskdb) chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}) @@ -1246,7 +1247,7 @@ func TestTrieForkGC(t *testing.T) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() genesis := new(Genesis).MustCommit(db) blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 2*triesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) @@ -1261,7 +1262,7 @@ func TestTrieForkGC(t *testing.T) { forks[i] = fork[0] } // Import the canonical and fork chain side by side, forcing the trie cache to cache both - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() new(Genesis).MustCommit(diskdb) chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}) @@ -1292,7 +1293,7 @@ func TestLargeReorgTrieGC(t *testing.T) { // Generate the original common chain segment and the two competing forks engine := ethash.NewFaker() - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() genesis := new(Genesis).MustCommit(db) shared, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 64, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) @@ -1300,7 +1301,7 @@ func TestLargeReorgTrieGC(t *testing.T) { competitor, _ := GenerateChain(params.TestChainConfig, shared[len(shared)-1], engine, db, 2*triesInMemory+1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{3}) }) // Import the shared chain and the original canonical one - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() new(Genesis).MustCommit(diskdb) chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}) @@ -1360,7 +1361,7 @@ func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks in ) // Generate the original common chain segment and the two competing forks engine := ethash.NewFaker() - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() genesis := gspec.MustCommit(db) blockGenerator := func(i int, block *BlockGen) { @@ -1382,7 +1383,7 @@ func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks in b.ResetTimer() for i := 0; i < b.N; i++ { // Import the shared chain and the original canonical one - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() gspec.MustCommit(diskdb) chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}) diff --git a/core/chain_indexer.go b/core/chain_indexer.go index eb6e88da54..319b55eb33 100644 --- a/core/chain_indexer.go +++ b/core/chain_indexer.go @@ -24,6 +24,7 @@ import ( "time" "github.com/EthereumCommonwealth/go-callisto/common" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/ethdb" "github.com/EthereumCommonwealth/go-callisto/event" @@ -206,7 +207,7 @@ func (c *ChainIndexer) eventLoop(currentHeader *types.Header, events chan ChainE // TODO(karalabe): This operation is expensive and might block, causing the event system to // potentially also lock up. We need to do with on a different thread somehow. - if h := FindCommonAncestor(c.chainDb, prevHeader, header); h != nil { + if h := rawdb.FindCommonAncestor(c.chainDb, prevHeader, header); h != nil { c.newHead(h.Number.Uint64(), true) } } @@ -349,11 +350,11 @@ func (c *ChainIndexer) processSection(section uint64, lastHead common.Hash) (com } for number := section * c.sectionSize; number < (section+1)*c.sectionSize; number++ { - hash := GetCanonicalHash(c.chainDb, number) + hash := rawdb.ReadCanonicalHash(c.chainDb, number) if hash == (common.Hash{}) { return common.Hash{}, fmt.Errorf("canonical block #%d unknown", number) } - header := GetHeader(c.chainDb, hash, number) + header := rawdb.ReadHeader(c.chainDb, hash, number) if header == nil { return common.Hash{}, fmt.Errorf("block #%d [%x…] not found", number, hash[:4]) } else if header.ParentHash != lastHead { diff --git a/core/chain_indexer_test.go b/core/chain_indexer_test.go index 17fe34abb4..c22f79557c 100644 --- a/core/chain_indexer_test.go +++ b/core/chain_indexer_test.go @@ -24,6 +24,7 @@ import ( "time" "github.com/EthereumCommonwealth/go-callisto/common" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/ethdb" ) @@ -47,7 +48,7 @@ func TestChainIndexerWithChildren(t *testing.T) { // multiple backends. The section size and required confirmation count parameters // are randomized. func testChainIndexer(t *testing.T, count int) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() defer db.Close() // Create a chain of indexers and ensure they all report empty @@ -92,10 +93,10 @@ func testChainIndexer(t *testing.T, count int) { inject := func(number uint64) { header := &types.Header{Number: big.NewInt(int64(number)), Extra: big.NewInt(rand.Int63()).Bytes()} if number > 0 { - header.ParentHash = GetCanonicalHash(db, number-1) + header.ParentHash = rawdb.ReadCanonicalHash(db, number-1) } - WriteHeader(db, header) - WriteCanonicalHash(db, header.Hash(), number) + rawdb.WriteHeader(db, header) + rawdb.WriteCanonicalHash(db, header.Hash(), number) } // Start indexer with an already existing chain for i := uint64(0); i <= 100; i++ { diff --git a/core/chain_makers.go b/core/chain_makers.go index 5018b44bd8..989e9e5ebc 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -256,11 +256,12 @@ func makeHeader(chain consensus.ChainReader, parent *types.Block, state *state.S // chain. Depending on the full flag, if creates either a full block chain or a // header only chain. func newCanonical(engine consensus.Engine, n int, full bool) (ethdb.Database, *BlockChain, error) { - // Initialize a fresh chain with only a genesis block - gspec := new(Genesis) - db, _ := ethdb.NewMemDatabase() - genesis := gspec.MustCommit(db) + var ( + db = ethdb.NewMemDatabase() + genesis = new(Genesis).MustCommit(db) + ) + // Initialize a fresh chain with only a genesis block blockchain, _ := NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}) // Create and inject the requested chain if n == 0 { diff --git a/core/chain_makers_test.go b/core/chain_makers_test.go index 2c87573f35..95922c5d57 100644 --- a/core/chain_makers_test.go +++ b/core/chain_makers_test.go @@ -36,7 +36,7 @@ func ExampleGenerateChain() { addr1 = crypto.PubkeyToAddress(key1.PublicKey) addr2 = crypto.PubkeyToAddress(key2.PublicKey) addr3 = crypto.PubkeyToAddress(key3.PublicKey) - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() ) // Ensure that key1 has some funds in the genesis block. diff --git a/core/dao_test.go b/core/dao_test.go index 9381f3cd85..5e46ce1a42 100644 --- a/core/dao_test.go +++ b/core/dao_test.go @@ -32,13 +32,13 @@ func TestDAOForkRangeExtradata(t *testing.T) { forkBlock := big.NewInt(32) // Generate a common prefix for both pro-forkers and non-forkers - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() gspec := new(Genesis) genesis := gspec.MustCommit(db) prefix, _ := GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, int(forkBlock.Int64()-1), func(i int, gen *BlockGen) {}) // Create the concurrent, conflicting two nodes - proDb, _ := ethdb.NewMemDatabase() + proDb := ethdb.NewMemDatabase() gspec.MustCommit(proDb) proConf := *params.TestChainConfig @@ -48,7 +48,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { proBc, _ := NewBlockChain(proDb, nil, &proConf, ethash.NewFaker(), vm.Config{}) defer proBc.Stop() - conDb, _ := ethdb.NewMemDatabase() + conDb := ethdb.NewMemDatabase() gspec.MustCommit(conDb) conConf := *params.TestChainConfig @@ -67,7 +67,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Try to expand both pro-fork and non-fork chains iteratively with other camp's blocks for i := int64(0); i < params.DAOForkExtraRange.Int64(); i++ { // Create a pro-fork block, and try to feed into the no-fork chain - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() gspec.MustCommit(db) bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{}) defer bc.Stop() @@ -92,7 +92,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { t.Fatalf("contra-fork chain didn't accepted no-fork block: %v", err) } // Create a no-fork block, and try to feed into the pro-fork chain - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() gspec.MustCommit(db) bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{}) defer bc.Stop() @@ -118,7 +118,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { } } // Verify that contra-forkers accept pro-fork extra-datas after forking finishes - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() gspec.MustCommit(db) bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{}) defer bc.Stop() @@ -138,7 +138,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { t.Fatalf("contra-fork chain didn't accept pro-fork block post-fork: %v", err) } // Verify that pro-forkers accept contra-fork extra-datas after forking finishes - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() gspec.MustCommit(db) bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{}) defer bc.Stop() diff --git a/core/events.go b/core/events.go index f8b80479d7..2979b8654a 100644 --- a/core/events.go +++ b/core/events.go @@ -21,8 +21,8 @@ import ( "github.com/EthereumCommonwealth/go-callisto/core/types" ) -// TxPreEvent is posted when a transaction enters the transaction pool. -type TxPreEvent struct{ Tx *types.Transaction } +// NewTxsEvent is posted when a batch of transactions enter the transaction pool. +type NewTxsEvent struct{ Txs []*types.Transaction } // PendingLogsEvent is posted pre mining and notifies of pending logs. type PendingLogsEvent struct { @@ -35,9 +35,6 @@ type PendingStateEvent struct{} // NewMinedBlockEvent is posted when a block has been imported. type NewMinedBlockEvent struct{ Block *types.Block } -// RemovedTransactionEvent is posted when a reorg happens -type RemovedTransactionEvent struct{ Txs types.Transactions } - // RemovedLogsEvent is posted when a reorg happens type RemovedLogsEvent struct{ Logs []*types.Log } diff --git a/core/genesis.go b/core/genesis.go index 2e51d4fc5d..c15b63c4c1 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -28,6 +28,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common" "github.com/EthereumCommonwealth/go-callisto/common/hexutil" "github.com/EthereumCommonwealth/go-callisto/common/math" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/state" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/ethdb" @@ -155,7 +156,7 @@ func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig } // Just commit the new block if there is no stored genesis block. - stored := GetCanonicalHash(db, 0) + stored := rawdb.ReadCanonicalHash(db, 0) if (stored == common.Hash{}) { if genesis == nil { log.Info("Writing default main-net genesis block") @@ -177,14 +178,11 @@ func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig // Get the existing chain configuration. newcfg := genesis.configOrDefault(stored) - storedcfg, err := GetChainConfig(db, stored) - if err != nil { - if err == ErrChainConfigNotFound { - // This case happens if a genesis write was interrupted. - log.Warn("Found genesis block without chain config") - err = WriteChainConfig(db, stored, newcfg) - } - return newcfg, stored, err + storedcfg := rawdb.ReadChainConfig(db, stored) + if storedcfg == nil { + log.Warn("Found genesis block without chain config") + rawdb.WriteChainConfig(db, stored, newcfg) + return newcfg, stored, nil } // Special case: don't change the existing config of a non-mainnet chain if no new // config is supplied. These chains would get AllProtocolChanges (and a compat error) @@ -195,15 +193,16 @@ func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig // Check config compatibility and write the config. Compatibility errors // are returned to the caller unless we're already at block zero. - height := GetBlockNumber(db, GetHeadHeaderHash(db)) - if height == missingNumber { + height := rawdb.ReadHeaderNumber(db, rawdb.ReadHeadHeaderHash(db)) + if height == nil { return newcfg, stored, fmt.Errorf("missing block number for head header hash") } - compatErr := storedcfg.CheckCompatible(newcfg, height) - if compatErr != nil && height != 0 && compatErr.RewindTo != 0 { + compatErr := storedcfg.CheckCompatible(newcfg, *height) + if compatErr != nil && *height != 0 && compatErr.RewindTo != 0 { return newcfg, stored, compatErr } - return newcfg, stored, WriteChainConfig(db, stored, newcfg) + rawdb.WriteChainConfig(db, stored, newcfg) + return newcfg, stored, nil } func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig { @@ -225,7 +224,7 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig { // to the given database (or discards it if nil). func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { if db == nil { - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() } statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) for addr, account := range g.Alloc { @@ -269,29 +268,19 @@ func (g *Genesis) Commit(db ethdb.Database) (*types.Block, error) { if block.Number().Sign() != 0 { return nil, fmt.Errorf("can't commit genesis block with number > 0") } - if err := WriteTd(db, block.Hash(), block.NumberU64(), g.Difficulty); err != nil { - return nil, err - } - if err := WriteBlock(db, block); err != nil { - return nil, err - } - if err := WriteBlockReceipts(db, block.Hash(), block.NumberU64(), nil); err != nil { - return nil, err - } - if err := WriteCanonicalHash(db, block.Hash(), block.NumberU64()); err != nil { - return nil, err - } - if err := WriteHeadBlockHash(db, block.Hash()); err != nil { - return nil, err - } - if err := WriteHeadHeaderHash(db, block.Hash()); err != nil { - return nil, err - } + rawdb.WriteTd(db, block.Hash(), block.NumberU64(), g.Difficulty) + rawdb.WriteBlock(db, block) + rawdb.WriteReceipts(db, block.Hash(), block.NumberU64(), nil) + rawdb.WriteCanonicalHash(db, block.Hash(), block.NumberU64()) + rawdb.WriteHeadBlockHash(db, block.Hash()) + rawdb.WriteHeadHeaderHash(db, block.Hash()) + config := g.Config if config == nil { config = params.AllEthashProtocolChanges } - return block, WriteChainConfig(db, block.Hash(), config) + rawdb.WriteChainConfig(db, block.Hash(), config) + return block, nil } // MustCommit writes the genesis block and state to db, panicking on error. diff --git a/core/genesis_test.go b/core/genesis_test.go index 975aeca0a0..3269bd49ba 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -23,6 +23,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common" "github.com/EthereumCommonwealth/go-callisto/consensus/ethash" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/vm" "github.com/EthereumCommonwealth/go-callisto/ethdb" "github.com/EthereumCommonwealth/go-callisto/params" @@ -140,7 +141,7 @@ func TestSetupGenesis(t *testing.T) { } for _, test := range tests { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() config, hash, err := test.fn(db) // Check the return values. if !reflect.DeepEqual(err, test.wantErr) { @@ -154,7 +155,7 @@ func TestSetupGenesis(t *testing.T) { t.Errorf("%s: returned hash %s, want %s", test.name, hash.Hex(), test.wantHash.Hex()) } else if err == nil { // Check database content. - stored := GetBlock(db, test.wantHash, 0) + stored := rawdb.ReadBlock(db, test.wantHash, 0) if stored.Hash() != test.wantHash { t.Errorf("%s: block in DB has hash %s, want %s", test.name, stored.Hash(), test.wantHash) } diff --git a/core/headerchain.go b/core/headerchain.go index e20a316802..b2208e0a72 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -28,6 +28,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common" "github.com/EthereumCommonwealth/go-callisto/consensus" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/ethdb" "github.com/EthereumCommonwealth/go-callisto/log" @@ -97,7 +98,7 @@ func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, engine c } hc.currentHeader.Store(hc.genesisHeader) - if head := GetHeadBlockHash(chainDb); head != (common.Hash{}) { + if head := rawdb.ReadHeadBlockHash(chainDb); head != (common.Hash{}) { if chead := hc.GetHeaderByHash(head); chead != nil { hc.currentHeader.Store(chead) } @@ -109,13 +110,14 @@ func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, engine c // GetBlockNumber retrieves the block number belonging to the given hash // from the cache or database -func (hc *HeaderChain) GetBlockNumber(hash common.Hash) uint64 { +func (hc *HeaderChain) GetBlockNumber(hash common.Hash) *uint64 { if cached, ok := hc.numberCache.Get(hash); ok { - return cached.(uint64) + number := cached.(uint64) + return &number } - number := GetBlockNumber(hc.chainDb, hash) - if number != missingNumber { - hc.numberCache.Add(hash, number) + number := rawdb.ReadHeaderNumber(hc.chainDb, hash) + if number != nil { + hc.numberCache.Add(hash, *number) } return number } @@ -147,20 +149,19 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er if err := hc.WriteTd(hash, number, externTd); err != nil { log.Crit("Failed to write header total difficulty", "err", err) } - if err := WriteHeader(hc.chainDb, header); err != nil { - log.Crit("Failed to write header content", "err", err) - } + rawdb.WriteHeader(hc.chainDb, header) + // If the total difficulty is higher than our known, add it to the canonical chain // Second clause in the if statement reduces the vulnerability to selfish mining. // Please refer to http://www.cs.cornell.edu/~ie53/publications/btcProcFC.pdf if externTd.Cmp(localTd) > 0 || (externTd.Cmp(localTd) == 0 && mrand.Float64() < 0.5) { // Delete any canonical number assignments above the new head for i := number + 1; ; i++ { - hash := GetCanonicalHash(hc.chainDb, i) + hash := rawdb.ReadCanonicalHash(hc.chainDb, i) if hash == (common.Hash{}) { break } - DeleteCanonicalHash(hc.chainDb, i) + rawdb.DeleteCanonicalHash(hc.chainDb, i) } // Overwrite any stale canonical number assignments var ( @@ -168,20 +169,17 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er headNumber = header.Number.Uint64() - 1 headHeader = hc.GetHeader(headHash, headNumber) ) - for GetCanonicalHash(hc.chainDb, headNumber) != headHash { - WriteCanonicalHash(hc.chainDb, headHash, headNumber) + for rawdb.ReadCanonicalHash(hc.chainDb, headNumber) != headHash { + rawdb.WriteCanonicalHash(hc.chainDb, headHash, headNumber) headHash = headHeader.ParentHash headNumber = headHeader.Number.Uint64() - 1 headHeader = hc.GetHeader(headHash, headNumber) } // Extend the canonical chain with the new header - if err := WriteCanonicalHash(hc.chainDb, hash, number); err != nil { - log.Crit("Failed to insert header number", "err", err) - } - if err := WriteHeadHeaderHash(hc.chainDb, hash); err != nil { - log.Crit("Failed to insert head header hash", "err", err) - } + rawdb.WriteCanonicalHash(hc.chainDb, hash, number) + rawdb.WriteHeadHeaderHash(hc.chainDb, hash) + hc.currentHeaderHash = hash hc.currentHeader.Store(types.CopyHeader(header)) @@ -316,7 +314,7 @@ func (hc *HeaderChain) GetTd(hash common.Hash, number uint64) *big.Int { if cached, ok := hc.tdCache.Get(hash); ok { return cached.(*big.Int) } - td := GetTd(hc.chainDb, hash, number) + td := rawdb.ReadTd(hc.chainDb, hash, number) if td == nil { return nil } @@ -328,15 +326,17 @@ func (hc *HeaderChain) GetTd(hash common.Hash, number uint64) *big.Int { // GetTdByHash retrieves a block's total difficulty in the canonical chain from the // database by hash, caching it if found. func (hc *HeaderChain) GetTdByHash(hash common.Hash) *big.Int { - return hc.GetTd(hash, hc.GetBlockNumber(hash)) + number := hc.GetBlockNumber(hash) + if number == nil { + return nil + } + return hc.GetTd(hash, *number) } // WriteTd stores a block's total difficulty into the database, also caching it // along the way. func (hc *HeaderChain) WriteTd(hash common.Hash, number uint64, td *big.Int) error { - if err := WriteTd(hc.chainDb, hash, number, td); err != nil { - return err - } + rawdb.WriteTd(hc.chainDb, hash, number, td) hc.tdCache.Add(hash, new(big.Int).Set(td)) return nil } @@ -348,7 +348,7 @@ func (hc *HeaderChain) GetHeader(hash common.Hash, number uint64) *types.Header if header, ok := hc.headerCache.Get(hash); ok { return header.(*types.Header) } - header := GetHeader(hc.chainDb, hash, number) + header := rawdb.ReadHeader(hc.chainDb, hash, number) if header == nil { return nil } @@ -360,7 +360,11 @@ func (hc *HeaderChain) GetHeader(hash common.Hash, number uint64) *types.Header // GetHeaderByHash retrieves a block header from the database by hash, caching it if // found. func (hc *HeaderChain) GetHeaderByHash(hash common.Hash) *types.Header { - return hc.GetHeader(hash, hc.GetBlockNumber(hash)) + number := hc.GetBlockNumber(hash) + if number == nil { + return nil + } + return hc.GetHeader(hash, *number) } // HasHeader checks if a block header is present in the database or not. @@ -368,14 +372,13 @@ func (hc *HeaderChain) HasHeader(hash common.Hash, number uint64) bool { if hc.numberCache.Contains(hash) || hc.headerCache.Contains(hash) { return true } - ok, _ := hc.chainDb.Has(headerKey(hash, number)) - return ok + return rawdb.HasHeader(hc.chainDb, hash, number) } // GetHeaderByNumber retrieves a block header from the database by number, // caching it (associated with its hash) if found. func (hc *HeaderChain) GetHeaderByNumber(number uint64) *types.Header { - hash := GetCanonicalHash(hc.chainDb, number) + hash := rawdb.ReadCanonicalHash(hc.chainDb, number) if hash == (common.Hash{}) { return nil } @@ -390,9 +393,8 @@ func (hc *HeaderChain) CurrentHeader() *types.Header { // SetCurrentHeader sets the current head header of the canonical chain. func (hc *HeaderChain) SetCurrentHeader(head *types.Header) { - if err := WriteHeadHeaderHash(hc.chainDb, head.Hash()); err != nil { - log.Crit("Failed to insert head header hash", "err", err) - } + rawdb.WriteHeadHeaderHash(hc.chainDb, head.Hash()) + hc.currentHeader.Store(head) hc.currentHeaderHash = head.Hash() } @@ -416,13 +418,14 @@ func (hc *HeaderChain) SetHead(head uint64, delFn DeleteCallback) { if delFn != nil { delFn(hash, num) } - DeleteHeader(hc.chainDb, hash, num) - DeleteTd(hc.chainDb, hash, num) + rawdb.DeleteHeader(hc.chainDb, hash, num) + rawdb.DeleteTd(hc.chainDb, hash, num) + hc.currentHeader.Store(hc.GetHeader(hdr.ParentHash, hdr.Number.Uint64()-1)) } // Roll back the canonical chain numbering for i := height; i > head; i-- { - DeleteCanonicalHash(hc.chainDb, i) + rawdb.DeleteCanonicalHash(hc.chainDb, i) } // Clear out any stale content from the caches hc.headerCache.Purge() @@ -434,9 +437,7 @@ func (hc *HeaderChain) SetHead(head uint64, delFn DeleteCallback) { } hc.currentHeaderHash = hc.CurrentHeader().Hash() - if err := WriteHeadHeaderHash(hc.chainDb, hc.currentHeaderHash); err != nil { - log.Crit("Failed to reset head header hash", "err", err) - } + rawdb.WriteHeadHeaderHash(hc.chainDb, hc.currentHeaderHash) } // SetGenesis sets a new genesis block header for the chain diff --git a/core/helper_test.go b/core/helper_test.go index 0afb6344b6..91ab27e32f 100644 --- a/core/helper_test.go +++ b/core/helper_test.go @@ -18,7 +18,6 @@ package core import ( "container/list" - "fmt" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/ethdb" @@ -77,18 +76,11 @@ func (tm *TestManager) Db() ethdb.Database { } func NewTestManager() *TestManager { - db, err := ethdb.NewMemDatabase() - if err != nil { - fmt.Println("Could not create mem-db, failing") - return nil - } - testManager := &TestManager{} testManager.eventMux = new(event.TypeMux) - testManager.db = db + testManager.db = ethdb.NewMemDatabase() // testManager.txPool = NewTxPool(testManager) // testManager.blockChain = NewBlockChain(testManager) // testManager.stateManager = NewStateManager(testManager) - return testManager } diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go new file mode 100644 index 0000000000..4d17d20b7c --- /dev/null +++ b/core/rawdb/accessors_chain.go @@ -0,0 +1,381 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "bytes" + "encoding/binary" + "math/big" + + "github.com/EthereumCommonwealth/go-callisto/common" + "github.com/EthereumCommonwealth/go-callisto/core/types" + "github.com/EthereumCommonwealth/go-callisto/log" + "github.com/EthereumCommonwealth/go-callisto/rlp" +) + +// ReadCanonicalHash retrieves the hash assigned to a canonical block number. +func ReadCanonicalHash(db DatabaseReader, number uint64) common.Hash { + data, _ := db.Get(append(append(headerPrefix, encodeBlockNumber(number)...), headerHashSuffix...)) + if len(data) == 0 { + return common.Hash{} + } + return common.BytesToHash(data) +} + +// WriteCanonicalHash stores the hash assigned to a canonical block number. +func WriteCanonicalHash(db DatabaseWriter, hash common.Hash, number uint64) { + key := append(append(headerPrefix, encodeBlockNumber(number)...), headerHashSuffix...) + if err := db.Put(key, hash.Bytes()); err != nil { + log.Crit("Failed to store number to hash mapping", "err", err) + } +} + +// DeleteCanonicalHash removes the number to hash canonical mapping. +func DeleteCanonicalHash(db DatabaseDeleter, number uint64) { + if err := db.Delete(append(append(headerPrefix, encodeBlockNumber(number)...), headerHashSuffix...)); err != nil { + log.Crit("Failed to delete number to hash mapping", "err", err) + } +} + +// ReadHeaderNumber returns the header number assigned to a hash. +func ReadHeaderNumber(db DatabaseReader, hash common.Hash) *uint64 { + data, _ := db.Get(append(headerNumberPrefix, hash.Bytes()...)) + if len(data) != 8 { + return nil + } + number := binary.BigEndian.Uint64(data) + return &number +} + +// ReadHeadHeaderHash retrieves the hash of the current canonical head header. +func ReadHeadHeaderHash(db DatabaseReader) common.Hash { + data, _ := db.Get(headHeaderKey) + if len(data) == 0 { + return common.Hash{} + } + return common.BytesToHash(data) +} + +// WriteHeadHeaderHash stores the hash of the current canonical head header. +func WriteHeadHeaderHash(db DatabaseWriter, hash common.Hash) { + if err := db.Put(headHeaderKey, hash.Bytes()); err != nil { + log.Crit("Failed to store last header's hash", "err", err) + } +} + +// ReadHeadBlockHash retrieves the hash of the current canonical head block. +func ReadHeadBlockHash(db DatabaseReader) common.Hash { + data, _ := db.Get(headBlockKey) + if len(data) == 0 { + return common.Hash{} + } + return common.BytesToHash(data) +} + +// WriteHeadBlockHash stores the head block's hash. +func WriteHeadBlockHash(db DatabaseWriter, hash common.Hash) { + if err := db.Put(headBlockKey, hash.Bytes()); err != nil { + log.Crit("Failed to store last block's hash", "err", err) + } +} + +// ReadHeadFastBlockHash retrieves the hash of the current fast-sync head block. +func ReadHeadFastBlockHash(db DatabaseReader) common.Hash { + data, _ := db.Get(headFastBlockKey) + if len(data) == 0 { + return common.Hash{} + } + return common.BytesToHash(data) +} + +// WriteHeadFastBlockHash stores the hash of the current fast-sync head block. +func WriteHeadFastBlockHash(db DatabaseWriter, hash common.Hash) { + if err := db.Put(headFastBlockKey, hash.Bytes()); err != nil { + log.Crit("Failed to store last fast block's hash", "err", err) + } +} + +// ReadFastTrieProgress retrieves the number of tries nodes fast synced to allow +// reporting correct numbers across restarts. +func ReadFastTrieProgress(db DatabaseReader) uint64 { + data, _ := db.Get(fastTrieProgressKey) + if len(data) == 0 { + return 0 + } + return new(big.Int).SetBytes(data).Uint64() +} + +// WriteFastTrieProgress stores the fast sync trie process counter to support +// retrieving it across restarts. +func WriteFastTrieProgress(db DatabaseWriter, count uint64) { + if err := db.Put(fastTrieProgressKey, new(big.Int).SetUint64(count).Bytes()); err != nil { + log.Crit("Failed to store fast sync trie progress", "err", err) + } +} + +// ReadHeaderRLP retrieves a block header in its raw RLP database encoding. +func ReadHeaderRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue { + data, _ := db.Get(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) + return data +} + +// HasHeader verifies the existence of a block header corresponding to the hash. +func HasHeader(db DatabaseReader, hash common.Hash, number uint64) bool { + key := append(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) + if has, err := db.Has(key); !has || err != nil { + return false + } + return true +} + +// ReadHeader retrieves the block header corresponding to the hash. +func ReadHeader(db DatabaseReader, hash common.Hash, number uint64) *types.Header { + data := ReadHeaderRLP(db, hash, number) + if len(data) == 0 { + return nil + } + header := new(types.Header) + if err := rlp.Decode(bytes.NewReader(data), header); err != nil { + log.Error("Invalid block header RLP", "hash", hash, "err", err) + return nil + } + return header +} + +// WriteHeader stores a block header into the database and also stores the hash- +// to-number mapping. +func WriteHeader(db DatabaseWriter, header *types.Header) { + // Write the hash -> number mapping + var ( + hash = header.Hash().Bytes() + number = header.Number.Uint64() + encoded = encodeBlockNumber(number) + ) + key := append(headerNumberPrefix, hash...) + if err := db.Put(key, encoded); err != nil { + log.Crit("Failed to store hash to number mapping", "err", err) + } + // Write the encoded header + data, err := rlp.EncodeToBytes(header) + if err != nil { + log.Crit("Failed to RLP encode header", "err", err) + } + key = append(append(headerPrefix, encoded...), hash...) + if err := db.Put(key, data); err != nil { + log.Crit("Failed to store header", "err", err) + } +} + +// DeleteHeader removes all block header data associated with a hash. +func DeleteHeader(db DatabaseDeleter, hash common.Hash, number uint64) { + if err := db.Delete(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...)); err != nil { + log.Crit("Failed to delete header", "err", err) + } + if err := db.Delete(append(headerNumberPrefix, hash.Bytes()...)); err != nil { + log.Crit("Failed to delete hash to number mapping", "err", err) + } +} + +// ReadBodyRLP retrieves the block body (transactions and uncles) in RLP encoding. +func ReadBodyRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue { + data, _ := db.Get(append(append(blockBodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) + return data +} + +// WriteBodyRLP stores an RLP encoded block body into the database. +func WriteBodyRLP(db DatabaseWriter, hash common.Hash, number uint64, rlp rlp.RawValue) { + key := append(append(blockBodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...) + if err := db.Put(key, rlp); err != nil { + log.Crit("Failed to store block body", "err", err) + } +} + +// HasBody verifies the existence of a block body corresponding to the hash. +func HasBody(db DatabaseReader, hash common.Hash, number uint64) bool { + key := append(append(blockBodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...) + if has, err := db.Has(key); !has || err != nil { + return false + } + return true +} + +// ReadBody retrieves the block body corresponding to the hash. +func ReadBody(db DatabaseReader, hash common.Hash, number uint64) *types.Body { + data := ReadBodyRLP(db, hash, number) + if len(data) == 0 { + return nil + } + body := new(types.Body) + if err := rlp.Decode(bytes.NewReader(data), body); err != nil { + log.Error("Invalid block body RLP", "hash", hash, "err", err) + return nil + } + return body +} + +// WriteBody storea a block body into the database. +func WriteBody(db DatabaseWriter, hash common.Hash, number uint64, body *types.Body) { + data, err := rlp.EncodeToBytes(body) + if err != nil { + log.Crit("Failed to RLP encode body", "err", err) + } + WriteBodyRLP(db, hash, number, data) +} + +// DeleteBody removes all block body data associated with a hash. +func DeleteBody(db DatabaseDeleter, hash common.Hash, number uint64) { + if err := db.Delete(append(append(blockBodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...)); err != nil { + log.Crit("Failed to delete block body", "err", err) + } +} + +// ReadTd retrieves a block's total difficulty corresponding to the hash. +func ReadTd(db DatabaseReader, hash common.Hash, number uint64) *big.Int { + data, _ := db.Get(append(append(append(headerPrefix, encodeBlockNumber(number)...), hash[:]...), headerTDSuffix...)) + if len(data) == 0 { + return nil + } + td := new(big.Int) + if err := rlp.Decode(bytes.NewReader(data), td); err != nil { + log.Error("Invalid block total difficulty RLP", "hash", hash, "err", err) + return nil + } + return td +} + +// WriteTd stores the total difficulty of a block into the database. +func WriteTd(db DatabaseWriter, hash common.Hash, number uint64, td *big.Int) { + data, err := rlp.EncodeToBytes(td) + if err != nil { + log.Crit("Failed to RLP encode block total difficulty", "err", err) + } + key := append(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...), headerTDSuffix...) + if err := db.Put(key, data); err != nil { + log.Crit("Failed to store block total difficulty", "err", err) + } +} + +// DeleteTd removes all block total difficulty data associated with a hash. +func DeleteTd(db DatabaseDeleter, hash common.Hash, number uint64) { + if err := db.Delete(append(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...), headerTDSuffix...)); err != nil { + log.Crit("Failed to delete block total difficulty", "err", err) + } +} + +// ReadReceipts retrieves all the transaction receipts belonging to a block. +func ReadReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Receipts { + // Retrieve the flattened receipt slice + data, _ := db.Get(append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash[:]...)) + if len(data) == 0 { + return nil + } + // Convert the revceipts from their storage form to their internal representation + storageReceipts := []*types.ReceiptForStorage{} + if err := rlp.DecodeBytes(data, &storageReceipts); err != nil { + log.Error("Invalid receipt array RLP", "hash", hash, "err", err) + return nil + } + receipts := make(types.Receipts, len(storageReceipts)) + for i, receipt := range storageReceipts { + receipts[i] = (*types.Receipt)(receipt) + } + return receipts +} + +// WriteReceipts stores all the transaction receipts belonging to a block. +func WriteReceipts(db DatabaseWriter, hash common.Hash, number uint64, receipts types.Receipts) { + // Convert the receipts into their storage form and serialize them + storageReceipts := make([]*types.ReceiptForStorage, len(receipts)) + for i, receipt := range receipts { + storageReceipts[i] = (*types.ReceiptForStorage)(receipt) + } + bytes, err := rlp.EncodeToBytes(storageReceipts) + if err != nil { + log.Crit("Failed to encode block receipts", "err", err) + } + // Store the flattened receipt slice + key := append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash.Bytes()...) + if err := db.Put(key, bytes); err != nil { + log.Crit("Failed to store block receipts", "err", err) + } +} + +// DeleteReceipts removes all receipt data associated with a block hash. +func DeleteReceipts(db DatabaseDeleter, hash common.Hash, number uint64) { + if err := db.Delete(append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash.Bytes()...)); err != nil { + log.Crit("Failed to delete block receipts", "err", err) + } +} + +// ReadBlock retrieves an entire block corresponding to the hash, assembling it +// back from the stored header and body. If either the header or body could not +// be retrieved nil is returned. +// +// Note, due to concurrent download of header and block body the header and thus +// canonical hash can be stored in the database but the body data not (yet). +func ReadBlock(db DatabaseReader, hash common.Hash, number uint64) *types.Block { + header := ReadHeader(db, hash, number) + if header == nil { + return nil + } + body := ReadBody(db, hash, number) + if body == nil { + return nil + } + return types.NewBlockWithHeader(header).WithBody(body.Transactions, body.Uncles) +} + +// WriteBlock serializes a block into the database, header and body separately. +func WriteBlock(db DatabaseWriter, block *types.Block) { + WriteBody(db, block.Hash(), block.NumberU64(), block.Body()) + WriteHeader(db, block.Header()) +} + +// DeleteBlock removes all block data associated with a hash. +func DeleteBlock(db DatabaseDeleter, hash common.Hash, number uint64) { + DeleteReceipts(db, hash, number) + DeleteHeader(db, hash, number) + DeleteBody(db, hash, number) + DeleteTd(db, hash, number) +} + +// FindCommonAncestor returns the last common ancestor of two block headers +func FindCommonAncestor(db DatabaseReader, a, b *types.Header) *types.Header { + for bn := b.Number.Uint64(); a.Number.Uint64() > bn; { + a = ReadHeader(db, a.ParentHash, a.Number.Uint64()-1) + if a == nil { + return nil + } + } + for an := a.Number.Uint64(); an < b.Number.Uint64(); { + b = ReadHeader(db, b.ParentHash, b.Number.Uint64()-1) + if b == nil { + return nil + } + } + for a.Hash() != b.Hash() { + a = ReadHeader(db, a.ParentHash, a.Number.Uint64()-1) + if a == nil { + return nil + } + b = ReadHeader(db, b.ParentHash, b.Number.Uint64()-1) + if b == nil { + return nil + } + } + return a +} diff --git a/core/database_util_test.go b/core/rawdb/accessors_chain_test.go similarity index 60% rename from core/database_util_test.go rename to core/rawdb/accessors_chain_test.go index 1c60ff6f29..77dcf6972c 100644 --- a/core/database_util_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 The go-ethereum Authors +// Copyright 2018 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package core +package rawdb import ( "bytes" @@ -30,23 +30,21 @@ import ( // Tests block header storage and retrieval operations. func TestHeaderStorage(t *testing.T) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() // Create a test header to move around the database and make sure it's really new header := &types.Header{Number: big.NewInt(42), Extra: []byte("test header")} - if entry := GetHeader(db, header.Hash(), header.Number.Uint64()); entry != nil { + if entry := ReadHeader(db, header.Hash(), header.Number.Uint64()); entry != nil { t.Fatalf("Non existent header returned: %v", entry) } // Write and verify the header in the database - if err := WriteHeader(db, header); err != nil { - t.Fatalf("Failed to write header into database: %v", err) - } - if entry := GetHeader(db, header.Hash(), header.Number.Uint64()); entry == nil { + WriteHeader(db, header) + if entry := ReadHeader(db, header.Hash(), header.Number.Uint64()); entry == nil { t.Fatalf("Stored header not found") } else if entry.Hash() != header.Hash() { t.Fatalf("Retrieved header mismatch: have %v, want %v", entry, header) } - if entry := GetHeaderRLP(db, header.Hash(), header.Number.Uint64()); entry == nil { + if entry := ReadHeaderRLP(db, header.Hash(), header.Number.Uint64()); entry == nil { t.Fatalf("Stored header RLP not found") } else { hasher := sha3.NewKeccak256() @@ -58,14 +56,14 @@ func TestHeaderStorage(t *testing.T) { } // Delete the header and verify the execution DeleteHeader(db, header.Hash(), header.Number.Uint64()) - if entry := GetHeader(db, header.Hash(), header.Number.Uint64()); entry != nil { + if entry := ReadHeader(db, header.Hash(), header.Number.Uint64()); entry != nil { t.Fatalf("Deleted header returned: %v", entry) } } // Tests block body storage and retrieval operations. func TestBodyStorage(t *testing.T) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() // Create a test body to move around the database and make sure it's really new body := &types.Body{Uncles: []*types.Header{{Extra: []byte("test header")}}} @@ -74,19 +72,17 @@ func TestBodyStorage(t *testing.T) { rlp.Encode(hasher, body) hash := common.BytesToHash(hasher.Sum(nil)) - if entry := GetBody(db, hash, 0); entry != nil { + if entry := ReadBody(db, hash, 0); entry != nil { t.Fatalf("Non existent body returned: %v", entry) } // Write and verify the body in the database - if err := WriteBody(db, hash, 0, body); err != nil { - t.Fatalf("Failed to write body into database: %v", err) - } - if entry := GetBody(db, hash, 0); entry == nil { + WriteBody(db, hash, 0, body) + if entry := ReadBody(db, hash, 0); entry == nil { t.Fatalf("Stored body not found") } else if types.DeriveSha(types.Transactions(entry.Transactions)) != types.DeriveSha(types.Transactions(body.Transactions)) || types.CalcUncleHash(entry.Uncles) != types.CalcUncleHash(body.Uncles) { t.Fatalf("Retrieved body mismatch: have %v, want %v", entry, body) } - if entry := GetBodyRLP(db, hash, 0); entry == nil { + if entry := ReadBodyRLP(db, hash, 0); entry == nil { t.Fatalf("Stored body RLP not found") } else { hasher := sha3.NewKeccak256() @@ -98,14 +94,14 @@ func TestBodyStorage(t *testing.T) { } // Delete the body and verify the execution DeleteBody(db, hash, 0) - if entry := GetBody(db, hash, 0); entry != nil { + if entry := ReadBody(db, hash, 0); entry != nil { t.Fatalf("Deleted body returned: %v", entry) } } // Tests block storage and retrieval operations. func TestBlockStorage(t *testing.T) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() // Create a test block to move around the database and make sure it's really new block := types.NewBlockWithHeader(&types.Header{ @@ -114,50 +110,48 @@ func TestBlockStorage(t *testing.T) { TxHash: types.EmptyRootHash, ReceiptHash: types.EmptyRootHash, }) - if entry := GetBlock(db, block.Hash(), block.NumberU64()); entry != nil { + if entry := ReadBlock(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Non existent block returned: %v", entry) } - if entry := GetHeader(db, block.Hash(), block.NumberU64()); entry != nil { + if entry := ReadHeader(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Non existent header returned: %v", entry) } - if entry := GetBody(db, block.Hash(), block.NumberU64()); entry != nil { + if entry := ReadBody(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Non existent body returned: %v", entry) } // Write and verify the block in the database - if err := WriteBlock(db, block); err != nil { - t.Fatalf("Failed to write block into database: %v", err) - } - if entry := GetBlock(db, block.Hash(), block.NumberU64()); entry == nil { + WriteBlock(db, block) + if entry := ReadBlock(db, block.Hash(), block.NumberU64()); entry == nil { t.Fatalf("Stored block not found") } else if entry.Hash() != block.Hash() { t.Fatalf("Retrieved block mismatch: have %v, want %v", entry, block) } - if entry := GetHeader(db, block.Hash(), block.NumberU64()); entry == nil { + if entry := ReadHeader(db, block.Hash(), block.NumberU64()); entry == nil { t.Fatalf("Stored header not found") } else if entry.Hash() != block.Header().Hash() { t.Fatalf("Retrieved header mismatch: have %v, want %v", entry, block.Header()) } - if entry := GetBody(db, block.Hash(), block.NumberU64()); entry == nil { + if entry := ReadBody(db, block.Hash(), block.NumberU64()); entry == nil { t.Fatalf("Stored body not found") } else if types.DeriveSha(types.Transactions(entry.Transactions)) != types.DeriveSha(block.Transactions()) || types.CalcUncleHash(entry.Uncles) != types.CalcUncleHash(block.Uncles()) { t.Fatalf("Retrieved body mismatch: have %v, want %v", entry, block.Body()) } // Delete the block and verify the execution DeleteBlock(db, block.Hash(), block.NumberU64()) - if entry := GetBlock(db, block.Hash(), block.NumberU64()); entry != nil { + if entry := ReadBlock(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Deleted block returned: %v", entry) } - if entry := GetHeader(db, block.Hash(), block.NumberU64()); entry != nil { + if entry := ReadHeader(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Deleted header returned: %v", entry) } - if entry := GetBody(db, block.Hash(), block.NumberU64()); entry != nil { + if entry := ReadBody(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Deleted body returned: %v", entry) } } // Tests that partial block contents don't get reassembled into full blocks. func TestPartialBlockStorage(t *testing.T) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() block := types.NewBlockWithHeader(&types.Header{ Extra: []byte("test block"), UncleHash: types.EmptyUncleHash, @@ -165,31 +159,24 @@ func TestPartialBlockStorage(t *testing.T) { ReceiptHash: types.EmptyRootHash, }) // Store a header and check that it's not recognized as a block - if err := WriteHeader(db, block.Header()); err != nil { - t.Fatalf("Failed to write header into database: %v", err) - } - if entry := GetBlock(db, block.Hash(), block.NumberU64()); entry != nil { + WriteHeader(db, block.Header()) + if entry := ReadBlock(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Non existent block returned: %v", entry) } DeleteHeader(db, block.Hash(), block.NumberU64()) // Store a body and check that it's not recognized as a block - if err := WriteBody(db, block.Hash(), block.NumberU64(), block.Body()); err != nil { - t.Fatalf("Failed to write body into database: %v", err) - } - if entry := GetBlock(db, block.Hash(), block.NumberU64()); entry != nil { + WriteBody(db, block.Hash(), block.NumberU64(), block.Body()) + if entry := ReadBlock(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Non existent block returned: %v", entry) } DeleteBody(db, block.Hash(), block.NumberU64()) // Store a header and a body separately and check reassembly - if err := WriteHeader(db, block.Header()); err != nil { - t.Fatalf("Failed to write header into database: %v", err) - } - if err := WriteBody(db, block.Hash(), block.NumberU64(), block.Body()); err != nil { - t.Fatalf("Failed to write body into database: %v", err) - } - if entry := GetBlock(db, block.Hash(), block.NumberU64()); entry == nil { + WriteHeader(db, block.Header()) + WriteBody(db, block.Hash(), block.NumberU64(), block.Body()) + + if entry := ReadBlock(db, block.Hash(), block.NumberU64()); entry == nil { t.Fatalf("Stored block not found") } else if entry.Hash() != block.Hash() { t.Fatalf("Retrieved block mismatch: have %v, want %v", entry, block) @@ -198,142 +185,88 @@ func TestPartialBlockStorage(t *testing.T) { // Tests block total difficulty storage and retrieval operations. func TestTdStorage(t *testing.T) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() // Create a test TD to move around the database and make sure it's really new hash, td := common.Hash{}, big.NewInt(314) - if entry := GetTd(db, hash, 0); entry != nil { + if entry := ReadTd(db, hash, 0); entry != nil { t.Fatalf("Non existent TD returned: %v", entry) } // Write and verify the TD in the database - if err := WriteTd(db, hash, 0, td); err != nil { - t.Fatalf("Failed to write TD into database: %v", err) - } - if entry := GetTd(db, hash, 0); entry == nil { + WriteTd(db, hash, 0, td) + if entry := ReadTd(db, hash, 0); entry == nil { t.Fatalf("Stored TD not found") } else if entry.Cmp(td) != 0 { t.Fatalf("Retrieved TD mismatch: have %v, want %v", entry, td) } // Delete the TD and verify the execution DeleteTd(db, hash, 0) - if entry := GetTd(db, hash, 0); entry != nil { + if entry := ReadTd(db, hash, 0); entry != nil { t.Fatalf("Deleted TD returned: %v", entry) } } // Tests that canonical numbers can be mapped to hashes and retrieved. func TestCanonicalMappingStorage(t *testing.T) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() // Create a test canonical number and assinged hash to move around hash, number := common.Hash{0: 0xff}, uint64(314) - if entry := GetCanonicalHash(db, number); entry != (common.Hash{}) { + if entry := ReadCanonicalHash(db, number); entry != (common.Hash{}) { t.Fatalf("Non existent canonical mapping returned: %v", entry) } // Write and verify the TD in the database - if err := WriteCanonicalHash(db, hash, number); err != nil { - t.Fatalf("Failed to write canonical mapping into database: %v", err) - } - if entry := GetCanonicalHash(db, number); entry == (common.Hash{}) { + WriteCanonicalHash(db, hash, number) + if entry := ReadCanonicalHash(db, number); entry == (common.Hash{}) { t.Fatalf("Stored canonical mapping not found") } else if entry != hash { t.Fatalf("Retrieved canonical mapping mismatch: have %v, want %v", entry, hash) } // Delete the TD and verify the execution DeleteCanonicalHash(db, number) - if entry := GetCanonicalHash(db, number); entry != (common.Hash{}) { + if entry := ReadCanonicalHash(db, number); entry != (common.Hash{}) { t.Fatalf("Deleted canonical mapping returned: %v", entry) } } // Tests that head headers and head blocks can be assigned, individually. func TestHeadStorage(t *testing.T) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() blockHead := types.NewBlockWithHeader(&types.Header{Extra: []byte("test block header")}) blockFull := types.NewBlockWithHeader(&types.Header{Extra: []byte("test block full")}) blockFast := types.NewBlockWithHeader(&types.Header{Extra: []byte("test block fast")}) // Check that no head entries are in a pristine database - if entry := GetHeadHeaderHash(db); entry != (common.Hash{}) { + if entry := ReadHeadHeaderHash(db); entry != (common.Hash{}) { t.Fatalf("Non head header entry returned: %v", entry) } - if entry := GetHeadBlockHash(db); entry != (common.Hash{}) { + if entry := ReadHeadBlockHash(db); entry != (common.Hash{}) { t.Fatalf("Non head block entry returned: %v", entry) } - if entry := GetHeadFastBlockHash(db); entry != (common.Hash{}) { + if entry := ReadHeadFastBlockHash(db); entry != (common.Hash{}) { t.Fatalf("Non fast head block entry returned: %v", entry) } // Assign separate entries for the head header and block - if err := WriteHeadHeaderHash(db, blockHead.Hash()); err != nil { - t.Fatalf("Failed to write head header hash: %v", err) - } - if err := WriteHeadBlockHash(db, blockFull.Hash()); err != nil { - t.Fatalf("Failed to write head block hash: %v", err) - } - if err := WriteHeadFastBlockHash(db, blockFast.Hash()); err != nil { - t.Fatalf("Failed to write fast head block hash: %v", err) - } + WriteHeadHeaderHash(db, blockHead.Hash()) + WriteHeadBlockHash(db, blockFull.Hash()) + WriteHeadFastBlockHash(db, blockFast.Hash()) + // Check that both heads are present, and different (i.e. two heads maintained) - if entry := GetHeadHeaderHash(db); entry != blockHead.Hash() { + if entry := ReadHeadHeaderHash(db); entry != blockHead.Hash() { t.Fatalf("Head header hash mismatch: have %v, want %v", entry, blockHead.Hash()) } - if entry := GetHeadBlockHash(db); entry != blockFull.Hash() { + if entry := ReadHeadBlockHash(db); entry != blockFull.Hash() { t.Fatalf("Head block hash mismatch: have %v, want %v", entry, blockFull.Hash()) } - if entry := GetHeadFastBlockHash(db); entry != blockFast.Hash() { + if entry := ReadHeadFastBlockHash(db); entry != blockFast.Hash() { t.Fatalf("Fast head block hash mismatch: have %v, want %v", entry, blockFast.Hash()) } } -// Tests that positional lookup metadata can be stored and retrieved. -func TestLookupStorage(t *testing.T) { - db, _ := ethdb.NewMemDatabase() - - tx1 := types.NewTransaction(1, common.BytesToAddress([]byte{0x11}), big.NewInt(111), 1111, big.NewInt(11111), []byte{0x11, 0x11, 0x11}) - tx2 := types.NewTransaction(2, common.BytesToAddress([]byte{0x22}), big.NewInt(222), 2222, big.NewInt(22222), []byte{0x22, 0x22, 0x22}) - tx3 := types.NewTransaction(3, common.BytesToAddress([]byte{0x33}), big.NewInt(333), 3333, big.NewInt(33333), []byte{0x33, 0x33, 0x33}) - txs := []*types.Transaction{tx1, tx2, tx3} - - block := types.NewBlock(&types.Header{Number: big.NewInt(314)}, txs, nil, nil) - - // Check that no transactions entries are in a pristine database - for i, tx := range txs { - if txn, _, _, _ := GetTransaction(db, tx.Hash()); txn != nil { - t.Fatalf("tx #%d [%x]: non existent transaction returned: %v", i, tx.Hash(), txn) - } - } - // Insert all the transactions into the database, and verify contents - if err := WriteBlock(db, block); err != nil { - t.Fatalf("failed to write block contents: %v", err) - } - if err := WriteTxLookupEntries(db, block); err != nil { - t.Fatalf("failed to write transactions: %v", err) - } - for i, tx := range txs { - if txn, hash, number, index := GetTransaction(db, tx.Hash()); txn == nil { - t.Fatalf("tx #%d [%x]: transaction not found", i, tx.Hash()) - } else { - if hash != block.Hash() || number != block.NumberU64() || index != uint64(i) { - t.Fatalf("tx #%d [%x]: positional metadata mismatch: have %x/%d/%d, want %x/%v/%v", i, tx.Hash(), hash, number, index, block.Hash(), block.NumberU64(), i) - } - if tx.Hash() != txn.Hash() { - t.Fatalf("tx #%d [%x]: transaction mismatch: have %v, want %v", i, tx.Hash(), txn, tx) - } - } - } - // Delete the transactions and check purge - for i, tx := range txs { - DeleteTxLookupEntry(db, tx.Hash()) - if txn, _, _, _ := GetTransaction(db, tx.Hash()); txn != nil { - t.Fatalf("tx #%d [%x]: deleted transaction returned: %v", i, tx.Hash(), txn) - } - } -} - // Tests that receipts associated with a single block can be stored and retrieved. func TestBlockReceiptStorage(t *testing.T) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() receipt1 := &types.Receipt{ Status: types.ReceiptStatusFailed, @@ -361,14 +294,12 @@ func TestBlockReceiptStorage(t *testing.T) { // Check that no receipt entries are in a pristine database hash := common.BytesToHash([]byte{0x03, 0x14}) - if rs := GetBlockReceipts(db, hash, 0); len(rs) != 0 { + if rs := ReadReceipts(db, hash, 0); len(rs) != 0 { t.Fatalf("non existent receipts returned: %v", rs) } // Insert the receipt slice into the database and check presence - if err := WriteBlockReceipts(db, hash, 0, receipts); err != nil { - t.Fatalf("failed to write block receipts: %v", err) - } - if rs := GetBlockReceipts(db, hash, 0); len(rs) == 0 { + WriteReceipts(db, hash, 0, receipts) + if rs := ReadReceipts(db, hash, 0); len(rs) == 0 { t.Fatalf("no receipts returned") } else { for i := 0; i < len(receipts); i++ { @@ -381,8 +312,8 @@ func TestBlockReceiptStorage(t *testing.T) { } } // Delete the receipt slice and check purge - DeleteBlockReceipts(db, hash, 0) - if rs := GetBlockReceipts(db, hash, 0); len(rs) != 0 { + DeleteReceipts(db, hash, 0) + if rs := ReadReceipts(db, hash, 0); len(rs) != 0 { t.Fatalf("deleted receipts returned: %v", rs) } } diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go new file mode 100644 index 0000000000..a3df12c699 --- /dev/null +++ b/core/rawdb/accessors_indexes.go @@ -0,0 +1,119 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "encoding/binary" + + "github.com/EthereumCommonwealth/go-callisto/common" + "github.com/EthereumCommonwealth/go-callisto/core/types" + "github.com/EthereumCommonwealth/go-callisto/log" + "github.com/EthereumCommonwealth/go-callisto/rlp" +) + +// ReadTxLookupEntry retrieves the positional metadata associated with a transaction +// hash to allow retrieving the transaction or receipt by hash. +func ReadTxLookupEntry(db DatabaseReader, hash common.Hash) (common.Hash, uint64, uint64) { + data, _ := db.Get(append(txLookupPrefix, hash.Bytes()...)) + if len(data) == 0 { + return common.Hash{}, 0, 0 + } + var entry TxLookupEntry + if err := rlp.DecodeBytes(data, &entry); err != nil { + log.Error("Invalid transaction lookup entry RLP", "hash", hash, "err", err) + return common.Hash{}, 0, 0 + } + return entry.BlockHash, entry.BlockIndex, entry.Index +} + +// WriteTxLookupEntries stores a positional metadata for every transaction from +// a block, enabling hash based transaction and receipt lookups. +func WriteTxLookupEntries(db DatabaseWriter, block *types.Block) { + for i, tx := range block.Transactions() { + entry := TxLookupEntry{ + BlockHash: block.Hash(), + BlockIndex: block.NumberU64(), + Index: uint64(i), + } + data, err := rlp.EncodeToBytes(entry) + if err != nil { + log.Crit("Failed to encode transaction lookup entry", "err", err) + } + if err := db.Put(append(txLookupPrefix, tx.Hash().Bytes()...), data); err != nil { + log.Crit("Failed to store transaction lookup entry", "err", err) + } + } +} + +// DeleteTxLookupEntry removes all transaction data associated with a hash. +func DeleteTxLookupEntry(db DatabaseDeleter, hash common.Hash) { + db.Delete(append(txLookupPrefix, hash.Bytes()...)) +} + +// ReadTransaction retrieves a specific transaction from the database, along with +// its added positional metadata. +func ReadTransaction(db DatabaseReader, hash common.Hash) (*types.Transaction, common.Hash, uint64, uint64) { + blockHash, blockNumber, txIndex := ReadTxLookupEntry(db, hash) + if blockHash == (common.Hash{}) { + return nil, common.Hash{}, 0, 0 + } + body := ReadBody(db, blockHash, blockNumber) + if body == nil || len(body.Transactions) <= int(txIndex) { + log.Error("Transaction referenced missing", "number", blockNumber, "hash", blockHash, "index", txIndex) + return nil, common.Hash{}, 0, 0 + } + return body.Transactions[txIndex], blockHash, blockNumber, txIndex +} + +// ReadReceipt retrieves a specific transaction receipt from the database, along with +// its added positional metadata. +func ReadReceipt(db DatabaseReader, hash common.Hash) (*types.Receipt, common.Hash, uint64, uint64) { + blockHash, blockNumber, receiptIndex := ReadTxLookupEntry(db, hash) + if blockHash == (common.Hash{}) { + return nil, common.Hash{}, 0, 0 + } + receipts := ReadReceipts(db, blockHash, blockNumber) + if len(receipts) <= int(receiptIndex) { + log.Error("Receipt refereced missing", "number", blockNumber, "hash", blockHash, "index", receiptIndex) + return nil, common.Hash{}, 0, 0 + } + return receipts[receiptIndex], blockHash, blockNumber, receiptIndex +} + +// ReadBloomBits retrieves the compressed bloom bit vector belonging to the given +// section and bit index from the. +func ReadBloomBits(db DatabaseReader, bit uint, section uint64, head common.Hash) ([]byte, error) { + key := append(append(bloomBitsPrefix, make([]byte, 10)...), head.Bytes()...) + + binary.BigEndian.PutUint16(key[1:], uint16(bit)) + binary.BigEndian.PutUint64(key[3:], section) + + return db.Get(key) +} + +// WriteBloomBits stores the compressed bloom bits vector belonging to the given +// section and bit index. +func WriteBloomBits(db DatabaseWriter, bit uint, section uint64, head common.Hash, bits []byte) { + key := append(append(bloomBitsPrefix, make([]byte, 10)...), head.Bytes()...) + + binary.BigEndian.PutUint16(key[1:], uint16(bit)) + binary.BigEndian.PutUint64(key[3:], section) + + if err := db.Put(key, bits); err != nil { + log.Crit("Failed to store bloom bits", "err", err) + } +} diff --git a/core/rawdb/accessors_indexes_test.go b/core/rawdb/accessors_indexes_test.go new file mode 100644 index 0000000000..531bd8b1d1 --- /dev/null +++ b/core/rawdb/accessors_indexes_test.go @@ -0,0 +1,68 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "math/big" + "testing" + + "github.com/EthereumCommonwealth/go-callisto/common" + "github.com/EthereumCommonwealth/go-callisto/core/types" + "github.com/EthereumCommonwealth/go-callisto/ethdb" +) + +// Tests that positional lookup metadata can be stored and retrieved. +func TestLookupStorage(t *testing.T) { + db := ethdb.NewMemDatabase() + + tx1 := types.NewTransaction(1, common.BytesToAddress([]byte{0x11}), big.NewInt(111), 1111, big.NewInt(11111), []byte{0x11, 0x11, 0x11}) + tx2 := types.NewTransaction(2, common.BytesToAddress([]byte{0x22}), big.NewInt(222), 2222, big.NewInt(22222), []byte{0x22, 0x22, 0x22}) + tx3 := types.NewTransaction(3, common.BytesToAddress([]byte{0x33}), big.NewInt(333), 3333, big.NewInt(33333), []byte{0x33, 0x33, 0x33}) + txs := []*types.Transaction{tx1, tx2, tx3} + + block := types.NewBlock(&types.Header{Number: big.NewInt(314)}, txs, nil, nil) + + // Check that no transactions entries are in a pristine database + for i, tx := range txs { + if txn, _, _, _ := ReadTransaction(db, tx.Hash()); txn != nil { + t.Fatalf("tx #%d [%x]: non existent transaction returned: %v", i, tx.Hash(), txn) + } + } + // Insert all the transactions into the database, and verify contents + WriteBlock(db, block) + WriteTxLookupEntries(db, block) + + for i, tx := range txs { + if txn, hash, number, index := ReadTransaction(db, tx.Hash()); txn == nil { + t.Fatalf("tx #%d [%x]: transaction not found", i, tx.Hash()) + } else { + if hash != block.Hash() || number != block.NumberU64() || index != uint64(i) { + t.Fatalf("tx #%d [%x]: positional metadata mismatch: have %x/%d/%d, want %x/%v/%v", i, tx.Hash(), hash, number, index, block.Hash(), block.NumberU64(), i) + } + if tx.Hash() != txn.Hash() { + t.Fatalf("tx #%d [%x]: transaction mismatch: have %v, want %v", i, tx.Hash(), txn, tx) + } + } + } + // Delete the transactions and check purge + for i, tx := range txs { + DeleteTxLookupEntry(db, tx.Hash()) + if txn, _, _, _ := ReadTransaction(db, tx.Hash()); txn != nil { + t.Fatalf("tx #%d [%x]: deleted transaction returned: %v", i, tx.Hash(), txn) + } + } +} diff --git a/core/rawdb/accessors_metadata.go b/core/rawdb/accessors_metadata.go new file mode 100644 index 0000000000..b7e09b28d0 --- /dev/null +++ b/core/rawdb/accessors_metadata.go @@ -0,0 +1,90 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "encoding/json" + + "github.com/EthereumCommonwealth/go-callisto/common" + "github.com/EthereumCommonwealth/go-callisto/log" + "github.com/EthereumCommonwealth/go-callisto/params" + "github.com/EthereumCommonwealth/go-callisto/rlp" +) + +// ReadDatabaseVersion retrieves the version number of the database. +func ReadDatabaseVersion(db DatabaseReader) int { + var version int + + enc, _ := db.Get(databaseVerisionKey) + rlp.DecodeBytes(enc, &version) + + return version +} + +// WriteDatabaseVersion stores the version number of the database +func WriteDatabaseVersion(db DatabaseWriter, version int) { + enc, _ := rlp.EncodeToBytes(version) + if err := db.Put(databaseVerisionKey, enc); err != nil { + log.Crit("Failed to store the database version", "err", err) + } +} + +// ReadChainConfig retrieves the consensus settings based on the given genesis hash. +func ReadChainConfig(db DatabaseReader, hash common.Hash) *params.ChainConfig { + data, _ := db.Get(append(configPrefix, hash[:]...)) + if len(data) == 0 { + return nil + } + var config params.ChainConfig + if err := json.Unmarshal(data, &config); err != nil { + log.Error("Invalid chain config JSON", "hash", hash, "err", err) + return nil + } + return &config +} + +// WriteChainConfig writes the chain config settings to the database. +func WriteChainConfig(db DatabaseWriter, hash common.Hash, cfg *params.ChainConfig) { + if cfg == nil { + return + } + data, err := json.Marshal(cfg) + if err != nil { + log.Crit("Failed to JSON encode chain config", "err", err) + } + if err := db.Put(append(configPrefix, hash[:]...), data); err != nil { + log.Crit("Failed to store chain config", "err", err) + } +} + +// ReadPreimage retrieves a single preimage of the provided hash. +func ReadPreimage(db DatabaseReader, hash common.Hash) []byte { + data, _ := db.Get(append(preimagePrefix, hash.Bytes()...)) + return data +} + +// WritePreimages writes the provided set of preimages to the database. `number` is the +// current block number, and is used for debug messages only. +func WritePreimages(db DatabaseWriter, number uint64, preimages map[common.Hash][]byte) { + for hash, preimage := range preimages { + if err := db.Put(append(preimagePrefix, hash.Bytes()...), preimage); err != nil { + log.Crit("Failed to store trie preimage", "err", err) + } + } + preimageCounter.Inc(int64(len(preimages))) + preimageHitCounter.Inc(int64(len(preimages))) +} diff --git a/core/rawdb/interfaces.go b/core/rawdb/interfaces.go new file mode 100644 index 0000000000..3bdf55124a --- /dev/null +++ b/core/rawdb/interfaces.go @@ -0,0 +1,33 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +// DatabaseReader wraps the Has and Get method of a backing data store. +type DatabaseReader interface { + Has(key []byte) (bool, error) + Get(key []byte) ([]byte, error) +} + +// DatabaseWriter wraps the Put method of a backing data store. +type DatabaseWriter interface { + Put(key []byte, value []byte) error +} + +// DatabaseDeleter wraps the Delete method of a backing data store. +type DatabaseDeleter interface { + Delete(key []byte) error +} diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go new file mode 100644 index 0000000000..737c07f811 --- /dev/null +++ b/core/rawdb/schema.go @@ -0,0 +1,79 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package rawdb contains a collection of low level database accessors. +package rawdb + +import ( + "encoding/binary" + + "github.com/EthereumCommonwealth/go-callisto/common" + "github.com/EthereumCommonwealth/go-callisto/metrics" +) + +// The fields below define the low level database schema prefixing. +var ( + // databaseVerisionKey tracks the current database version. + databaseVerisionKey = []byte("DatabaseVersion") + + // headHeaderKey tracks the latest know header's hash. + headHeaderKey = []byte("LastHeader") + + // headBlockKey tracks the latest know full block's hash. + headBlockKey = []byte("LastBlock") + + // headFastBlockKey tracks the latest known incomplete block's hash duirng fast sync. + headFastBlockKey = []byte("LastFast") + + // fastTrieProgressKey tracks the number of trie entries imported during fast sync. + fastTrieProgressKey = []byte("TrieSync") + + // Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes). + headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header + headerTDSuffix = []byte("t") // headerPrefix + num (uint64 big endian) + hash + headerTDSuffix -> td + headerHashSuffix = []byte("n") // headerPrefix + num (uint64 big endian) + headerHashSuffix -> hash + headerNumberPrefix = []byte("H") // headerNumberPrefix + hash -> num (uint64 big endian) + + blockBodyPrefix = []byte("b") // blockBodyPrefix + num (uint64 big endian) + hash -> block body + blockReceiptsPrefix = []byte("r") // blockReceiptsPrefix + num (uint64 big endian) + hash -> block receipts + + txLookupPrefix = []byte("l") // txLookupPrefix + hash -> transaction/receipt lookup metadata + bloomBitsPrefix = []byte("B") // bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash -> bloom bits + + preimagePrefix = []byte("secure-key-") // preimagePrefix + hash -> preimage + configPrefix = []byte("ethereum-config-") // config prefix for the db + + // Chain index prefixes (use `i` + single byte to avoid mixing data types). + BloomBitsIndexPrefix = []byte("iB") // BloomBitsIndexPrefix is the data table of a chain indexer to track its progress + + preimageCounter = metrics.NewRegisteredCounter("db/preimage/total", nil) + preimageHitCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil) +) + +// TxLookupEntry is a positional metadata to help looking up the data content of +// a transaction or receipt given only its hash. +type TxLookupEntry struct { + BlockHash common.Hash + BlockIndex uint64 + Index uint64 +} + +// encodeBlockNumber encodes a block number as big endian uint64 +func encodeBlockNumber(number uint64) []byte { + enc := make([]byte, 8) + binary.BigEndian.PutUint64(enc, number) + return enc +} diff --git a/core/state/managed_state_test.go b/core/state/managed_state_test.go index 05ba05e515..87a7814f72 100644 --- a/core/state/managed_state_test.go +++ b/core/state/managed_state_test.go @@ -26,8 +26,7 @@ import ( var addr = common.BytesToAddress([]byte("test")) func create() (*ManagedState, *account) { - db, _ := ethdb.NewMemDatabase() - statedb, _ := New(common.Hash{}, NewDatabase(db)) + statedb, _ := New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase())) ms := ManageState(statedb) ms.StateDB.SetNonce(addr, 100) ms.accounts[addr] = newAccount(ms.StateDB.getStateObject(addr)) diff --git a/core/state/state_object.go b/core/state/state_object.go index a927aef63e..de208490b6 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -178,9 +178,7 @@ func (self *stateObject) GetState(db Database, key common.Hash) common.Hash { } value.SetBytes(content) } - if (value != common.Hash{}) { - self.cachedStorage[key] = value - } + self.cachedStorage[key] = value return value } @@ -197,7 +195,6 @@ func (self *stateObject) SetState(db Database, key, value common.Hash) { func (self *stateObject) setState(key, value common.Hash) { self.cachedStorage[key] = value self.dirtyStorage[key] = value - } // updateTrie writes cached storage modifications into the object's storage trie. diff --git a/core/state/state_test.go b/core/state/state_test.go index c4c0e5b214..f9ed3fb4b2 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -87,7 +87,7 @@ func (s *StateSuite) TestDump(c *checker.C) { } func (s *StateSuite) SetUpTest(c *checker.C) { - s.db, _ = ethdb.NewMemDatabase() + s.db = ethdb.NewMemDatabase() s.state, _ = New(common.Hash{}, NewDatabase(s.db)) } @@ -133,8 +133,7 @@ func (s *StateSuite) TestSnapshotEmpty(c *checker.C) { // use testing instead of checker because checker does not support // printing/logging in tests (-check.vv does not work) func TestSnapshot2(t *testing.T) { - db, _ := ethdb.NewMemDatabase() - state, _ := New(common.Hash{}, NewDatabase(db)) + state, _ := New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase())) stateobjaddr0 := toAddr([]byte("so0")) stateobjaddr1 := toAddr([]byte("so1")) diff --git a/core/state/statedb.go b/core/state/statedb.go index b2ca227078..4577f75ec4 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -358,7 +358,7 @@ func (self *StateDB) deleteStateObject(stateObject *stateObject) { self.setError(self.trie.TryDelete(addr[:])) } -// Retrieve a state object given my the address. Returns nil if not found. +// Retrieve a state object given by the address. Returns nil if not found. func (self *StateDB) getStateObject(addr common.Address) (stateObject *stateObject) { // Prefer 'live' objects. if obj := self.stateObjects[addr]; obj != nil { @@ -572,27 +572,6 @@ func (self *StateDB) Prepare(thash, bhash common.Hash, ti int) { self.txIndex = ti } -// DeleteSuicides flags the suicided objects for deletion so that it -// won't be referenced again when called / queried up on. -// -// DeleteSuicides should not be used for consensus related updates -// under any circumstances. -func (s *StateDB) DeleteSuicides() { - // Reset refund so that any used-gas calculations can use this method. - s.clearJournalAndRefund() - - for addr := range s.stateObjectsDirty { - stateObject := s.stateObjects[addr] - - // If the object has been removed by a suicide - // flag the object as deleted. - if stateObject.suicided { - stateObject.deleted = true - } - delete(s.stateObjectsDirty, addr) - } -} - func (s *StateDB) clearJournalAndRefund() { s.journal = newJournal() s.validRevisions = s.validRevisions[:0] diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 32a807de9e..68639cca4d 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -39,7 +39,7 @@ import ( // actually committing the state. func TestUpdateLeaks(t *testing.T) { // Create an empty state database - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() state, _ := New(common.Hash{}, NewDatabase(db)) // Update it with some accounts @@ -66,8 +66,8 @@ func TestUpdateLeaks(t *testing.T) { // only the one right before the commit. func TestIntermediateLeaks(t *testing.T) { // Create two state databases, one transitioning to the final state, the other final from the beginning - transDb, _ := ethdb.NewMemDatabase() - finalDb, _ := ethdb.NewMemDatabase() + transDb := ethdb.NewMemDatabase() + finalDb := ethdb.NewMemDatabase() transState, _ := New(common.Hash{}, NewDatabase(transDb)) finalState, _ := New(common.Hash{}, NewDatabase(finalDb)) @@ -122,8 +122,7 @@ func TestIntermediateLeaks(t *testing.T) { // https://github.com/EthereumCommonwealth/go-callisto/pull/15549. func TestCopy(t *testing.T) { // Create a random state test to copy and modify "independently" - db, _ := ethdb.NewMemDatabase() - orig, _ := New(common.Hash{}, NewDatabase(db)) + orig, _ := New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase())) for i := byte(0); i < 255; i++ { obj := orig.GetOrNewStateObject(common.BytesToAddress([]byte{i})) @@ -334,8 +333,7 @@ func (test *snapshotTest) String() string { func (test *snapshotTest) run() bool { // Run all actions and create snapshots. var ( - db, _ = ethdb.NewMemDatabase() - state, _ = New(common.Hash{}, NewDatabase(db)) + state, _ = New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase())) snapshotRevs = make([]int, len(test.snapshots)) sindex = 0 ) @@ -426,8 +424,7 @@ func (s *StateSuite) TestTouchDelete(c *check.C) { // TestCopyOfCopy tests that modified objects are carried over to the copy, and the copy of the copy. // See https://github.com/EthereumCommonwealth/go-callisto/pull/15225#issuecomment-380191512 func TestCopyOfCopy(t *testing.T) { - db, _ := ethdb.NewMemDatabase() - sdb, _ := New(common.Hash{}, NewDatabase(db)) + sdb, _ := New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase())) addr := common.HexToAddress("aaaa") sdb.SetBalance(addr, big.NewInt(42)) diff --git a/core/state/sync_test.go b/core/state/sync_test.go index 88ad89dcf8..60e7a202a5 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -38,8 +38,7 @@ type testAccount struct { // makeTestState create a sample test state to test node-wise reconstruction. func makeTestState() (Database, common.Hash, []*testAccount) { // Create an empty state - diskdb, _ := ethdb.NewMemDatabase() - db := NewDatabase(diskdb) + db := NewDatabase(ethdb.NewMemDatabase()) state, _ := New(common.Hash{}, db) // Fill it with some arbitrary data @@ -125,8 +124,7 @@ func checkStateConsistency(db ethdb.Database, root common.Hash) error { // Tests that an empty state is not scheduled for syncing. func TestEmptyStateSync(t *testing.T) { empty := common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") - db, _ := ethdb.NewMemDatabase() - if req := NewStateSync(empty, db).Missing(1); len(req) != 0 { + if req := NewStateSync(empty, ethdb.NewMemDatabase()).Missing(1); len(req) != 0 { t.Errorf("content requested for empty state: %v", req) } } @@ -141,7 +139,7 @@ func testIterativeStateSync(t *testing.T, batch int) { srcDb, srcRoot, srcAccounts := makeTestState() // Create a destination state and sync with the scheduler - dstDb, _ := ethdb.NewMemDatabase() + dstDb := ethdb.NewMemDatabase() sched := NewStateSync(srcRoot, dstDb) queue := append([]common.Hash{}, sched.Missing(batch)...) @@ -173,7 +171,7 @@ func TestIterativeDelayedStateSync(t *testing.T) { srcDb, srcRoot, srcAccounts := makeTestState() // Create a destination state and sync with the scheduler - dstDb, _ := ethdb.NewMemDatabase() + dstDb := ethdb.NewMemDatabase() sched := NewStateSync(srcRoot, dstDb) queue := append([]common.Hash{}, sched.Missing(0)...) @@ -210,7 +208,7 @@ func testIterativeRandomStateSync(t *testing.T, batch int) { srcDb, srcRoot, srcAccounts := makeTestState() // Create a destination state and sync with the scheduler - dstDb, _ := ethdb.NewMemDatabase() + dstDb := ethdb.NewMemDatabase() sched := NewStateSync(srcRoot, dstDb) queue := make(map[common.Hash]struct{}) @@ -250,7 +248,7 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) { srcDb, srcRoot, srcAccounts := makeTestState() // Create a destination state and sync with the scheduler - dstDb, _ := ethdb.NewMemDatabase() + dstDb := ethdb.NewMemDatabase() sched := NewStateSync(srcRoot, dstDb) queue := make(map[common.Hash]struct{}) @@ -297,7 +295,7 @@ func TestIncompleteStateSync(t *testing.T) { checkTrieConsistency(srcDb.TrieDB().DiskDB().(ethdb.Database), srcRoot) // Create a destination state and sync with the scheduler - dstDb, _ := ethdb.NewMemDatabase() + dstDb := ethdb.NewMemDatabase() sched := NewStateSync(srcRoot, dstDb) added := []common.Hash{} diff --git a/core/tx_journal.go b/core/tx_journal.go index fbd6ef996b..a552676fc8 100644 --- a/core/tx_journal.go +++ b/core/tx_journal.go @@ -56,7 +56,7 @@ func newTxJournal(path string) *txJournal { // load parses a transaction journal dump from disk, loading its contents into // the specified pool. -func (journal *txJournal) load(add func(*types.Transaction) error) error { +func (journal *txJournal) load(add func([]*types.Transaction) []error) error { // Skip the parsing if the journal file doens't exist at all if _, err := os.Stat(journal.path); os.IsNotExist(err) { return nil @@ -76,7 +76,21 @@ func (journal *txJournal) load(add func(*types.Transaction) error) error { stream := rlp.NewStream(input, 0) total, dropped := 0, 0 - var failure error + // Create a method to load a limited batch of transactions and bump the + // appropriate progress counters. Then use this method to load all the + // journalled transactions in small-ish batches. + loadBatch := func(txs types.Transactions) { + for _, err := range add(txs) { + if err != nil { + log.Debug("Failed to add journaled transaction", "err", err) + dropped++ + } + } + } + var ( + failure error + batch types.Transactions + ) for { // Parse the next transaction and terminate on error tx := new(types.Transaction) @@ -84,14 +98,17 @@ func (journal *txJournal) load(add func(*types.Transaction) error) error { if err != io.EOF { failure = err } + if batch.Len() > 0 { + loadBatch(batch) + } break } - // Import the transaction and bump the appropriate progress counters + // New transaction parsed, queue up for later, import if threnshold is reached total++ - if err = add(tx); err != nil { - log.Debug("Failed to add journaled transaction", "err", err) - dropped++ - continue + + if batch = append(batch, tx); batch.Len() > 1024 { + loadBatch(batch) + batch = batch[:0] } } log.Info("Loaded local transaction journal", "transactions", total, "dropped", dropped) diff --git a/core/tx_list.go b/core/tx_list.go index 096b9b8c55..572a9dcf38 100644 --- a/core/tx_list.go +++ b/core/tx_list.go @@ -397,13 +397,13 @@ func (h *priceHeap) Pop() interface{} { // txPricedList is a price-sorted heap to allow operating on transactions pool // contents in a price-incrementing way. type txPricedList struct { - all *map[common.Hash]*types.Transaction // Pointer to the map of all transactions - items *priceHeap // Heap of prices of all the stored transactions - stales int // Number of stale price points to (re-heap trigger) + all *txLookup // Pointer to the map of all transactions + items *priceHeap // Heap of prices of all the stored transactions + stales int // Number of stale price points to (re-heap trigger) } // newTxPricedList creates a new price-sorted transaction heap. -func newTxPricedList(all *map[common.Hash]*types.Transaction) *txPricedList { +func newTxPricedList(all *txLookup) *txPricedList { return &txPricedList{ all: all, items: new(priceHeap), @@ -425,12 +425,13 @@ func (l *txPricedList) Removed() { return } // Seems we've reached a critical number of stale transactions, reheap - reheap := make(priceHeap, 0, len(*l.all)) + reheap := make(priceHeap, 0, l.all.Count()) l.stales, l.items = 0, &reheap - for _, tx := range *l.all { + l.all.Range(func(hash common.Hash, tx *types.Transaction) bool { *l.items = append(*l.items, tx) - } + return true + }) heap.Init(l.items) } @@ -443,7 +444,7 @@ func (l *txPricedList) Cap(threshold *big.Int, local *accountSet) types.Transact for len(*l.items) > 0 { // Discard stale transactions if found during cleanup tx := heap.Pop(l.items).(*types.Transaction) - if _, ok := (*l.all)[tx.Hash()]; !ok { + if l.all.Get(tx.Hash()) == nil { l.stales-- continue } @@ -475,7 +476,7 @@ func (l *txPricedList) Underpriced(tx *types.Transaction, local *accountSet) boo // Discard stale price points if found at the heap start for len(*l.items) > 0 { head := []*types.Transaction(*l.items)[0] - if _, ok := (*l.all)[head.Hash()]; !ok { + if l.all.Get(head.Hash()) == nil { l.stales-- heap.Pop(l.items) continue @@ -500,7 +501,7 @@ func (l *txPricedList) Discard(count int, local *accountSet) types.Transactions for len(*l.items) > 0 && count > 0 { // Discard stale transactions if found during cleanup tx := heap.Pop(l.items).(*types.Transaction) - if _, ok := (*l.all)[tx.Hash()]; !ok { + if l.all.Get(tx.Hash()) == nil { l.stales-- continue } diff --git a/core/tx_pool.go b/core/tx_pool.go index fa28fca80c..c2e6cd4bc5 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -200,11 +200,11 @@ type TxPool struct { locals *accountSet // Set of local transaction to exempt from eviction rules journal *txJournal // Journal of local transaction to back up to disk - pending map[common.Address]*txList // All currently processable transactions - queue map[common.Address]*txList // Queued but non-processable transactions - beats map[common.Address]time.Time // Last heartbeat from each known account - all map[common.Hash]*types.Transaction // All transactions to allow lookups - priced *txPricedList // All transactions sorted by price + pending map[common.Address]*txList // All currently processable transactions + queue map[common.Address]*txList // Queued but non-processable transactions + beats map[common.Address]time.Time // Last heartbeat from each known account + all *txLookup // All transactions to allow lookups + priced *txPricedList // All transactions sorted by price wg sync.WaitGroup // for shutdown sync @@ -226,19 +226,19 @@ func NewTxPool(config TxPoolConfig, chainconfig *params.ChainConfig, chain block pending: make(map[common.Address]*txList), queue: make(map[common.Address]*txList), beats: make(map[common.Address]time.Time), - all: make(map[common.Hash]*types.Transaction), + all: newTxLookup(), chainHeadCh: make(chan ChainHeadEvent, chainHeadChanSize), gasPrice: new(big.Int).SetUint64(config.PriceLimit), } pool.locals = newAccountSet(pool.signer) - pool.priced = newTxPricedList(&pool.all) + pool.priced = newTxPricedList(pool.all) pool.reset(nil, chain.CurrentBlock().Header()) // If local transactions and journaling is enabled, load from disk if !config.NoLocals && config.Journal != "" { pool.journal = newTxJournal(config.Journal) - if err := pool.journal.load(pool.AddLocal); err != nil { + if err := pool.journal.load(pool.AddLocals); err != nil { log.Warn("Failed to load transaction journal", "err", err) } if err := pool.journal.rotate(pool.local()); err != nil { @@ -444,9 +444,9 @@ func (pool *TxPool) Stop() { log.Info("Transaction pool stopped") } -// SubscribeTxPreEvent registers a subscription of TxPreEvent and +// SubscribeNewTxsEvent registers a subscription of NewTxsEvent and // starts sending event to the given channel. -func (pool *TxPool) SubscribeTxPreEvent(ch chan<- TxPreEvent) event.Subscription { +func (pool *TxPool) SubscribeNewTxsEvent(ch chan<- NewTxsEvent) event.Subscription { return pool.scope.Track(pool.txFeed.Subscribe(ch)) } @@ -605,7 +605,7 @@ func (pool *TxPool) validateTx(tx *types.Transaction, local bool) error { func (pool *TxPool) add(tx *types.Transaction, local bool) (bool, error) { // If the transaction is already known, discard it hash := tx.Hash() - if pool.all[hash] != nil { + if pool.all.Get(hash) != nil { log.Trace("Discarding already known transaction", "hash", hash) return false, fmt.Errorf("known transaction: %x", hash) } @@ -616,15 +616,15 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (bool, error) { return false, err } // If the transaction pool is full, discard underpriced transactions - if uint64(len(pool.all)) >= pool.config.GlobalSlots+pool.config.GlobalQueue { + if uint64(pool.all.Count()) >= pool.config.GlobalSlots+pool.config.GlobalQueue { // If the new transaction is underpriced, don't accept it - if pool.priced.Underpriced(tx, pool.locals) { + if !local && pool.priced.Underpriced(tx, pool.locals) { log.Trace("Discarding underpriced transaction", "hash", hash, "price", tx.GasPrice()) underpricedTxCounter.Inc(1) return false, ErrUnderpriced } // New transaction is better than our worse ones, make room for it - drop := pool.priced.Discard(len(pool.all)-int(pool.config.GlobalSlots+pool.config.GlobalQueue-1), pool.locals) + drop := pool.priced.Discard(pool.all.Count()-int(pool.config.GlobalSlots+pool.config.GlobalQueue-1), pool.locals) for _, tx := range drop { log.Trace("Discarding freshly underpriced transaction", "hash", tx.Hash(), "price", tx.GasPrice()) underpricedTxCounter.Inc(1) @@ -642,18 +642,18 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (bool, error) { } // New transaction is better, replace old one if old != nil { - delete(pool.all, old.Hash()) + pool.all.Remove(old.Hash()) pool.priced.Removed() pendingReplaceCounter.Inc(1) } - pool.all[tx.Hash()] = tx + pool.all.Add(tx) pool.priced.Put(tx) pool.journalTx(from, tx) log.Trace("Pooled new executable transaction", "hash", hash, "from", from, "to", tx.To()) // We've directly injected a replacement transaction, notify subsystems - go pool.txFeed.Send(TxPreEvent{tx}) + go pool.txFeed.Send(NewTxsEvent{types.Transactions{tx}}) return old != nil, nil } @@ -689,12 +689,12 @@ func (pool *TxPool) enqueueTx(hash common.Hash, tx *types.Transaction) (bool, er } // Discard any previous transaction and mark this if old != nil { - delete(pool.all, old.Hash()) + pool.all.Remove(old.Hash()) pool.priced.Removed() queuedReplaceCounter.Inc(1) } - if pool.all[hash] == nil { - pool.all[hash] = tx + if pool.all.Get(hash) == nil { + pool.all.Add(tx) pool.priced.Put(tx) } return old != nil, nil @@ -712,10 +712,11 @@ func (pool *TxPool) journalTx(from common.Address, tx *types.Transaction) { } } -// promoteTx adds a transaction to the pending (processable) list of transactions. +// promoteTx adds a transaction to the pending (processable) list of transactions +// and returns whether it was inserted or an older was better. // // Note, this method assumes the pool lock is held! -func (pool *TxPool) promoteTx(addr common.Address, hash common.Hash, tx *types.Transaction) { +func (pool *TxPool) promoteTx(addr common.Address, hash common.Hash, tx *types.Transaction) bool { // Try to insert the transaction into the pending queue if pool.pending[addr] == nil { pool.pending[addr] = newTxList(true) @@ -725,29 +726,29 @@ func (pool *TxPool) promoteTx(addr common.Address, hash common.Hash, tx *types.T inserted, old := list.Add(tx, pool.config.PriceBump) if !inserted { // An older transaction was better, discard this - delete(pool.all, hash) + pool.all.Remove(hash) pool.priced.Removed() pendingDiscardCounter.Inc(1) - return + return false } // Otherwise discard any previous transaction and mark this if old != nil { - delete(pool.all, old.Hash()) + pool.all.Remove(old.Hash()) pool.priced.Removed() pendingReplaceCounter.Inc(1) } // Failsafe to work around direct pending inserts (tests) - if pool.all[hash] == nil { - pool.all[hash] = tx + if pool.all.Get(hash) == nil { + pool.all.Add(tx) pool.priced.Put(tx) } // Set the potentially new pending nonce and notify any subsystems of the new tx pool.beats[addr] = time.Now() pool.pendingState.SetNonce(addr, tx.Nonce()+1) - go pool.txFeed.Send(TxPreEvent{tx}) + return true } // AddLocal enqueues a single transaction into the pool if it is valid, marking @@ -839,7 +840,7 @@ func (pool *TxPool) Status(hashes []common.Hash) []TxStatus { status := make([]TxStatus, len(hashes)) for i, hash := range hashes { - if tx := pool.all[hash]; tx != nil { + if tx := pool.all.Get(hash); tx != nil { from, _ := types.Sender(pool.signer, tx) // already validated if pool.pending[from] != nil && pool.pending[from].txs.items[tx.Nonce()] != nil { status[i] = TxStatusPending @@ -854,24 +855,21 @@ func (pool *TxPool) Status(hashes []common.Hash) []TxStatus { // Get returns a transaction if it is contained in the pool // and nil otherwise. func (pool *TxPool) Get(hash common.Hash) *types.Transaction { - pool.mu.RLock() - defer pool.mu.RUnlock() - - return pool.all[hash] + return pool.all.Get(hash) } // removeTx removes a single transaction from the queue, moving all subsequent // transactions back to the future queue. func (pool *TxPool) removeTx(hash common.Hash, outofbound bool) { // Fetch the transaction we wish to delete - tx, ok := pool.all[hash] - if !ok { + tx := pool.all.Get(hash) + if tx == nil { return } addr, _ := types.Sender(pool.signer, tx) // already validated during insertion // Remove it from the list of known transactions - delete(pool.all, hash) + pool.all.Remove(hash) if outofbound { pool.priced.Removed() } @@ -907,6 +905,9 @@ func (pool *TxPool) removeTx(hash common.Hash, outofbound bool) { // future queue to the set of pending transactions. During this process, all // invalidated transactions (low nonce, low balance) are deleted. func (pool *TxPool) promoteExecutables(accounts []common.Address) { + // Track the promoted transactions to broadcast them at once + var promoted []*types.Transaction + // Gather all the accounts potentially needing updates if accounts == nil { accounts = make([]common.Address, 0, len(pool.queue)) @@ -924,7 +925,7 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) { for _, tx := range list.Forward(pool.currentState.GetNonce(addr)) { hash := tx.Hash() log.Trace("Removed old queued transaction", "hash", hash) - delete(pool.all, hash) + pool.all.Remove(hash) pool.priced.Removed() } // Drop all transactions that are too costly (low balance or out of gas) @@ -932,21 +933,23 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) { for _, tx := range drops { hash := tx.Hash() log.Trace("Removed unpayable queued transaction", "hash", hash) - delete(pool.all, hash) + pool.all.Remove(hash) pool.priced.Removed() queuedNofundsCounter.Inc(1) } // Gather all executable transactions and promote them for _, tx := range list.Ready(pool.pendingState.GetNonce(addr)) { hash := tx.Hash() - log.Trace("Promoting queued transaction", "hash", hash) - pool.promoteTx(addr, hash, tx) + if pool.promoteTx(addr, hash, tx) { + log.Trace("Promoting queued transaction", "hash", hash) + promoted = append(promoted, tx) + } } // Drop all transactions over the allowed limit if !pool.locals.contains(addr) { for _, tx := range list.Cap(int(pool.config.AccountQueue)) { hash := tx.Hash() - delete(pool.all, hash) + pool.all.Remove(hash) pool.priced.Removed() queuedRateLimitCounter.Inc(1) log.Trace("Removed cap-exceeding queued transaction", "hash", hash) @@ -957,6 +960,10 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) { delete(pool.queue, addr) } } + // Notify subsystem for new promoted transactions. + if len(promoted) > 0 { + pool.txFeed.Send(NewTxsEvent{promoted}) + } // If the pending limit is overflown, start equalizing allowances pending := uint64(0) for _, list := range pool.pending { @@ -991,7 +998,7 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) { for _, tx := range list.Cap(list.Len() - 1) { // Drop the transaction from the global pools too hash := tx.Hash() - delete(pool.all, hash) + pool.all.Remove(hash) pool.priced.Removed() // Update the account nonce to the dropped transaction @@ -1013,7 +1020,7 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) { for _, tx := range list.Cap(list.Len() - 1) { // Drop the transaction from the global pools too hash := tx.Hash() - delete(pool.all, hash) + pool.all.Remove(hash) pool.priced.Removed() // Update the account nonce to the dropped transaction @@ -1082,7 +1089,7 @@ func (pool *TxPool) demoteUnexecutables() { for _, tx := range list.Forward(nonce) { hash := tx.Hash() log.Trace("Removed old pending transaction", "hash", hash) - delete(pool.all, hash) + pool.all.Remove(hash) pool.priced.Removed() } // Drop all transactions that are too costly (low balance or out of gas), and queue any invalids back for later @@ -1090,7 +1097,7 @@ func (pool *TxPool) demoteUnexecutables() { for _, tx := range drops { hash := tx.Hash() log.Trace("Removed unpayable pending transaction", "hash", hash) - delete(pool.all, hash) + pool.all.Remove(hash) pool.priced.Removed() pendingNofundsCounter.Inc(1) } @@ -1162,3 +1169,68 @@ func (as *accountSet) containsTx(tx *types.Transaction) bool { func (as *accountSet) add(addr common.Address) { as.accounts[addr] = struct{}{} } + +// txLookup is used internally by TxPool to track transactions while allowing lookup without +// mutex contention. +// +// Note, although this type is properly protected against concurrent access, it +// is **not** a type that should ever be mutated or even exposed outside of the +// transaction pool, since its internal state is tightly coupled with the pools +// internal mechanisms. The sole purpose of the type is to permit out-of-bound +// peeking into the pool in TxPool.Get without having to acquire the widely scoped +// TxPool.mu mutex. +type txLookup struct { + all map[common.Hash]*types.Transaction + lock sync.RWMutex +} + +// newTxLookup returns a new txLookup structure. +func newTxLookup() *txLookup { + return &txLookup{ + all: make(map[common.Hash]*types.Transaction), + } +} + +// Range calls f on each key and value present in the map. +func (t *txLookup) Range(f func(hash common.Hash, tx *types.Transaction) bool) { + t.lock.RLock() + defer t.lock.RUnlock() + + for key, value := range t.all { + if !f(key, value) { + break + } + } +} + +// Get returns a transaction if it exists in the lookup, or nil if not found. +func (t *txLookup) Get(hash common.Hash) *types.Transaction { + t.lock.RLock() + defer t.lock.RUnlock() + + return t.all[hash] +} + +// Count returns the current number of items in the lookup. +func (t *txLookup) Count() int { + t.lock.RLock() + defer t.lock.RUnlock() + + return len(t.all) +} + +// Add adds a transaction to the lookup. +func (t *txLookup) Add(tx *types.Transaction) { + t.lock.Lock() + defer t.lock.Unlock() + + t.all[tx.Hash()] = tx +} + +// Remove removes a transaction from the lookup. +func (t *txLookup) Remove(hash common.Hash) { + t.lock.Lock() + defer t.lock.Unlock() + + delete(t.all, hash) +} diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index fd132d30b8..a630160a88 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -78,8 +78,7 @@ func pricedTransaction(nonce uint64, gaslimit uint64, gasprice *big.Int, key *ec } func setupTxPool() (*TxPool, *ecdsa.PrivateKey) { - diskdb, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(diskdb)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} key, _ := crypto.GenerateKey() @@ -95,7 +94,7 @@ func validateTxPoolInternals(pool *TxPool) error { // Ensure the total transaction set is consistent with pending + queued pending, queued := pool.stats() - if total := len(pool.all); total != pending+queued { + if total := pool.all.Count(); total != pending+queued { return fmt.Errorf("total transaction count %d != %d pending + %d queued", total, pending, queued) } if priced := pool.priced.items.Len() - pool.priced.stales; priced != pending+queued { @@ -119,21 +118,27 @@ func validateTxPoolInternals(pool *TxPool) error { // validateEvents checks that the correct number of transaction addition events // were fired on the pool's event feed. -func validateEvents(events chan TxPreEvent, count int) error { - for i := 0; i < count; i++ { +func validateEvents(events chan NewTxsEvent, count int) error { + var received []*types.Transaction + + for len(received) < count { select { - case <-events: + case ev := <-events: + received = append(received, ev.Txs...) case <-time.After(time.Second): - return fmt.Errorf("event #%d not fired", i) + return fmt.Errorf("event #%d not fired", received) } } + if len(received) > count { + return fmt.Errorf("more than %d events fired: %v", count, received[count:]) + } select { - case tx := <-events: - return fmt.Errorf("more than %d events fired: %v", count, tx.Tx) + case ev := <-events: + return fmt.Errorf("more than %d events fired: %v", count, ev.Txs) case <-time.After(50 * time.Millisecond): // This branch should be "default", but it's a data race between goroutines, - // reading the event channel and pushng into it, so better wait a bit ensuring + // reading the event channel and pushing into it, so better wait a bit ensuring // really nothing gets injected. } return nil @@ -158,8 +163,7 @@ func (c *testChain) State() (*state.StateDB, error) { // a state change between those fetches. stdb := c.statedb if *c.trigger { - db, _ := ethdb.NewMemDatabase() - c.statedb, _ = state.New(common.Hash{}, state.NewDatabase(db)) + c.statedb, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) // simulate that the new head block included tx0 and tx1 c.statedb.SetNonce(c.address, 2) c.statedb.SetBalance(c.address, new(big.Int).SetUint64(params.Ether)) @@ -175,10 +179,9 @@ func TestStateChangeDuringTransactionPoolReset(t *testing.T) { t.Parallel() var ( - db, _ = ethdb.NewMemDatabase() key, _ = crypto.GenerateKey() address = crypto.PubkeyToAddress(key.PublicKey) - statedb, _ = state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) trigger = false ) @@ -332,8 +335,7 @@ func TestTransactionChainFork(t *testing.T) { addr := crypto.PubkeyToAddress(key.PublicKey) resetState := func() { - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) statedb.AddBalance(addr, big.NewInt(100000000000000)) pool.chain = &testBlockChain{statedb, 1000000, new(event.Feed)} @@ -362,8 +364,7 @@ func TestTransactionDoubleNonce(t *testing.T) { addr := crypto.PubkeyToAddress(key.PublicKey) resetState := func() { - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) statedb.AddBalance(addr, big.NewInt(100000000000000)) pool.chain = &testBlockChain{statedb, 1000000, new(event.Feed)} @@ -400,8 +401,8 @@ func TestTransactionDoubleNonce(t *testing.T) { t.Errorf("transaction mismatch: have %x, want %x", tx.Hash(), tx2.Hash()) } // Ensure the total transaction count is correct - if len(pool.all) != 1 { - t.Error("expected 1 total transactions, got", len(pool.all)) + if pool.all.Count() != 1 { + t.Error("expected 1 total transactions, got", pool.all.Count()) } } @@ -423,8 +424,8 @@ func TestTransactionMissingNonce(t *testing.T) { if pool.queue[addr].Len() != 1 { t.Error("expected 1 queued transaction, got", pool.queue[addr].Len()) } - if len(pool.all) != 1 { - t.Error("expected 1 total transactions, got", len(pool.all)) + if pool.all.Count() != 1 { + t.Error("expected 1 total transactions, got", pool.all.Count()) } } @@ -487,8 +488,8 @@ func TestTransactionDropping(t *testing.T) { if pool.queue[account].Len() != 3 { t.Errorf("queued transaction mismatch: have %d, want %d", pool.queue[account].Len(), 3) } - if len(pool.all) != 6 { - t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), 6) + if pool.all.Count() != 6 { + t.Errorf("total transaction mismatch: have %d, want %d", pool.all.Count(), 6) } pool.lockedReset(nil, nil) if pool.pending[account].Len() != 3 { @@ -497,8 +498,8 @@ func TestTransactionDropping(t *testing.T) { if pool.queue[account].Len() != 3 { t.Errorf("queued transaction mismatch: have %d, want %d", pool.queue[account].Len(), 3) } - if len(pool.all) != 6 { - t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), 6) + if pool.all.Count() != 6 { + t.Errorf("total transaction mismatch: have %d, want %d", pool.all.Count(), 6) } // Reduce the balance of the account, and check that invalidated transactions are dropped pool.currentState.AddBalance(account, big.NewInt(-650)) @@ -522,8 +523,8 @@ func TestTransactionDropping(t *testing.T) { if _, ok := pool.queue[account].txs.items[tx12.Nonce()]; ok { t.Errorf("out-of-fund queued transaction present: %v", tx11) } - if len(pool.all) != 4 { - t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), 4) + if pool.all.Count() != 4 { + t.Errorf("total transaction mismatch: have %d, want %d", pool.all.Count(), 4) } // Reduce the block gas limit, check that invalidated transactions are dropped pool.chain.(*testBlockChain).gasLimit = 100 @@ -541,8 +542,8 @@ func TestTransactionDropping(t *testing.T) { if _, ok := pool.queue[account].txs.items[tx11.Nonce()]; ok { t.Errorf("over-gased queued transaction present: %v", tx11) } - if len(pool.all) != 2 { - t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), 2) + if pool.all.Count() != 2 { + t.Errorf("total transaction mismatch: have %d, want %d", pool.all.Count(), 2) } } @@ -553,8 +554,7 @@ func TestTransactionPostponing(t *testing.T) { t.Parallel() // Create the pool to test the postponing with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) @@ -596,8 +596,8 @@ func TestTransactionPostponing(t *testing.T) { if len(pool.queue) != 0 { t.Errorf("queued accounts mismatch: have %d, want %d", len(pool.queue), 0) } - if len(pool.all) != len(txs) { - t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), len(txs)) + if pool.all.Count() != len(txs) { + t.Errorf("total transaction mismatch: have %d, want %d", pool.all.Count(), len(txs)) } pool.lockedReset(nil, nil) if pending := pool.pending[accs[0]].Len() + pool.pending[accs[1]].Len(); pending != len(txs) { @@ -606,8 +606,8 @@ func TestTransactionPostponing(t *testing.T) { if len(pool.queue) != 0 { t.Errorf("queued accounts mismatch: have %d, want %d", len(pool.queue), 0) } - if len(pool.all) != len(txs) { - t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), len(txs)) + if pool.all.Count() != len(txs) { + t.Errorf("total transaction mismatch: have %d, want %d", pool.all.Count(), len(txs)) } // Reduce the balance of the account, and check that transactions are reorganised for _, addr := range accs { @@ -656,8 +656,8 @@ func TestTransactionPostponing(t *testing.T) { } } } - if len(pool.all) != len(txs)/2 { - t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), len(txs)/2) + if pool.all.Count() != len(txs)/2 { + t.Errorf("total transaction mismatch: have %d, want %d", pool.all.Count(), len(txs)/2) } } @@ -675,7 +675,7 @@ func TestTransactionGapFilling(t *testing.T) { pool.currentState.AddBalance(account, big.NewInt(1000000)) // Keep track of transaction events to ensure all executables get announced - events := make(chan TxPreEvent, testTxPoolConfig.AccountQueue+5) + events := make(chan NewTxsEvent, testTxPoolConfig.AccountQueue+5) sub := pool.txFeed.Subscribe(events) defer sub.Unsubscribe() @@ -748,8 +748,8 @@ func TestTransactionQueueAccountLimiting(t *testing.T) { } } } - if len(pool.all) != int(testTxPoolConfig.AccountQueue) { - t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), testTxPoolConfig.AccountQueue) + if pool.all.Count() != int(testTxPoolConfig.AccountQueue) { + t.Errorf("total transaction mismatch: have %d, want %d", pool.all.Count(), testTxPoolConfig.AccountQueue) } } @@ -769,8 +769,7 @@ func testTransactionQueueGlobalLimiting(t *testing.T, nolocals bool) { t.Parallel() // Create the pool to test the limit enforcement with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -858,8 +857,7 @@ func testTransactionQueueTimeLimiting(t *testing.T, nolocals bool) { evictionInterval = time.Second // Create the pool to test the non-expiration enforcement - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -928,7 +926,7 @@ func TestTransactionPendingLimiting(t *testing.T) { pool.currentState.AddBalance(account, big.NewInt(1000000)) // Keep track of transaction events to ensure all executables get announced - events := make(chan TxPreEvent, testTxPoolConfig.AccountQueue+5) + events := make(chan NewTxsEvent, testTxPoolConfig.AccountQueue+5) sub := pool.txFeed.Subscribe(events) defer sub.Unsubscribe() @@ -944,8 +942,8 @@ func TestTransactionPendingLimiting(t *testing.T) { t.Errorf("tx %d: queue size mismatch: have %d, want %d", i, pool.queue[account].Len(), 0) } } - if len(pool.all) != int(testTxPoolConfig.AccountQueue+5) { - t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), testTxPoolConfig.AccountQueue+5) + if pool.all.Count() != int(testTxPoolConfig.AccountQueue+5) { + t.Errorf("total transaction mismatch: have %d, want %d", pool.all.Count(), testTxPoolConfig.AccountQueue+5) } if err := validateEvents(events, int(testTxPoolConfig.AccountQueue+5)); err != nil { t.Fatalf("event firing failed: %v", err) @@ -995,8 +993,8 @@ func testTransactionLimitingEquivalency(t *testing.T, origin uint64) { if len(pool1.queue) != len(pool2.queue) { t.Errorf("queued transaction count mismatch: one-by-one algo: %d, batch algo: %d", len(pool1.queue), len(pool2.queue)) } - if len(pool1.all) != len(pool2.all) { - t.Errorf("total transaction count mismatch: one-by-one algo %d, batch algo %d", len(pool1.all), len(pool2.all)) + if pool1.all.Count() != pool2.all.Count() { + t.Errorf("total transaction count mismatch: one-by-one algo %d, batch algo %d", pool1.all.Count(), pool2.all.Count()) } if err := validateTxPoolInternals(pool1); err != nil { t.Errorf("pool 1 internal state corrupted: %v", err) @@ -1013,8 +1011,7 @@ func TestTransactionPendingGlobalLimiting(t *testing.T) { t.Parallel() // Create the pool to test the limit enforcement with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1060,8 +1057,7 @@ func TestTransactionCapClearsFromAll(t *testing.T) { t.Parallel() // Create the pool to test the limit enforcement with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1095,8 +1091,7 @@ func TestTransactionPendingMinimumAllowance(t *testing.T) { t.Parallel() // Create the pool to test the limit enforcement with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1144,15 +1139,14 @@ func TestTransactionPoolRepricing(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) defer pool.Stop() // Keep track of transaction events to ensure all executables get announced - events := make(chan TxPreEvent, 32) + events := make(chan NewTxsEvent, 32) sub := pool.txFeed.Subscribe(events) defer sub.Unsubscribe() @@ -1266,8 +1260,7 @@ func TestTransactionPoolRepricingKeepsLocals(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) @@ -1329,8 +1322,7 @@ func TestTransactionPoolUnderpricing(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1341,12 +1333,12 @@ func TestTransactionPoolUnderpricing(t *testing.T) { defer pool.Stop() // Keep track of transaction events to ensure all executables get announced - events := make(chan TxPreEvent, 32) + events := make(chan NewTxsEvent, 32) sub := pool.txFeed.Subscribe(events) defer sub.Unsubscribe() // Create a number of test accounts and fund them - keys := make([]*ecdsa.PrivateKey, 3) + keys := make([]*ecdsa.PrivateKey, 4) for i := 0; i < len(keys); i++ { keys[i], _ = crypto.GenerateKey() pool.currentState.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) @@ -1406,18 +1398,22 @@ func TestTransactionPoolUnderpricing(t *testing.T) { t.Fatalf("pool internal state corrupted: %v", err) } // Ensure that adding local transactions can push out even higher priced ones - tx := pricedTransaction(1, 100000, big.NewInt(0), keys[2]) - if err := pool.AddLocal(tx); err != nil { - t.Fatalf("failed to add underpriced local transaction: %v", err) + ltx = pricedTransaction(1, 100000, big.NewInt(0), keys[2]) + if err := pool.AddLocal(ltx); err != nil { + t.Fatalf("failed to append underpriced local transaction: %v", err) + } + ltx = pricedTransaction(0, 100000, big.NewInt(0), keys[3]) + if err := pool.AddLocal(ltx); err != nil { + t.Fatalf("failed to add new underpriced local transaction: %v", err) } pending, queued = pool.Stats() - if pending != 2 { - t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 2) + if pending != 3 { + t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 3) } - if queued != 2 { - t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 2) + if queued != 1 { + t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 1) } - if err := validateEvents(events, 1); err != nil { + if err := validateEvents(events, 2); err != nil { t.Fatalf("local event firing failed: %v", err) } if err := validateTxPoolInternals(pool); err != nil { @@ -1432,8 +1428,7 @@ func TestTransactionPoolStableUnderpricing(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1444,7 +1439,7 @@ func TestTransactionPoolStableUnderpricing(t *testing.T) { defer pool.Stop() // Keep track of transaction events to ensure all executables get announced - events := make(chan TxPreEvent, 32) + events := make(chan NewTxsEvent, 32) sub := pool.txFeed.Subscribe(events) defer sub.Unsubscribe() @@ -1499,15 +1494,14 @@ func TestTransactionReplacement(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) defer pool.Stop() // Keep track of transaction events to ensure all executables get announced - events := make(chan TxPreEvent, 32) + events := make(chan NewTxsEvent, 32) sub := pool.txFeed.Subscribe(events) defer sub.Unsubscribe() @@ -1594,8 +1588,7 @@ func testTransactionJournaling(t *testing.T, nolocals bool) { os.Remove(journal) // Create the original pool to inject transaction into the journal - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1693,8 +1686,7 @@ func TestTransactionStatusCheck(t *testing.T) { t.Parallel() // Create the pool to test the status retrievals with - db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) diff --git a/core/types/gen_receipt_json.go b/core/types/gen_receipt_json.go index fba440df8f..076f9b7fbe 100644 --- a/core/types/gen_receipt_json.go +++ b/core/types/gen_receipt_json.go @@ -12,10 +12,11 @@ import ( var _ = (*receiptMarshaling)(nil) +// MarshalJSON marshals as JSON. func (r Receipt) MarshalJSON() ([]byte, error) { type Receipt struct { PostState hexutil.Bytes `json:"root"` - Status hexutil.Uint `json:"status"` + Status hexutil.Uint64 `json:"status"` CumulativeGasUsed hexutil.Uint64 `json:"cumulativeGasUsed" gencodec:"required"` Bloom Bloom `json:"logsBloom" gencodec:"required"` Logs []*Log `json:"logs" gencodec:"required"` @@ -25,7 +26,7 @@ func (r Receipt) MarshalJSON() ([]byte, error) { } var enc Receipt enc.PostState = r.PostState - enc.Status = hexutil.Uint(r.Status) + enc.Status = hexutil.Uint64(r.Status) enc.CumulativeGasUsed = hexutil.Uint64(r.CumulativeGasUsed) enc.Bloom = r.Bloom enc.Logs = r.Logs @@ -35,10 +36,11 @@ func (r Receipt) MarshalJSON() ([]byte, error) { return json.Marshal(&enc) } +// UnmarshalJSON unmarshals from JSON. func (r *Receipt) UnmarshalJSON(input []byte) error { type Receipt struct { PostState *hexutil.Bytes `json:"root"` - Status *hexutil.Uint `json:"status"` + Status *hexutil.Uint64 `json:"status"` CumulativeGasUsed *hexutil.Uint64 `json:"cumulativeGasUsed" gencodec:"required"` Bloom *Bloom `json:"logsBloom" gencodec:"required"` Logs []*Log `json:"logs" gencodec:"required"` @@ -54,7 +56,7 @@ func (r *Receipt) UnmarshalJSON(input []byte) error { r.PostState = *dec.PostState } if dec.Status != nil { - r.Status = uint(*dec.Status) + r.Status = uint64(*dec.Status) } if dec.CumulativeGasUsed == nil { return errors.New("missing required field 'cumulativeGasUsed' for Receipt") diff --git a/core/types/receipt.go b/core/types/receipt.go index 36b0b5c97d..1fef1a922f 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -36,17 +36,17 @@ var ( const ( // ReceiptStatusFailed is the status code of a transaction if execution failed. - ReceiptStatusFailed = uint(0) + ReceiptStatusFailed = uint64(0) // ReceiptStatusSuccessful is the status code of a transaction if execution succeeded. - ReceiptStatusSuccessful = uint(1) + ReceiptStatusSuccessful = uint64(1) ) // Receipt represents the results of a transaction. type Receipt struct { // Consensus fields PostState []byte `json:"root"` - Status uint `json:"status"` + Status uint64 `json:"status"` CumulativeGasUsed uint64 `json:"cumulativeGasUsed" gencodec:"required"` Bloom Bloom `json:"logsBloom" gencodec:"required"` Logs []*Log `json:"logs" gencodec:"required"` @@ -59,7 +59,7 @@ type Receipt struct { type receiptMarshaling struct { PostState hexutil.Bytes - Status hexutil.Uint + Status hexutil.Uint64 CumulativeGasUsed hexutil.Uint64 GasUsed hexutil.Uint64 } diff --git a/core/types/transaction.go b/core/types/transaction.go index 9410526692..5583631703 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -339,11 +339,14 @@ type TransactionsByPriceAndNonce struct { func NewTransactionsByPriceAndNonce(signer Signer, txs map[common.Address]Transactions) *TransactionsByPriceAndNonce { // Initialize a price based heap with the head transactions heads := make(TxByPrice, 0, len(txs)) - for _, accTxs := range txs { + for from, accTxs := range txs { heads = append(heads, accTxs[0]) // Ensure the sender address is from the signer acc, _ := Sender(signer, accTxs[0]) txs[acc] = accTxs[1:] + if from != acc { + delete(txs, from) + } } heap.Init(&heads) diff --git a/core/vm/contract.go b/core/vm/contract.go index bee98fb377..7bbda646f0 100644 --- a/core/vm/contract.go +++ b/core/vm/contract.go @@ -139,15 +139,15 @@ func (c *Contract) Value() *big.Int { } // SetCode sets the code to the contract -func (self *Contract) SetCode(hash common.Hash, code []byte) { - self.Code = code - self.CodeHash = hash +func (c *Contract) SetCode(hash common.Hash, code []byte) { + c.Code = code + c.CodeHash = hash } // SetCallCode sets the code of the contract and address of the backing data // object -func (self *Contract) SetCallCode(addr *common.Address, hash common.Hash, code []byte) { - self.Code = code - self.CodeHash = hash - self.CodeAddr = addr +func (c *Contract) SetCallCode(addr *common.Address, hash common.Hash, code []byte) { + c.Code = code + c.CodeHash = hash + c.CodeAddr = addr } diff --git a/core/vm/evm.go b/core/vm/evm.go index 9467bc26f8..51f4706c05 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -160,6 +160,11 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas precompiles = PrecompiledContractsByzantium } if precompiles[addr] == nil && evm.ChainConfig().IsEIP158(evm.BlockNumber) && value.Sign() == 0 { + // Calling a non existing account, don't do antything, but ping the tracer + if evm.vmConfig.Debug && evm.depth == 0 { + evm.vmConfig.Tracer.CaptureStart(caller.Address(), addr, false, input, gas, value) + evm.vmConfig.Tracer.CaptureEnd(ret, 0, 0, nil) + } return nil, gas, nil } evm.StateDB.CreateAccount(addr) diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 7a901f2928..c5055a23a7 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -850,7 +850,7 @@ func makePush(size uint64, pushByteSize int) executionFunc { } } -// make push instruction function +// make dup instruction function func makeDup(size int64) executionFunc { return func(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { stack.dup(evm.interpreter.intPool, int(size)) diff --git a/core/vm/logger.go b/core/vm/logger.go index 179f3abe3b..91d7626918 100644 --- a/core/vm/logger.go +++ b/core/vm/logger.go @@ -31,9 +31,9 @@ import ( type Storage map[common.Hash]common.Hash -func (self Storage) Copy() Storage { +func (s Storage) Copy() Storage { cpy := make(Storage) - for key, value := range self { + for key, value := range s { cpy[key] = value } diff --git a/core/vm/memory.go b/core/vm/memory.go index 99a84d2271..d5cc7870bc 100644 --- a/core/vm/memory.go +++ b/core/vm/memory.go @@ -51,14 +51,14 @@ func (m *Memory) Resize(size uint64) { } // Get returns offset + size as a new slice -func (self *Memory) Get(offset, size int64) (cpy []byte) { +func (m *Memory) Get(offset, size int64) (cpy []byte) { if size == 0 { return nil } - if len(self.store) > int(offset) { + if len(m.store) > int(offset) { cpy = make([]byte, size) - copy(cpy, self.store[offset:offset+size]) + copy(cpy, m.store[offset:offset+size]) return } @@ -67,13 +67,13 @@ func (self *Memory) Get(offset, size int64) (cpy []byte) { } // GetPtr returns the offset + size -func (self *Memory) GetPtr(offset, size int64) []byte { +func (m *Memory) GetPtr(offset, size int64) []byte { if size == 0 { return nil } - if len(self.store) > int(offset) { - return self.store[offset : offset+size] + if len(m.store) > int(offset) { + return m.store[offset : offset+size] } return nil diff --git a/core/vm/opcodes.go b/core/vm/opcodes.go index 7fe55b72f6..e3568eb000 100644 --- a/core/vm/opcodes.go +++ b/core/vm/opcodes.go @@ -375,10 +375,10 @@ var opCodeToString = map[OpCode]string{ SWAP: "SWAP", } -func (o OpCode) String() string { - str := opCodeToString[o] +func (op OpCode) String() string { + str := opCodeToString[op] if len(str) == 0 { - return fmt.Sprintf("Missing opcode 0x%x", int(o)) + return fmt.Sprintf("Missing opcode 0x%x", int(op)) } return str diff --git a/core/vm/runtime/runtime.go b/core/vm/runtime/runtime.go index aea36f4dfc..02fd637ec2 100644 --- a/core/vm/runtime/runtime.go +++ b/core/vm/runtime/runtime.go @@ -99,8 +99,7 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) { setDefaults(cfg) if cfg.State == nil { - db, _ := ethdb.NewMemDatabase() - cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(db)) + cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) } var ( address = common.BytesToAddress([]byte("contract")) @@ -130,8 +129,7 @@ func Create(input []byte, cfg *Config) ([]byte, common.Address, uint64, error) { setDefaults(cfg) if cfg.State == nil { - db, _ := ethdb.NewMemDatabase() - cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(db)) + cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) } var ( vmenv = NewEnv(cfg) diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index acaaefa763..8085d2efc2 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -94,8 +94,7 @@ func TestExecute(t *testing.T) { } func TestCall(t *testing.T) { - db, _ := ethdb.NewMemDatabase() - state, _ := state.New(common.Hash{}, state.NewDatabase(db)) + state, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) address := common.HexToAddress("0x0a") state.SetCode(address, []byte{ byte(vm.PUSH1), 10, diff --git a/crypto/crypto.go b/crypto/crypto.go index 281a6763d7..ff7d181773 100644 --- a/crypto/crypto.go +++ b/crypto/crypto.go @@ -35,8 +35,8 @@ import ( ) var ( - secp256k1_N, _ = new(big.Int).SetString("fffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141", 16) - secp256k1_halfN = new(big.Int).Div(secp256k1_N, big.NewInt(2)) + secp256k1N, _ = new(big.Int).SetString("fffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141", 16) + secp256k1halfN = new(big.Int).Div(secp256k1N, big.NewInt(2)) ) // Keccak256 calculates and returns the Keccak256 hash of the input data. @@ -68,7 +68,7 @@ func Keccak512(data ...[]byte) []byte { return d.Sum(nil) } -// Creates an ethereum address given the bytes and the nonce +// CreateAddress creates an ethereum address given the bytes and the nonce func CreateAddress(b common.Address, nonce uint64) common.Address { data, _ := rlp.EncodeToBytes([]interface{}{b, nonce}) return common.BytesToAddress(Keccak256(data)[12:]) @@ -99,7 +99,7 @@ func toECDSA(d []byte, strict bool) (*ecdsa.PrivateKey, error) { priv.D = new(big.Int).SetBytes(d) // The priv.D must < N - if priv.D.Cmp(secp256k1_N) >= 0 { + if priv.D.Cmp(secp256k1N) >= 0 { return nil, fmt.Errorf("invalid private key, >=N") } // The priv.D must not be zero or negative. @@ -184,11 +184,11 @@ func ValidateSignatureValues(v byte, r, s *big.Int, homestead bool) bool { } // reject upper range of s values (ECDSA malleability) // see discussion in secp256k1/libsecp256k1/include/secp256k1.h - if homestead && s.Cmp(secp256k1_halfN) > 0 { + if homestead && s.Cmp(secp256k1halfN) > 0 { return false } // Frontier: allow s to be in full N range - return r.Cmp(secp256k1_N) < 0 && s.Cmp(secp256k1_N) < 0 && (v == 0 || v == 1) + return r.Cmp(secp256k1N) < 0 && s.Cmp(secp256k1N) < 0 && (v == 0 || v == 1) } func PubkeyToAddress(p ecdsa.PublicKey) common.Address { diff --git a/crypto/crypto_test.go b/crypto/crypto_test.go index 3c12d9a70b..18e495fdb0 100644 --- a/crypto/crypto_test.go +++ b/crypto/crypto_test.go @@ -154,7 +154,7 @@ func TestValidateSignatureValues(t *testing.T) { minusOne := big.NewInt(-1) one := common.Big1 zero := common.Big0 - secp256k1nMinus1 := new(big.Int).Sub(secp256k1_N, common.Big1) + secp256k1nMinus1 := new(big.Int).Sub(secp256k1N, common.Big1) // correct v,r,s check(true, 0, one, one) @@ -181,9 +181,9 @@ func TestValidateSignatureValues(t *testing.T) { // correct sig with max r,s check(true, 0, secp256k1nMinus1, secp256k1nMinus1) // correct v, combinations of incorrect r,s at upper limit - check(false, 0, secp256k1_N, secp256k1nMinus1) - check(false, 0, secp256k1nMinus1, secp256k1_N) - check(false, 0, secp256k1_N, secp256k1_N) + check(false, 0, secp256k1N, secp256k1nMinus1) + check(false, 0, secp256k1nMinus1, secp256k1N) + check(false, 0, secp256k1N, secp256k1N) // current callers ensures r,s cannot be negative, but let's test for that too // as crypto package could be used stand-alone diff --git a/crypto/secp256k1/curve.go b/crypto/secp256k1/curve.go index b790f9c5f4..a45356dd41 100644 --- a/crypto/secp256k1/curve.go +++ b/crypto/secp256k1/curve.go @@ -77,7 +77,7 @@ func (BitCurve *BitCurve) Params() *elliptic.CurveParams { } } -// IsOnBitCurve returns true if the given (x,y) lies on the BitCurve. +// IsOnCurve returns true if the given (x,y) lies on the BitCurve. func (BitCurve *BitCurve) IsOnCurve(x, y *big.Int) bool { // y² = x³ + b y2 := new(big.Int).Mul(y, y) //y² diff --git a/crypto/secp256k1/secp256_test.go b/crypto/secp256k1/secp256_test.go index d109a6517a..006c8b859f 100644 --- a/crypto/secp256k1/secp256_test.go +++ b/crypto/secp256k1/secp256_test.go @@ -49,7 +49,7 @@ func randSig() []byte { // tests for malleability // highest bit of signature ECDSA s value must be 0, in the 33th byte func compactSigCheck(t *testing.T, sig []byte) { - var b int = int(sig[32]) + var b = int(sig[32]) if b < 0 { t.Errorf("highest bit is negative: %d", b) } diff --git a/crypto/signature_nocgo.go b/crypto/signature_nocgo.go index f636b23772..e8fa18ed47 100644 --- a/crypto/signature_nocgo.go +++ b/crypto/signature_nocgo.go @@ -88,7 +88,7 @@ func VerifySignature(pubkey, hash, signature []byte) bool { return false } // Reject malleable signatures. libsecp256k1 does this check but btcec doesn't. - if sig.S.Cmp(secp256k1_halfN) > 0 { + if sig.S.Cmp(secp256k1halfN) > 0 { return false } return sig.Verify(hash, key) diff --git a/eth/api.go b/eth/api.go index ae7ad18941..875e19dfae 100644 --- a/eth/api.go +++ b/eth/api.go @@ -19,6 +19,7 @@ package eth import ( "compress/gzip" "context" + "errors" "fmt" "io" "math/big" @@ -28,6 +29,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common" "github.com/EthereumCommonwealth/go-callisto/common/hexutil" "github.com/EthereumCommonwealth/go-callisto/core" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/state" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/log" @@ -343,8 +345,10 @@ func NewPrivateDebugAPI(config *params.ChainConfig, eth *Ethereum) *PrivateDebug // Preimage is a debug API function that returns the preimage for a sha3 hash, if known. func (api *PrivateDebugAPI) Preimage(ctx context.Context, hash common.Hash) (hexutil.Bytes, error) { - db := core.PreimageTable(api.eth.ChainDb()) - return db.Get(hash.Bytes()) + if preimage := rawdb.ReadPreimage(api.eth.ChainDb(), hash); preimage != nil { + return preimage, nil + } + return nil, errors.New("unknown preimage") } // GetBadBLocks returns a list of the last 'bad blocks' that the client has seen on the network diff --git a/eth/api_backend.go b/eth/api_backend.go index b4a7745a7d..59b2c05e77 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -25,6 +25,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common/math" "github.com/EthereumCommonwealth/go-callisto/core" "github.com/EthereumCommonwealth/go-callisto/core/bloombits" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/state" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/core/vm" @@ -36,26 +37,26 @@ import ( "github.com/EthereumCommonwealth/go-callisto/rpc" ) -// EthApiBackend implements ethapi.Backend for full nodes -type EthApiBackend struct { +// EthAPIBackend implements ethapi.Backend for full nodes +type EthAPIBackend struct { eth *Ethereum gpo *gasprice.Oracle } -func (b *EthApiBackend) ChainConfig() *params.ChainConfig { +func (b *EthAPIBackend) ChainConfig() *params.ChainConfig { return b.eth.chainConfig } -func (b *EthApiBackend) CurrentBlock() *types.Block { +func (b *EthAPIBackend) CurrentBlock() *types.Block { return b.eth.blockchain.CurrentBlock() } -func (b *EthApiBackend) SetHead(number uint64) { +func (b *EthAPIBackend) SetHead(number uint64) { b.eth.protocolManager.downloader.Cancel() b.eth.blockchain.SetHead(number) } -func (b *EthApiBackend) HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) { +func (b *EthAPIBackend) HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) { // Pending block is only known by the miner if blockNr == rpc.PendingBlockNumber { block := b.eth.miner.PendingBlock() @@ -68,7 +69,7 @@ func (b *EthApiBackend) HeaderByNumber(ctx context.Context, blockNr rpc.BlockNum return b.eth.blockchain.GetHeaderByNumber(uint64(blockNr)), nil } -func (b *EthApiBackend) BlockByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Block, error) { +func (b *EthAPIBackend) BlockByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Block, error) { // Pending block is only known by the miner if blockNr == rpc.PendingBlockNumber { block := b.eth.miner.PendingBlock() @@ -81,7 +82,7 @@ func (b *EthApiBackend) BlockByNumber(ctx context.Context, blockNr rpc.BlockNumb return b.eth.blockchain.GetBlockByNumber(uint64(blockNr)), nil } -func (b *EthApiBackend) StateAndHeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*state.StateDB, *types.Header, error) { +func (b *EthAPIBackend) StateAndHeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*state.StateDB, *types.Header, error) { // Pending state is only known by the miner if blockNr == rpc.PendingBlockNumber { block, state := b.eth.miner.Pending() @@ -96,16 +97,23 @@ func (b *EthApiBackend) StateAndHeaderByNumber(ctx context.Context, blockNr rpc. return stateDb, header, err } -func (b *EthApiBackend) GetBlock(ctx context.Context, blockHash common.Hash) (*types.Block, error) { - return b.eth.blockchain.GetBlockByHash(blockHash), nil +func (b *EthAPIBackend) GetBlock(ctx context.Context, hash common.Hash) (*types.Block, error) { + return b.eth.blockchain.GetBlockByHash(hash), nil } -func (b *EthApiBackend) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) { - return core.GetBlockReceipts(b.eth.chainDb, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash)), nil +func (b *EthAPIBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { + if number := rawdb.ReadHeaderNumber(b.eth.chainDb, hash); number != nil { + return rawdb.ReadReceipts(b.eth.chainDb, hash, *number), nil + } + return nil, nil } -func (b *EthApiBackend) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) { - receipts := core.GetBlockReceipts(b.eth.chainDb, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash)) +func (b *EthAPIBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { + number := rawdb.ReadHeaderNumber(b.eth.chainDb, hash) + if number == nil { + return nil, nil + } + receipts := rawdb.ReadReceipts(b.eth.chainDb, hash, *number) if receipts == nil { return nil, nil } @@ -116,11 +124,11 @@ func (b *EthApiBackend) GetLogs(ctx context.Context, blockHash common.Hash) ([][ return logs, nil } -func (b *EthApiBackend) GetTd(blockHash common.Hash) *big.Int { +func (b *EthAPIBackend) GetTd(blockHash common.Hash) *big.Int { return b.eth.blockchain.GetTdByHash(blockHash) } -func (b *EthApiBackend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) { +func (b *EthAPIBackend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) { state.SetBalance(msg.From(), math.MaxBig256) vmError := func() error { return nil } @@ -128,31 +136,31 @@ func (b *EthApiBackend) GetEVM(ctx context.Context, msg core.Message, state *sta return vm.NewEVM(context, state, b.eth.chainConfig, vmCfg), vmError, nil } -func (b *EthApiBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { +func (b *EthAPIBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { return b.eth.BlockChain().SubscribeRemovedLogsEvent(ch) } -func (b *EthApiBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { +func (b *EthAPIBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { return b.eth.BlockChain().SubscribeChainEvent(ch) } -func (b *EthApiBackend) SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription { +func (b *EthAPIBackend) SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription { return b.eth.BlockChain().SubscribeChainHeadEvent(ch) } -func (b *EthApiBackend) SubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription { +func (b *EthAPIBackend) SubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription { return b.eth.BlockChain().SubscribeChainSideEvent(ch) } -func (b *EthApiBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { +func (b *EthAPIBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { return b.eth.BlockChain().SubscribeLogsEvent(ch) } -func (b *EthApiBackend) SendTx(ctx context.Context, signedTx *types.Transaction) error { +func (b *EthAPIBackend) SendTx(ctx context.Context, signedTx *types.Transaction) error { return b.eth.txPool.AddLocal(signedTx) } -func (b *EthApiBackend) GetPoolTransactions() (types.Transactions, error) { +func (b *EthAPIBackend) GetPoolTransactions() (types.Transactions, error) { pending, err := b.eth.txPool.Pending() if err != nil { return nil, err @@ -164,56 +172,56 @@ func (b *EthApiBackend) GetPoolTransactions() (types.Transactions, error) { return txs, nil } -func (b *EthApiBackend) GetPoolTransaction(hash common.Hash) *types.Transaction { +func (b *EthAPIBackend) GetPoolTransaction(hash common.Hash) *types.Transaction { return b.eth.txPool.Get(hash) } -func (b *EthApiBackend) GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) { +func (b *EthAPIBackend) GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) { return b.eth.txPool.State().GetNonce(addr), nil } -func (b *EthApiBackend) Stats() (pending int, queued int) { +func (b *EthAPIBackend) Stats() (pending int, queued int) { return b.eth.txPool.Stats() } -func (b *EthApiBackend) TxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions) { +func (b *EthAPIBackend) TxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions) { return b.eth.TxPool().Content() } -func (b *EthApiBackend) SubscribeTxPreEvent(ch chan<- core.TxPreEvent) event.Subscription { - return b.eth.TxPool().SubscribeTxPreEvent(ch) +func (b *EthAPIBackend) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subscription { + return b.eth.TxPool().SubscribeNewTxsEvent(ch) } -func (b *EthApiBackend) Downloader() *downloader.Downloader { +func (b *EthAPIBackend) Downloader() *downloader.Downloader { return b.eth.Downloader() } -func (b *EthApiBackend) ProtocolVersion() int { +func (b *EthAPIBackend) ProtocolVersion() int { return b.eth.EthVersion() } -func (b *EthApiBackend) SuggestPrice(ctx context.Context) (*big.Int, error) { +func (b *EthAPIBackend) SuggestPrice(ctx context.Context) (*big.Int, error) { return b.gpo.SuggestPrice(ctx) } -func (b *EthApiBackend) ChainDb() ethdb.Database { +func (b *EthAPIBackend) ChainDb() ethdb.Database { return b.eth.ChainDb() } -func (b *EthApiBackend) EventMux() *event.TypeMux { +func (b *EthAPIBackend) EventMux() *event.TypeMux { return b.eth.EventMux() } -func (b *EthApiBackend) AccountManager() *accounts.Manager { +func (b *EthAPIBackend) AccountManager() *accounts.Manager { return b.eth.AccountManager() } -func (b *EthApiBackend) BloomStatus() (uint64, uint64) { +func (b *EthAPIBackend) BloomStatus() (uint64, uint64) { sections, _, _ := b.eth.bloomIndexer.Sections() return params.BloomBitsBlocks, sections } -func (b *EthApiBackend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) { +func (b *EthAPIBackend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) { for i := 0; i < bloomFilterThreads; i++ { go session.Multiplex(bloomRetrievalBatch, bloomRetrievalWait, b.eth.bloomRequests) } diff --git a/eth/api_test.go b/eth/api_test.go index a4918da09c..b723ba14b9 100644 --- a/eth/api_test.go +++ b/eth/api_test.go @@ -31,8 +31,7 @@ var dumper = spew.ConfigState{Indent: " "} func TestStorageRangeAt(t *testing.T) { // Create a state where account 0x010000... has a few storage entries. var ( - db, _ = ethdb.NewMemDatabase() - state, _ = state.New(common.Hash{}, state.NewDatabase(db)) + state, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) addr = common.Address{0x01} keys = []common.Hash{ // hashes of Keys of storage common.HexToHash("340dd630ad21bf010b4e676dbfa9ba9a02175262d1fa356232cfde6cb5b47ef2"), diff --git a/eth/api_tracer.go b/eth/api_tracer.go index 4d61ecb2ce..14e48274db 100644 --- a/eth/api_tracer.go +++ b/eth/api_tracer.go @@ -29,6 +29,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common" "github.com/EthereumCommonwealth/go-callisto/common/hexutil" "github.com/EthereumCommonwealth/go-callisto/core" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/state" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/core/vm" @@ -201,7 +202,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl log.Warn("Tracing failed", "hash", tx.Hash(), "block", task.block.NumberU64(), "err", err) break } - task.statedb.DeleteSuicides() + task.statedb.Finalise(true) task.results[i] = &txTraceResult{Result: res} } // Stream the result back to the user or abort on teardown @@ -533,7 +534,7 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (* // and returns them as a JSON object. func (api *PrivateDebugAPI) TraceTransaction(ctx context.Context, hash common.Hash, config *TraceConfig) (interface{}, error) { // Retrieve the transaction and assemble its EVM context - tx, blockHash, _, index := core.GetTransaction(api.eth.ChainDb(), hash) + tx, blockHash, _, index := rawdb.ReadTransaction(api.eth.ChainDb(), hash) if tx == nil { return nil, fmt.Errorf("transaction %x not found", hash) } @@ -640,7 +641,8 @@ func (api *PrivateDebugAPI) computeTxEnv(blockHash common.Hash, txIndex int, ree if _, _, _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { return nil, vm.Context{}, nil, fmt.Errorf("tx %x failed: %v", tx.Hash(), err) } - statedb.DeleteSuicides() + // Ensure any modifications are committed to the state + statedb.Finalise(true) } return nil, vm.Context{}, nil, fmt.Errorf("tx index %d out of range for block %x", txIndex, blockHash) } diff --git a/eth/backend.go b/eth/backend.go index aefe68a98b..43267303be 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -33,6 +33,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/consensus/ethash" "github.com/EthereumCommonwealth/go-callisto/core" "github.com/EthereumCommonwealth/go-callisto/core/bloombits" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/core/vm" "github.com/EthereumCommonwealth/go-callisto/eth/downloader" @@ -63,8 +64,7 @@ type Ethereum struct { chainConfig *params.ChainConfig // Channel for shutting down the service - shutdownChan chan bool // Channel for shutting down the Ethereum - stopDbUpgrade func() error // stop chain db sequential key upgrade + shutdownChan chan bool // Channel for shutting down the Ethereum // Handlers txPool *core.TxPool @@ -82,7 +82,7 @@ type Ethereum struct { bloomRequests chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests bloomIndexer *core.ChainIndexer // Bloom indexer operating during block imports - ApiBackend *EthApiBackend + APIBackend *EthAPIBackend miner *miner.Miner gasPrice *big.Int @@ -112,7 +112,6 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { if err != nil { return nil, err } - stopDbUpgrade := upgradeDeduplicateData(chainDb) chainConfig, genesisHash, genesisErr := core.SetupGenesisBlock(chainDb, config.Genesis) if _, ok := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !ok { return nil, genesisErr @@ -127,7 +126,6 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { accountManager: ctx.AccountManager, engine: CreateConsensusEngine(ctx, &config.Ethash, chainConfig, chainDb), shutdownChan: make(chan bool), - stopDbUpgrade: stopDbUpgrade, networkId: config.NetworkId, gasPrice: config.GasPrice, etherbase: config.Etherbase, @@ -138,11 +136,11 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { log.Info("Initialising Ethereum protocol", "versions", ProtocolVersions, "network", config.NetworkId) if !config.SkipBcVersionCheck { - bcVersion := core.GetBlockChainVersion(chainDb) + bcVersion := rawdb.ReadDatabaseVersion(chainDb) if bcVersion != core.BlockChainVersion && bcVersion != 0 { return nil, fmt.Errorf("Blockchain DB version mismatch (%d / %d). Run geth upgradedb.\n", bcVersion, core.BlockChainVersion) } - core.WriteBlockChainVersion(chainDb, core.BlockChainVersion) + rawdb.WriteDatabaseVersion(chainDb, core.BlockChainVersion) } var ( vmConfig = vm.Config{EnablePreimageRecording: config.EnablePreimageRecording} @@ -156,7 +154,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { if compat, ok := genesisErr.(*params.ConfigCompatError); ok { log.Warn("Rewinding chain to upgrade configuration", "err", compat) eth.blockchain.SetHead(compat.RewindTo) - core.WriteChainConfig(chainDb, genesisHash, chainConfig) + rawdb.WriteChainConfig(chainDb, genesisHash, chainConfig) } eth.bloomIndexer.Start(eth.blockchain) @@ -171,12 +169,12 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { eth.miner = miner.New(eth, eth.chainConfig, eth.EventMux(), eth.engine) eth.miner.SetExtra(makeExtraData(config.ExtraData)) - eth.ApiBackend = &EthApiBackend{eth, nil} + eth.APIBackend = &EthAPIBackend{eth, nil} gpoParams := config.GPO if gpoParams.Default == nil { gpoParams.Default = config.GasPrice } - eth.ApiBackend.gpo = gasprice.NewOracle(eth.ApiBackend, gpoParams) + eth.APIBackend.gpo = gasprice.NewOracle(eth.APIBackend, gpoParams) return eth, nil } @@ -217,14 +215,14 @@ func CreateConsensusEngine(ctx *node.ServiceContext, config *ethash.Config, chai return clique.New(chainConfig.Clique, db) } // Otherwise assume proof-of-work - switch { - case config.PowMode == ethash.ModeFake: + switch config.PowMode { + case ethash.ModeFake: log.Warn("Ethash used in fake mode") return ethash.NewFaker() - case config.PowMode == ethash.ModeTest: + case ethash.ModeTest: log.Warn("Ethash used in test mode") return ethash.NewTester() - case config.PowMode == ethash.ModeShared: + case ethash.ModeShared: log.Warn("Ethash used in shared mode") return ethash.NewShared() default: @@ -241,10 +239,10 @@ func CreateConsensusEngine(ctx *node.ServiceContext, config *ethash.Config, chai } } -// APIs returns the collection of RPC services the ethereum package offers. +// APIs return the collection of RPC services the ethereum package offers. // NOTE, some of these services probably need to be moved to somewhere else. func (s *Ethereum) APIs() []rpc.API { - apis := ethapi.GetAPIs(s.ApiBackend) + apis := ethapi.GetAPIs(s.APIBackend) // Append any APIs exposed explicitly by the consensus engine apis = append(apis, s.engine.APIs(s.BlockChain())...) @@ -274,7 +272,7 @@ func (s *Ethereum) APIs() []rpc.API { }, { Namespace: "eth", Version: "1.0", - Service: filters.NewPublicFilterAPI(s.ApiBackend, false), + Service: filters.NewPublicFilterAPI(s.APIBackend, false), Public: true, }, { Namespace: "admin", @@ -325,13 +323,13 @@ func (s *Ethereum) Etherbase() (eb common.Address, err error) { return common.Address{}, fmt.Errorf("etherbase must be explicitly specified") } -// set in js console via admin interface or wrapper from cli flags -func (self *Ethereum) SetEtherbase(etherbase common.Address) { - self.lock.Lock() - self.etherbase = etherbase - self.lock.Unlock() +// SetEtherbase sets the mining reward address. +func (s *Ethereum) SetEtherbase(etherbase common.Address) { + s.lock.Lock() + s.etherbase = etherbase + s.lock.Unlock() - self.miner.SetEtherbase(etherbase) + s.miner.SetEtherbase(etherbase) } func (s *Ethereum) StartMining(local bool) error { @@ -411,9 +409,6 @@ func (s *Ethereum) Start(srvr *p2p.Server) error { // Stop implements node.Service, terminating all internal goroutines used by the // Ethereum protocol. func (s *Ethereum) Stop() error { - if s.stopDbUpgrade != nil { - s.stopDbUpgrade() - } s.bloomIndexer.Close() s.blockchain.Stop() s.protocolManager.Stop() diff --git a/eth/bloombits.go b/eth/bloombits.go index 0acc476530..0dbb6f8ffe 100644 --- a/eth/bloombits.go +++ b/eth/bloombits.go @@ -23,6 +23,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common/bitutil" "github.com/EthereumCommonwealth/go-callisto/core" "github.com/EthereumCommonwealth/go-callisto/core/bloombits" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/ethdb" "github.com/EthereumCommonwealth/go-callisto/params" @@ -60,8 +61,8 @@ func (eth *Ethereum) startBloomHandlers() { task := <-request task.Bitsets = make([][]byte, len(task.Sections)) for i, section := range task.Sections { - head := core.GetCanonicalHash(eth.chainDb, (section+1)*params.BloomBitsBlocks-1) - if compVector, err := core.GetBloomBits(eth.chainDb, task.Bit, section, head); err == nil { + head := rawdb.ReadCanonicalHash(eth.chainDb, (section+1)*params.BloomBitsBlocks-1) + if compVector, err := rawdb.ReadBloomBits(eth.chainDb, task.Bit, section, head); err == nil { if blob, err := bitutil.DecompressBytes(compVector, int(params.BloomBitsBlocks)/8); err == nil { task.Bitsets[i] = blob } else { @@ -107,7 +108,7 @@ func NewBloomIndexer(db ethdb.Database, size uint64) *core.ChainIndexer { db: db, size: size, } - table := ethdb.NewTable(db, string(core.BloomBitsIndexPrefix)) + table := ethdb.NewTable(db, string(rawdb.BloomBitsIndexPrefix)) return core.NewChainIndexer(db, table, backend, size, bloomConfirms, bloomThrottling, "bloombits") } @@ -137,7 +138,7 @@ func (b *BloomIndexer) Commit() error { if err != nil { return err } - core.WriteBloomBits(batch, uint(i), b.section, b.head, bitutil.CompressBytes(bits)) + rawdb.WriteBloomBits(batch, uint(i), b.section, b.head, bitutil.CompressBytes(bits)) } return batch.Write() } diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index b4fe3f95a4..7da30b6988 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -27,7 +27,7 @@ import ( ethereum "github.com/EthereumCommonwealth/go-callisto" "github.com/EthereumCommonwealth/go-callisto/common" - "github.com/EthereumCommonwealth/go-callisto/core" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/ethdb" "github.com/EthereumCommonwealth/go-callisto/event" @@ -224,7 +224,7 @@ func New(mode SyncMode, stateDb ethdb.Database, mux *event.TypeMux, chain BlockC stateCh: make(chan dataPack), stateSyncStart: make(chan *stateSync), syncStatsState: stateSyncStats{ - processed: core.GetTrieSyncProgress(stateDb), + processed: rawdb.ReadFastTrieProgress(stateDb), }, trackStateReq: make(chan *stateReq), } diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 79314fc025..81d114e29b 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -75,7 +75,7 @@ type downloadTester struct { // newTester creates a new downloader test mocker. func newTester() *downloadTester { - testdb, _ := ethdb.NewMemDatabase() + testdb := ethdb.NewMemDatabase() genesis := core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000)) tester := &downloadTester{ @@ -93,7 +93,7 @@ func newTester() *downloadTester { peerChainTds: make(map[string]map[common.Hash]*big.Int), peerMissingStates: make(map[string]map[common.Hash]bool), } - tester.stateDb, _ = ethdb.NewMemDatabase() + tester.stateDb = ethdb.NewMemDatabase() tester.stateDb.Put(genesis.Root().Bytes(), []byte{0x00}) tester.downloader = New(FullSync, tester.stateDb, new(event.TypeMux), tester, nil, tester.dropPeer) diff --git a/eth/downloader/fakepeer.go b/eth/downloader/fakepeer.go index de15ee5657..5a75e59adc 100644 --- a/eth/downloader/fakepeer.go +++ b/eth/downloader/fakepeer.go @@ -21,6 +21,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common" "github.com/EthereumCommonwealth/go-callisto/core" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/ethdb" ) @@ -126,7 +127,7 @@ func (p *FakePeer) RequestBodies(hashes []common.Hash) error { uncles [][]*types.Header ) for _, hash := range hashes { - block := core.GetBlock(p.db, hash, p.hc.GetBlockNumber(hash)) + block := rawdb.ReadBlock(p.db, hash, *p.hc.GetBlockNumber(hash)) txs = append(txs, block.Transactions()) uncles = append(uncles, block.Uncles()) @@ -140,7 +141,7 @@ func (p *FakePeer) RequestBodies(hashes []common.Hash) error { func (p *FakePeer) RequestReceipts(hashes []common.Hash) error { var receipts [][]*types.Receipt for _, hash := range hashes { - receipts = append(receipts, core.GetBlockReceipts(p.db, hash, p.hc.GetBlockNumber(hash))) + receipts = append(receipts, rawdb.ReadReceipts(p.db, hash, *p.hc.GetBlockNumber(hash))) } p.dl.DeliverReceipts(p.id, receipts) return nil diff --git a/eth/downloader/statesync.go b/eth/downloader/statesync.go index 37bc29f3c3..119cb12f68 100644 --- a/eth/downloader/statesync.go +++ b/eth/downloader/statesync.go @@ -23,7 +23,7 @@ import ( "time" "github.com/EthereumCommonwealth/go-callisto/common" - "github.com/EthereumCommonwealth/go-callisto/core" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/state" "github.com/EthereumCommonwealth/go-callisto/crypto/sha3" "github.com/EthereumCommonwealth/go-callisto/ethdb" @@ -476,6 +476,6 @@ func (s *stateSync) updateStats(written, duplicate, unexpected int, duration tim log.Info("Imported new state entries", "count", written, "elapsed", common.PrettyDuration(duration), "processed", s.d.syncStatsState.processed, "pending", s.d.syncStatsState.pending, "retry", len(s.tasks), "duplicate", s.d.syncStatsState.duplicate, "unexpected", s.d.syncStatsState.unexpected) } if written > 0 { - core.WriteTrieSyncProgress(s.d.stateDB, s.d.syncStatsState.processed) + rawdb.WriteFastTrieProgress(s.d.stateDB, s.d.syncStatsState.processed) } } diff --git a/eth/fetcher/fetcher_test.go b/eth/fetcher/fetcher_test.go index e43db4a3b3..796d52b3eb 100644 --- a/eth/fetcher/fetcher_test.go +++ b/eth/fetcher/fetcher_test.go @@ -34,7 +34,7 @@ import ( ) var ( - testdb, _ = ethdb.NewMemDatabase() + testdb = ethdb.NewMemDatabase() testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") testAddress = crypto.PubkeyToAddress(testKey.PublicKey) genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000)) diff --git a/eth/filters/api.go b/eth/filters/api.go index 697103ff3a..2afab26a43 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -104,8 +104,8 @@ func (api *PublicFilterAPI) timeoutLoop() { // https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_newpendingtransactionfilter func (api *PublicFilterAPI) NewPendingTransactionFilter() rpc.ID { var ( - pendingTxs = make(chan common.Hash) - pendingTxSub = api.events.SubscribePendingTxEvents(pendingTxs) + pendingTxs = make(chan []common.Hash) + pendingTxSub = api.events.SubscribePendingTxs(pendingTxs) ) api.filtersMu.Lock() @@ -118,7 +118,7 @@ func (api *PublicFilterAPI) NewPendingTransactionFilter() rpc.ID { case ph := <-pendingTxs: api.filtersMu.Lock() if f, found := api.filters[pendingTxSub.ID]; found { - f.hashes = append(f.hashes, ph) + f.hashes = append(f.hashes, ph...) } api.filtersMu.Unlock() case <-pendingTxSub.Err(): @@ -144,13 +144,17 @@ func (api *PublicFilterAPI) NewPendingTransactions(ctx context.Context) (*rpc.Su rpcSub := notifier.CreateSubscription() go func() { - txHashes := make(chan common.Hash) - pendingTxSub := api.events.SubscribePendingTxEvents(txHashes) + txHashes := make(chan []common.Hash, 128) + pendingTxSub := api.events.SubscribePendingTxs(txHashes) for { select { - case h := <-txHashes: - notifier.Notify(rpcSub.ID, h) + case hashes := <-txHashes: + // To keep the original behaviour, send a single tx hash in one notification. + // TODO(rjl493456442) Send a batch of tx hashes in one notification + for _, h := range hashes { + notifier.Notify(rpcSub.ID, h) + } case <-rpcSub.Err(): pendingTxSub.Unsubscribe() return @@ -268,14 +272,8 @@ func (api *PublicFilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc } // FilterCriteria represents a request to create a new filter. -// -// TODO(karalabe): Kill this in favor of ethereum.FilterQuery. -type FilterCriteria struct { - FromBlock *big.Int - ToBlock *big.Int - Addresses []common.Address - Topics [][]common.Hash -} +// Same as ethereum.FilterQuery but with UnmarshalJSON() method. +type FilterCriteria ethereum.FilterQuery // NewFilter creates a new filter and returns the filter id. It can be // used to retrieve logs when the state changes. This method cannot be diff --git a/eth/filters/bench_test.go b/eth/filters/bench_test.go index 9c6b21e04b..02b60cc97b 100644 --- a/eth/filters/bench_test.go +++ b/eth/filters/bench_test.go @@ -25,8 +25,8 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common" "github.com/EthereumCommonwealth/go-callisto/common/bitutil" - "github.com/EthereumCommonwealth/go-callisto/core" "github.com/EthereumCommonwealth/go-callisto/core/bloombits" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/ethdb" "github.com/EthereumCommonwealth/go-callisto/event" @@ -71,20 +71,20 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) { if err != nil { b.Fatalf("error opening database at %v: %v", benchDataDir, err) } - head := core.GetHeadBlockHash(db) + head := rawdb.ReadHeadBlockHash(db) if head == (common.Hash{}) { b.Fatalf("chain data not found at %v", benchDataDir) } clearBloomBits(db) fmt.Println("Generating bloombits data...") - headNum := core.GetBlockNumber(db, head) - if headNum < sectionSize+512 { + headNum := rawdb.ReadHeaderNumber(db, head) + if headNum == nil || *headNum < sectionSize+512 { b.Fatalf("not enough blocks for running a benchmark") } start := time.Now() - cnt := (headNum - 512) / sectionSize + cnt := (*headNum - 512) / sectionSize var dataSize, compSize uint64 for sectionIdx := uint64(0); sectionIdx < cnt; sectionIdx++ { bc, err := bloombits.NewGenerator(uint(sectionSize)) @@ -93,14 +93,14 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) { } var header *types.Header for i := sectionIdx * sectionSize; i < (sectionIdx+1)*sectionSize; i++ { - hash := core.GetCanonicalHash(db, i) - header = core.GetHeader(db, hash, i) + hash := rawdb.ReadCanonicalHash(db, i) + header = rawdb.ReadHeader(db, hash, i) if header == nil { b.Fatalf("Error creating bloomBits data") } bc.AddBloom(uint(i-sectionIdx*sectionSize), header.Bloom) } - sectionHead := core.GetCanonicalHash(db, (sectionIdx+1)*sectionSize-1) + sectionHead := rawdb.ReadCanonicalHash(db, (sectionIdx+1)*sectionSize-1) for i := 0; i < types.BloomBitLength; i++ { data, err := bc.Bitset(uint(i)) if err != nil { @@ -109,7 +109,7 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) { comp := bitutil.CompressBytes(data) dataSize += uint64(len(data)) compSize += uint64(len(comp)) - core.WriteBloomBits(db, uint(i), sectionIdx, sectionHead, comp) + rawdb.WriteBloomBits(db, uint(i), sectionIdx, sectionHead, comp) } //if sectionIdx%50 == 0 { // fmt.Println(" section", sectionIdx, "/", cnt) @@ -180,11 +180,11 @@ func BenchmarkNoBloomBits(b *testing.B) { if err != nil { b.Fatalf("error opening database at %v: %v", benchDataDir, err) } - head := core.GetHeadBlockHash(db) + head := rawdb.ReadHeadBlockHash(db) if head == (common.Hash{}) { b.Fatalf("chain data not found at %v", benchDataDir) } - headNum := core.GetBlockNumber(db, head) + headNum := rawdb.ReadHeaderNumber(db, head) clearBloomBits(db) @@ -192,10 +192,10 @@ func BenchmarkNoBloomBits(b *testing.B) { start := time.Now() mux := new(event.TypeMux) backend := &testBackend{mux, db, 0, new(event.Feed), new(event.Feed), new(event.Feed), new(event.Feed)} - filter := New(backend, 0, int64(headNum), []common.Address{{}}, nil) + filter := New(backend, 0, int64(*headNum), []common.Address{{}}, nil) filter.Logs(context.Background()) d := time.Since(start) fmt.Println("Finished running filter benchmarks") - fmt.Println(" ", d, "total ", d*time.Duration(1000000)/time.Duration(headNum+1), "per million blocks") + fmt.Println(" ", d, "total ", d*time.Duration(1000000)/time.Duration(*headNum+1), "per million blocks") db.Close() } diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 26a8f8b90f..6f3689b119 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -36,7 +36,7 @@ type Backend interface { GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) - SubscribeTxPreEvent(chan<- core.TxPreEvent) event.Subscription + SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index be58cfad0a..00f61c3b02 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -28,8 +28,10 @@ import ( ethereum "github.com/EthereumCommonwealth/go-callisto" "github.com/EthereumCommonwealth/go-callisto/common" "github.com/EthereumCommonwealth/go-callisto/core" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/event" + "github.com/EthereumCommonwealth/go-callisto/log" "github.com/EthereumCommonwealth/go-callisto/rpc" ) @@ -57,7 +59,7 @@ const ( const ( - // txChanSize is the size of channel listening to TxPreEvent. + // txChanSize is the size of channel listening to NewTxsEvent. // The number is referenced from the size of tx pool. txChanSize = 4096 // rmLogsChanSize is the size of channel listening to RemovedLogsEvent. @@ -78,7 +80,7 @@ type subscription struct { created time.Time logsCrit ethereum.FilterQuery logs chan []*types.Log - hashes chan common.Hash + hashes chan []common.Hash headers chan *types.Header installed chan struct{} // closed when the filter is installed err chan error // closed when the filter is uninstalled @@ -91,8 +93,21 @@ type EventSystem struct { backend Backend lightMode bool lastHead *types.Header - install chan *subscription // install filter for event notification - uninstall chan *subscription // remove filter for event notification + + // Subscriptions + txsSub event.Subscription // Subscription for new transaction event + logsSub event.Subscription // Subscription for new log event + rmLogsSub event.Subscription // Subscription for removed log event + chainSub event.Subscription // Subscription for new chain event + pendingLogSub *event.TypeMuxSubscription // Subscription for pending log event + + // Channels + install chan *subscription // install filter for event notification + uninstall chan *subscription // remove filter for event notification + txsCh chan core.NewTxsEvent // Channel to receive new transactions event + logsCh chan []*types.Log // Channel to receive new log event + rmLogsCh chan core.RemovedLogsEvent // Channel to receive removed log event + chainCh chan core.ChainEvent // Channel to receive new chain event } // NewEventSystem creates a new manager that listens for event on the given mux, @@ -108,10 +123,27 @@ func NewEventSystem(mux *event.TypeMux, backend Backend, lightMode bool) *EventS lightMode: lightMode, install: make(chan *subscription), uninstall: make(chan *subscription), + txsCh: make(chan core.NewTxsEvent, txChanSize), + logsCh: make(chan []*types.Log, logsChanSize), + rmLogsCh: make(chan core.RemovedLogsEvent, rmLogsChanSize), + chainCh: make(chan core.ChainEvent, chainEvChanSize), + } + + // Subscribe events + m.txsSub = m.backend.SubscribeNewTxsEvent(m.txsCh) + m.logsSub = m.backend.SubscribeLogsEvent(m.logsCh) + m.rmLogsSub = m.backend.SubscribeRemovedLogsEvent(m.rmLogsCh) + m.chainSub = m.backend.SubscribeChainEvent(m.chainCh) + // TODO(rjl493456442): use feed to subscribe pending log event + m.pendingLogSub = m.mux.Subscribe(core.PendingLogsEvent{}) + + // Make sure none of the subscriptions are empty + if m.txsSub == nil || m.logsSub == nil || m.rmLogsSub == nil || m.chainSub == nil || + m.pendingLogSub.Closed() { + log.Crit("Subscribe for event system failed") } go m.eventLoop() - return m } @@ -208,7 +240,7 @@ func (es *EventSystem) subscribeMinedPendingLogs(crit ethereum.FilterQuery, logs logsCrit: crit, created: time.Now(), logs: logs, - hashes: make(chan common.Hash), + hashes: make(chan []common.Hash), headers: make(chan *types.Header), installed: make(chan struct{}), err: make(chan error), @@ -225,7 +257,7 @@ func (es *EventSystem) subscribeLogs(crit ethereum.FilterQuery, logs chan []*typ logsCrit: crit, created: time.Now(), logs: logs, - hashes: make(chan common.Hash), + hashes: make(chan []common.Hash), headers: make(chan *types.Header), installed: make(chan struct{}), err: make(chan error), @@ -242,7 +274,7 @@ func (es *EventSystem) subscribePendingLogs(crit ethereum.FilterQuery, logs chan logsCrit: crit, created: time.Now(), logs: logs, - hashes: make(chan common.Hash), + hashes: make(chan []common.Hash), headers: make(chan *types.Header), installed: make(chan struct{}), err: make(chan error), @@ -258,7 +290,7 @@ func (es *EventSystem) SubscribeNewHeads(headers chan *types.Header) *Subscripti typ: BlocksSubscription, created: time.Now(), logs: make(chan []*types.Log), - hashes: make(chan common.Hash), + hashes: make(chan []common.Hash), headers: headers, installed: make(chan struct{}), err: make(chan error), @@ -266,9 +298,9 @@ func (es *EventSystem) SubscribeNewHeads(headers chan *types.Header) *Subscripti return es.subscribe(sub) } -// SubscribePendingTxEvents creates a subscription that writes transaction hashes for +// SubscribePendingTxs creates a subscription that writes transaction hashes for // transactions that enter the transaction pool. -func (es *EventSystem) SubscribePendingTxEvents(hashes chan common.Hash) *Subscription { +func (es *EventSystem) SubscribePendingTxs(hashes chan []common.Hash) *Subscription { sub := &subscription{ id: rpc.NewID(), typ: PendingTransactionsSubscription, @@ -316,9 +348,13 @@ func (es *EventSystem) broadcast(filters filterIndex, ev interface{}) { } } } - case core.TxPreEvent: + case core.NewTxsEvent: + hashes := make([]common.Hash, 0, len(e.Txs)) + for _, tx := range e.Txs { + hashes = append(hashes, tx.Hash()) + } for _, f := range filters[PendingTransactionsSubscription] { - f.hashes <- e.Tx.Hash() + f.hashes <- hashes } case core.ChainEvent: for _, f := range filters[BlocksSubscription] { @@ -348,11 +384,11 @@ func (es *EventSystem) lightFilterNewHead(newHeader *types.Header, callBack func for oldh.Hash() != newh.Hash() { if oldh.Number.Uint64() >= newh.Number.Uint64() { oldHeaders = append(oldHeaders, oldh) - oldh = core.GetHeader(es.backend.ChainDb(), oldh.ParentHash, oldh.Number.Uint64()-1) + oldh = rawdb.ReadHeader(es.backend.ChainDb(), oldh.ParentHash, oldh.Number.Uint64()-1) } if oldh.Number.Uint64() < newh.Number.Uint64() { newHeaders = append(newHeaders, newh) - newh = core.GetHeader(es.backend.ChainDb(), newh.ParentHash, newh.Number.Uint64()-1) + newh = rawdb.ReadHeader(es.backend.ChainDb(), newh.ParentHash, newh.Number.Uint64()-1) if newh == nil { // happens when CHT syncing, nothing to do newh = oldh @@ -411,52 +447,37 @@ func (es *EventSystem) lightFilterLogs(header *types.Header, addresses []common. // eventLoop (un)installs filters and processes mux events. func (es *EventSystem) eventLoop() { - var ( - index = make(filterIndex) - sub = es.mux.Subscribe(core.PendingLogsEvent{}) - // Subscribe TxPreEvent form txpool - txCh = make(chan core.TxPreEvent, txChanSize) - txSub = es.backend.SubscribeTxPreEvent(txCh) - // Subscribe RemovedLogsEvent - rmLogsCh = make(chan core.RemovedLogsEvent, rmLogsChanSize) - rmLogsSub = es.backend.SubscribeRemovedLogsEvent(rmLogsCh) - // Subscribe []*types.Log - logsCh = make(chan []*types.Log, logsChanSize) - logsSub = es.backend.SubscribeLogsEvent(logsCh) - // Subscribe ChainEvent - chainEvCh = make(chan core.ChainEvent, chainEvChanSize) - chainEvSub = es.backend.SubscribeChainEvent(chainEvCh) - ) - - // Unsubscribe all events - defer sub.Unsubscribe() - defer txSub.Unsubscribe() - defer rmLogsSub.Unsubscribe() - defer logsSub.Unsubscribe() - defer chainEvSub.Unsubscribe() + // Ensure all subscriptions get cleaned up + defer func() { + es.pendingLogSub.Unsubscribe() + es.txsSub.Unsubscribe() + es.logsSub.Unsubscribe() + es.rmLogsSub.Unsubscribe() + es.chainSub.Unsubscribe() + }() + index := make(filterIndex) for i := UnknownSubscription; i < LastIndexSubscription; i++ { index[i] = make(map[rpc.ID]*subscription) } for { select { - case ev, active := <-sub.Chan(): + // Handle subscribed events + case ev := <-es.txsCh: + es.broadcast(index, ev) + case ev := <-es.logsCh: + es.broadcast(index, ev) + case ev := <-es.rmLogsCh: + es.broadcast(index, ev) + case ev := <-es.chainCh: + es.broadcast(index, ev) + case ev, active := <-es.pendingLogSub.Chan(): if !active { // system stopped return } es.broadcast(index, ev) - // Handle subscribed events - case ev := <-txCh: - es.broadcast(index, ev) - case ev := <-rmLogsCh: - es.broadcast(index, ev) - case ev := <-logsCh: - es.broadcast(index, ev) - case ev := <-chainEvCh: - es.broadcast(index, ev) - case f := <-es.install: if f.typ == MinedAndPendingLogsSubscription { // the type are logs and pending logs subscriptions @@ -466,6 +487,7 @@ func (es *EventSystem) eventLoop() { index[f.typ][f.id] = f } close(f.installed) + case f := <-es.uninstall: if f.typ == MinedAndPendingLogsSubscription { // the type are logs and pending logs subscriptions @@ -477,13 +499,13 @@ func (es *EventSystem) eventLoop() { close(f.err) // System stopped - case <-txSub.Err(): + case <-es.txsSub.Err(): return - case <-rmLogsSub.Err(): + case <-es.logsSub.Err(): return - case <-logsSub.Err(): + case <-es.rmLogsSub.Err(): return - case <-chainEvSub.Err(): + case <-es.chainSub.Err(): return } } diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 0c23f47c93..ec68c7ab32 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -30,6 +30,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/consensus/ethash" "github.com/EthereumCommonwealth/go-callisto/core" "github.com/EthereumCommonwealth/go-callisto/core/bloombits" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/ethdb" "github.com/EthereumCommonwealth/go-callisto/event" @@ -56,26 +57,37 @@ func (b *testBackend) EventMux() *event.TypeMux { } func (b *testBackend) HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) { - var hash common.Hash - var num uint64 + var ( + hash common.Hash + num uint64 + ) if blockNr == rpc.LatestBlockNumber { - hash = core.GetHeadBlockHash(b.db) - num = core.GetBlockNumber(b.db, hash) + hash = rawdb.ReadHeadBlockHash(b.db) + number := rawdb.ReadHeaderNumber(b.db, hash) + if number == nil { + return nil, nil + } + num = *number } else { num = uint64(blockNr) - hash = core.GetCanonicalHash(b.db, num) + hash = rawdb.ReadCanonicalHash(b.db, num) } - return core.GetHeader(b.db, hash, num), nil + return rawdb.ReadHeader(b.db, hash, num), nil } -func (b *testBackend) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) { - number := core.GetBlockNumber(b.db, blockHash) - return core.GetBlockReceipts(b.db, blockHash, number), nil +func (b *testBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { + if number := rawdb.ReadHeaderNumber(b.db, hash); number != nil { + return rawdb.ReadReceipts(b.db, hash, *number), nil + } + return nil, nil } -func (b *testBackend) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) { - number := core.GetBlockNumber(b.db, blockHash) - receipts := core.GetBlockReceipts(b.db, blockHash, number) +func (b *testBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { + number := rawdb.ReadHeaderNumber(b.db, hash) + if number == nil { + return nil, nil + } + receipts := rawdb.ReadReceipts(b.db, hash, *number) logs := make([][]*types.Log, len(receipts)) for i, receipt := range receipts { @@ -84,7 +96,7 @@ func (b *testBackend) GetLogs(ctx context.Context, blockHash common.Hash) ([][]* return logs, nil } -func (b *testBackend) SubscribeTxPreEvent(ch chan<- core.TxPreEvent) event.Subscription { +func (b *testBackend) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subscription { return b.txFeed.Subscribe(ch) } @@ -121,8 +133,8 @@ func (b *testBackend) ServiceFilter(ctx context.Context, session *bloombits.Matc task.Bitsets = make([][]byte, len(task.Sections)) for i, section := range task.Sections { if rand.Int()%4 != 0 { // Handle occasional missing deliveries - head := core.GetCanonicalHash(b.db, (section+1)*params.BloomBitsBlocks-1) - task.Bitsets[i], _ = core.GetBloomBits(b.db, task.Bit, section, head) + head := rawdb.ReadCanonicalHash(b.db, (section+1)*params.BloomBitsBlocks-1) + task.Bitsets[i], _ = rawdb.ReadBloomBits(b.db, task.Bit, section, head) } } request <- task @@ -141,7 +153,7 @@ func TestBlockSubscription(t *testing.T) { var ( mux = new(event.TypeMux) - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() txFeed = new(event.Feed) rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) @@ -198,7 +210,7 @@ func TestPendingTxFilter(t *testing.T) { var ( mux = new(event.TypeMux) - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() txFeed = new(event.Feed) rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) @@ -220,10 +232,7 @@ func TestPendingTxFilter(t *testing.T) { fid0 := api.NewPendingTransactionFilter() time.Sleep(1 * time.Second) - for _, tx := range transactions { - ev := core.TxPreEvent{Tx: tx} - txFeed.Send(ev) - } + txFeed.Send(core.NewTxsEvent{Txs: transactions}) timeout := time.Now().Add(1 * time.Second) for { @@ -261,7 +270,7 @@ func TestPendingTxFilter(t *testing.T) { func TestLogFilterCreation(t *testing.T) { var ( mux = new(event.TypeMux) - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() txFeed = new(event.Feed) rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) @@ -310,7 +319,7 @@ func TestInvalidLogFilterCreation(t *testing.T) { var ( mux = new(event.TypeMux) - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() txFeed = new(event.Feed) rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) @@ -340,7 +349,7 @@ func TestLogFilter(t *testing.T) { var ( mux = new(event.TypeMux) - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() txFeed = new(event.Feed) rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) @@ -459,7 +468,7 @@ func TestPendingLogsSubscription(t *testing.T) { var ( mux = new(event.TypeMux) - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() txFeed = new(event.Feed) rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index 80c64a33c9..694609b74d 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -26,6 +26,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common" "github.com/EthereumCommonwealth/go-callisto/consensus/ethash" "github.com/EthereumCommonwealth/go-callisto/core" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/crypto" "github.com/EthereumCommonwealth/go-callisto/ethdb" @@ -84,16 +85,10 @@ func BenchmarkFilters(b *testing.B) { } }) for i, block := range chain { - core.WriteBlock(db, block) - if err := core.WriteCanonicalHash(db, block.Hash(), block.NumberU64()); err != nil { - b.Fatalf("failed to insert block number: %v", err) - } - if err := core.WriteHeadBlockHash(db, block.Hash()); err != nil { - b.Fatalf("failed to insert block number: %v", err) - } - if err := core.WriteBlockReceipts(db, block.Hash(), block.NumberU64(), receipts[i]); err != nil { - b.Fatal("error writing block receipts:", err) - } + rawdb.WriteBlock(db, block) + rawdb.WriteCanonicalHash(db, block.Hash(), block.NumberU64()) + rawdb.WriteHeadBlockHash(db, block.Hash()) + rawdb.WriteReceipts(db, block.Hash(), block.NumberU64(), receipts[i]) } b.ResetTimer() @@ -174,16 +169,10 @@ func TestFilters(t *testing.T) { } }) for i, block := range chain { - core.WriteBlock(db, block) - if err := core.WriteCanonicalHash(db, block.Hash(), block.NumberU64()); err != nil { - t.Fatalf("failed to insert block number: %v", err) - } - if err := core.WriteHeadBlockHash(db, block.Hash()); err != nil { - t.Fatalf("failed to insert block number: %v", err) - } - if err := core.WriteBlockReceipts(db, block.Hash(), block.NumberU64(), receipts[i]); err != nil { - t.Fatal("error writing block receipts:", err) - } + rawdb.WriteBlock(db, block) + rawdb.WriteCanonicalHash(db, block.Hash(), block.NumberU64()) + rawdb.WriteHeadBlockHash(db, block.Hash()) + rawdb.WriteReceipts(db, block.Hash(), block.NumberU64(), receipts[i]) } filter := New(backend, 0, -1, []common.Address{addr}, [][]common.Hash{{hash1, hash2, hash3, hash4}}) diff --git a/eth/handler.go b/eth/handler.go index 83e67c39cb..8648cbcd43 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -46,7 +46,7 @@ const ( softResponseLimit = 2 * 1024 * 1024 // Target maximum size of returned blocks, headers or node data. estHeaderRlpSize = 500 // Approximate size of an RLP encoded block header - // txChanSize is the size of channel listening to TxPreEvent. + // txChanSize is the size of channel listening to NewTxsEvent. // The number is referenced from the size of tx pool. txChanSize = 4096 ) @@ -81,8 +81,8 @@ type ProtocolManager struct { SubProtocols []p2p.Protocol eventMux *event.TypeMux - txCh chan core.TxPreEvent - txSub event.Subscription + txsCh chan core.NewTxsEvent + txsSub event.Subscription minedBlockSub *event.TypeMuxSubscription // channels for fetcher, syncer, txsyncLoop @@ -204,8 +204,8 @@ func (pm *ProtocolManager) Start(maxPeers int) { pm.maxPeers = maxPeers // broadcast transactions - pm.txCh = make(chan core.TxPreEvent, txChanSize) - pm.txSub = pm.txpool.SubscribeTxPreEvent(pm.txCh) + pm.txsCh = make(chan core.NewTxsEvent, txChanSize) + pm.txsSub = pm.txpool.SubscribeNewTxsEvent(pm.txsCh) go pm.txBroadcastLoop() // broadcast mined blocks @@ -220,7 +220,7 @@ func (pm *ProtocolManager) Start(maxPeers int) { func (pm *ProtocolManager) Stop() { log.Info("Stopping Ethereum protocol") - pm.txSub.Unsubscribe() // quits txBroadcastLoop + pm.txsSub.Unsubscribe() // quits txBroadcastLoop pm.minedBlockSub.Unsubscribe() // quits blockBroadcastLoop // Quit the sync loop. @@ -698,7 +698,7 @@ func (pm *ProtocolManager) BroadcastBlock(block *types.Block, propagate bool) { // Send the block to a subset of our peers transfer := peers[:int(math.Sqrt(float64(len(peers))))] for _, peer := range transfer { - peer.SendNewBlock(block, td) + peer.AsyncSendNewBlock(block, td) } log.Trace("Propagated block", "hash", hash, "recipients", len(transfer), "duration", common.PrettyDuration(time.Since(block.ReceivedAt))) return @@ -706,44 +706,51 @@ func (pm *ProtocolManager) BroadcastBlock(block *types.Block, propagate bool) { // Otherwise if the block is indeed in out own chain, announce it if pm.blockchain.HasBlock(hash, block.NumberU64()) { for _, peer := range peers { - peer.SendNewBlockHashes([]common.Hash{hash}, []uint64{block.NumberU64()}) + peer.AsyncSendNewBlockHash(block) } log.Trace("Announced block", "hash", hash, "recipients", len(peers), "duration", common.PrettyDuration(time.Since(block.ReceivedAt))) } } -// BroadcastTx will propagate a transaction to all peers which are not known to +// BroadcastTxs will propagate a batch of transactions to all peers which are not known to // already have the given transaction. -func (pm *ProtocolManager) BroadcastTx(hash common.Hash, tx *types.Transaction) { - // Broadcast transaction to a batch of peers not knowing about it - peers := pm.peers.PeersWithoutTx(hash) - //FIXME include this again: peers = peers[:int(math.Sqrt(float64(len(peers))))] - for _, peer := range peers { - peer.SendTransactions(types.Transactions{tx}) +func (pm *ProtocolManager) BroadcastTxs(txs types.Transactions) { + var txset = make(map[*peer]types.Transactions) + + // Broadcast transactions to a batch of peers not knowing about it + for _, tx := range txs { + peers := pm.peers.PeersWithoutTx(tx.Hash()) + for _, peer := range peers { + txset[peer] = append(txset[peer], tx) + } + log.Trace("Broadcast transaction", "hash", tx.Hash(), "recipients", len(peers)) + } + // FIXME include this again: peers = peers[:int(math.Sqrt(float64(len(peers))))] + for peer, txs := range txset { + peer.AsyncSendTransactions(txs) } - log.Trace("Broadcast transaction", "hash", hash, "recipients", len(peers)) } // Mined broadcast loop -func (self *ProtocolManager) minedBroadcastLoop() { +func (pm *ProtocolManager) minedBroadcastLoop() { // automatically stops if unsubscribe - for obj := range self.minedBlockSub.Chan() { + for obj := range pm.minedBlockSub.Chan() { switch ev := obj.Data.(type) { case core.NewMinedBlockEvent: - self.BroadcastBlock(ev.Block, true) // First propagate block to peers - self.BroadcastBlock(ev.Block, false) // Only then announce to the rest + pm.BroadcastBlock(ev.Block, true) // First propagate block to peers + pm.BroadcastBlock(ev.Block, false) // Only then announce to the rest } } } -func (self *ProtocolManager) txBroadcastLoop() { +func (pm *ProtocolManager) txBroadcastLoop() { for { select { - case event := <-self.txCh: - self.BroadcastTx(event.Tx.Hash(), event.Tx) + case event := <-pm.txsCh: + pm.BroadcastTxs(event.Txs) // Err() channel will be closed when unsubscribing. - case <-self.txSub.Err(): + case <-pm.txsSub.Err(): return } } @@ -760,13 +767,13 @@ type NodeInfo struct { } // NodeInfo retrieves some protocol metadata about the running host node. -func (self *ProtocolManager) NodeInfo() *NodeInfo { - currentBlock := self.blockchain.CurrentBlock() +func (pm *ProtocolManager) NodeInfo() *NodeInfo { + currentBlock := pm.blockchain.CurrentBlock() return &NodeInfo{ - Network: self.networkId, - Difficulty: self.blockchain.GetTd(currentBlock.Hash(), currentBlock.NumberU64()), - Genesis: self.blockchain.Genesis().Hash(), - Config: self.blockchain.Config(), + Network: pm.networkId, + Difficulty: pm.blockchain.GetTd(currentBlock.Hash(), currentBlock.NumberU64()), + Genesis: pm.blockchain.Genesis().Hash(), + Config: pm.blockchain.Config(), Head: currentBlock.Hash(), } } diff --git a/eth/handler_test.go b/eth/handler_test.go index dcae93b73d..4c66351362 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -366,7 +366,7 @@ func testGetNodeData(t *testing.T, protocol int) { t.Errorf("data hash mismatch: have %x, want %x", hash, want) } } - statedb, _ := ethdb.NewMemDatabase() + statedb := ethdb.NewMemDatabase() for i := 0; i < len(data); i++ { statedb.Put(hashes[i].Bytes(), data[i]) } @@ -468,7 +468,7 @@ func testDAOChallenge(t *testing.T, localForked, remoteForked bool, timeout bool var ( evmux = new(event.TypeMux) pow = ethash.NewFaker() - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() config = ¶ms.ChainConfig{DAOForkBlock: big.NewInt(1), DAOForkSupport: localForked} gspec = &core.Genesis{Config: config} genesis = gspec.MustCommit(db) diff --git a/eth/helper_test.go b/eth/helper_test.go index 2c93e77755..c604fb31b1 100644 --- a/eth/helper_test.go +++ b/eth/helper_test.go @@ -53,7 +53,7 @@ func newTestProtocolManager(mode downloader.SyncMode, blocks int, generator func var ( evmux = new(event.TypeMux) engine = ethash.NewFaker() - db, _ = ethdb.NewMemDatabase() + db = ethdb.NewMemDatabase() gspec = &core.Genesis{ Config: params.TestChainConfig, Alloc: core.GenesisAlloc{testBank: {Balance: big.NewInt(1000000)}}, @@ -124,7 +124,7 @@ func (p *testTxPool) Pending() (map[common.Address]types.Transactions, error) { return batches, nil } -func (p *testTxPool) SubscribeTxPreEvent(ch chan<- core.TxPreEvent) event.Subscription { +func (p *testTxPool) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subscription { return p.txFeed.Subscribe(ch) } diff --git a/eth/peer.go b/eth/peer.go index ed8c4f6d5e..b9cc4a7f28 100644 --- a/eth/peer.go +++ b/eth/peer.go @@ -37,8 +37,24 @@ var ( ) const ( - maxKnownTxs = 32768 // Maximum transactions hashes to keep in the known list (prevent DOS) - maxKnownBlocks = 1024 // Maximum block hashes to keep in the known list (prevent DOS) + maxKnownTxs = 32768 // Maximum transactions hashes to keep in the known list (prevent DOS) + maxKnownBlocks = 1024 // Maximum block hashes to keep in the known list (prevent DOS) + + // maxQueuedTxs is the maximum number of transaction lists to queue up before + // dropping broadcasts. This is a sensitive number as a transaction list might + // contain a single transaction, or thousands. + maxQueuedTxs = 128 + + // maxQueuedProps is the maximum number of block propagations to queue up before + // dropping broadcasts. There's not much point in queueing stale blocks, so a few + // that might cover uncles should be enough. + maxQueuedProps = 4 + + // maxQueuedAnns is the maximum number of block announcements to queue up before + // dropping broadcasts. Similarly to block propagations, there's no point to queue + // above some healthy uncle limit, so use that. + maxQueuedAnns = 4 + handshakeTimeout = 5 * time.Second ) @@ -50,6 +66,12 @@ type PeerInfo struct { Head string `json:"head"` // SHA3 hash of the peer's best owned block } +// propEvent is a block propagation, waiting for its turn in the broadcast queue. +type propEvent struct { + block *types.Block + td *big.Int +} + type peer struct { id string @@ -63,23 +85,64 @@ type peer struct { td *big.Int lock sync.RWMutex - knownTxs *set.Set // Set of transaction hashes known to be known by this peer - knownBlocks *set.Set // Set of block hashes known to be known by this peer + knownTxs *set.Set // Set of transaction hashes known to be known by this peer + knownBlocks *set.Set // Set of block hashes known to be known by this peer + queuedTxs chan []*types.Transaction // Queue of transactions to broadcast to the peer + queuedProps chan *propEvent // Queue of blocks to broadcast to the peer + queuedAnns chan *types.Block // Queue of blocks to announce to the peer + term chan struct{} // Termination channel to stop the broadcaster } func newPeer(version int, p *p2p.Peer, rw p2p.MsgReadWriter) *peer { - id := p.ID() - return &peer{ Peer: p, rw: rw, version: version, - id: fmt.Sprintf("%x", id[:8]), + id: fmt.Sprintf("%x", p.ID().Bytes()[:8]), knownTxs: set.New(), knownBlocks: set.New(), + queuedTxs: make(chan []*types.Transaction, maxQueuedTxs), + queuedProps: make(chan *propEvent, maxQueuedProps), + queuedAnns: make(chan *types.Block, maxQueuedAnns), + term: make(chan struct{}), } } +// broadcast is a write loop that multiplexes block propagations, announcements +// and transaction broadcasts into the remote peer. The goal is to have an async +// writer that does not lock up node internals. +func (p *peer) broadcast() { + for { + select { + case txs := <-p.queuedTxs: + if err := p.SendTransactions(txs); err != nil { + return + } + p.Log().Trace("Broadcast transactions", "count", len(txs)) + + case prop := <-p.queuedProps: + if err := p.SendNewBlock(prop.block, prop.td); err != nil { + return + } + p.Log().Trace("Propagated block", "number", prop.block.Number(), "hash", prop.block.Hash(), "td", prop.td) + + case block := <-p.queuedAnns: + if err := p.SendNewBlockHashes([]common.Hash{block.Hash()}, []uint64{block.NumberU64()}); err != nil { + return + } + p.Log().Trace("Announced block", "number", block.Number(), "hash", block.Hash()) + + case <-p.term: + return + } + } +} + +// close signals the broadcast goroutine to terminate. +func (p *peer) close() { + close(p.term) +} + // Info gathers and returns a collection of metadata known about a peer. func (p *peer) Info() *PeerInfo { hash, td := p.Head() @@ -139,6 +202,19 @@ func (p *peer) SendTransactions(txs types.Transactions) error { return p2p.Send(p.rw, TxMsg, txs) } +// AsyncSendTransactions queues list of transactions propagation to a remote +// peer. If the peer's broadcast queue is full, the event is silently dropped. +func (p *peer) AsyncSendTransactions(txs []*types.Transaction) { + select { + case p.queuedTxs <- txs: + for _, tx := range txs { + p.knownTxs.Add(tx.Hash()) + } + default: + p.Log().Debug("Dropping transaction propagation", "count", len(txs)) + } +} + // SendNewBlockHashes announces the availability of a number of blocks through // a hash notification. func (p *peer) SendNewBlockHashes(hashes []common.Hash, numbers []uint64) error { @@ -153,12 +229,35 @@ func (p *peer) SendNewBlockHashes(hashes []common.Hash, numbers []uint64) error return p2p.Send(p.rw, NewBlockHashesMsg, request) } +// AsyncSendNewBlockHash queues the availability of a block for propagation to a +// remote peer. If the peer's broadcast queue is full, the event is silently +// dropped. +func (p *peer) AsyncSendNewBlockHash(block *types.Block) { + select { + case p.queuedAnns <- block: + p.knownBlocks.Add(block.Hash()) + default: + p.Log().Debug("Dropping block announcement", "number", block.NumberU64(), "hash", block.Hash()) + } +} + // SendNewBlock propagates an entire block to a remote peer. func (p *peer) SendNewBlock(block *types.Block, td *big.Int) error { p.knownBlocks.Add(block.Hash()) return p2p.Send(p.rw, NewBlockMsg, []interface{}{block, td}) } +// AsyncSendNewBlock queues an entire block for propagation to a remote peer. If +// the peer's broadcast queue is full, the event is silently dropped. +func (p *peer) AsyncSendNewBlock(block *types.Block, td *big.Int) { + select { + case p.queuedProps <- &propEvent{block: block, td: td}: + p.knownBlocks.Add(block.Hash()) + default: + p.Log().Debug("Dropping block propagation", "number", block.NumberU64(), "hash", block.Hash()) + } +} + // SendBlockHeaders sends a batch of block headers to the remote peer. func (p *peer) SendBlockHeaders(headers []*types.Header) error { return p2p.Send(p.rw, BlockHeadersMsg, headers) @@ -313,7 +412,8 @@ func newPeerSet() *peerSet { } // Register injects a new peer into the working set, or returns an error if the -// peer is already known. +// peer is already known. If a new peer it registered, its broadcast loop is also +// started. func (ps *peerSet) Register(p *peer) error { ps.lock.Lock() defer ps.lock.Unlock() @@ -325,6 +425,8 @@ func (ps *peerSet) Register(p *peer) error { return errAlreadyRegistered } ps.peers[p.id] = p + go p.broadcast() + return nil } @@ -334,10 +436,13 @@ func (ps *peerSet) Unregister(id string) error { ps.lock.Lock() defer ps.lock.Unlock() - if _, ok := ps.peers[id]; !ok { + p, ok := ps.peers[id] + if !ok { return errNotRegistered } delete(ps.peers, id) + p.close() + return nil } diff --git a/eth/protocol.go b/eth/protocol.go index 213f8521e0..d3893c5353 100644 --- a/eth/protocol.go +++ b/eth/protocol.go @@ -34,13 +34,13 @@ const ( eth63 = 63 ) -// Official short name of the protocol used during capability negotiation. +// ProtocolName is the official short name of the protocol used during capability negotiation. var ProtocolName = "eth" -// Supported versions of the eth protocol (first is primary). +// ProtocolVersions are the upported versions of the eth protocol (first is primary). var ProtocolVersions = []uint{eth63, eth62} -// Number of implemented message corresponding to different protocol versions. +// ProtocolLengths are the number of implemented message corresponding to different protocol versions. var ProtocolLengths = []uint64{17, 8} const ProtocolMaxMsgSize = 10 * 1024 * 1024 // Maximum cap on the size of a protocol message @@ -103,9 +103,9 @@ type txPool interface { // The slice should be modifiable by the caller. Pending() (map[common.Address]types.Transactions, error) - // SubscribeTxPreEvent should return an event subscription of - // TxPreEvent and send events to the given channel. - SubscribeTxPreEvent(chan<- core.TxPreEvent) event.Subscription + // SubscribeNewTxsEvent should return an event subscription of + // NewTxsEvent and send events to the given channel. + SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription } // statusData is the network packet for the status message. diff --git a/eth/protocol_test.go b/eth/protocol_test.go index abfd562a23..f376e03fe6 100644 --- a/eth/protocol_test.go +++ b/eth/protocol_test.go @@ -116,7 +116,7 @@ func testRecvTransactions(t *testing.T, protocol int) { t.Errorf("added wrong tx hash: got %v, want %v", added[0].Hash(), tx.Hash()) } case <-time.After(2 * time.Second): - t.Errorf("no TxPreEvent received within 2 seconds") + t.Errorf("no NewTxsEvent received within 2 seconds") } } diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index 83887c717c..bca94ff33f 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -159,8 +159,7 @@ func TestCallTracer(t *testing.T) { GasLimit: uint64(test.Context.GasLimit), GasPrice: tx.GasPrice(), } - db, _ := ethdb.NewMemDatabase() - statedb := tests.MakePreState(db, test.Genesis.Alloc) + statedb := tests.MakePreState(ethdb.NewMemDatabase(), test.Genesis.Alloc) // Create the tracer, the EVM environment and run it tracer, err := New("callTracer") diff --git a/ethdb/database.go b/ethdb/database.go index 8d85688ec8..1f2267a3bd 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -207,6 +207,7 @@ func (db *LDBDatabase) meter(refresh time.Duration) { delaystats [2]int64 lastWriteDelay time.Time lastWriteDelayN time.Time + lastWritePaused time.Time ) // Iterate ad infinitum and collect the stats @@ -267,8 +268,9 @@ func (db *LDBDatabase) meter(refresh time.Duration) { delayN int64 delayDuration string duration time.Duration + paused bool ) - if n, err := fmt.Sscanf(writedelay, "DelayN:%d Delay:%s", &delayN, &delayDuration); n != 2 || err != nil { + if n, err := fmt.Sscanf(writedelay, "DelayN:%d Delay:%s Paused:%t", &delayN, &delayDuration, &paused); n != 3 || err != nil { db.log.Error("Write delay statistic not found") return } @@ -301,6 +303,14 @@ func (db *LDBDatabase) meter(refresh time.Duration) { lastWriteDelay = time.Now() } } + // If a warning that db is performing compaction has been displayed, any subsequent + // warnings will be withheld for one minute not to overwhelm the user. + if paused && delayN-delaystats[0] == 0 && duration.Nanoseconds()-delaystats[1] == 0 && + time.Now().After(lastWritePaused.Add(writeDelayWarningThrottler)) { + db.log.Warn("Database compacting, degraded performance") + lastWritePaused = time.Now() + } + delaystats[0], delaystats[1] = delayN, duration.Nanoseconds() // Retrieve the database iostats. diff --git a/ethdb/database_test.go b/ethdb/database_test.go index bcf4f1e021..8f12e4c8c8 100644 --- a/ethdb/database_test.go +++ b/ethdb/database_test.go @@ -53,8 +53,7 @@ func TestLDB_PutGet(t *testing.T) { } func TestMemoryDB_PutGet(t *testing.T) { - db, _ := ethdb.NewMemDatabase() - testPutGet(db, t) + testPutGet(ethdb.NewMemDatabase(), t) } func testPutGet(db ethdb.Database, t *testing.T) { @@ -131,8 +130,7 @@ func TestLDB_ParallelPutGet(t *testing.T) { } func TestMemoryDB_ParallelPutGet(t *testing.T) { - db, _ := ethdb.NewMemDatabase() - testParallelPutGet(db, t) + testParallelPutGet(ethdb.NewMemDatabase(), t) } func testParallelPutGet(db ethdb.Database, t *testing.T) { diff --git a/ethdb/memory_database.go b/ethdb/memory_database.go index 754d78ab3b..51ddd5d62b 100644 --- a/ethdb/memory_database.go +++ b/ethdb/memory_database.go @@ -31,16 +31,16 @@ type MemDatabase struct { lock sync.RWMutex } -func NewMemDatabase() (*MemDatabase, error) { +func NewMemDatabase() *MemDatabase { return &MemDatabase{ db: make(map[string][]byte), - }, nil + } } -func NewMemDatabaseWithCap(size int) (*MemDatabase, error) { +func NewMemDatabaseWithCap(size int) *MemDatabase { return &MemDatabase{ db: make(map[string][]byte, size), - }, nil + } } func (db *MemDatabase) Put(key []byte, value []byte) error { diff --git a/ethstats/ethstats.go b/ethstats/ethstats.go index b0efe59560..f45fd20e2b 100644 --- a/ethstats/ethstats.go +++ b/ethstats/ethstats.go @@ -49,7 +49,7 @@ const ( // history request. historyUpdateRange = 50 - // txChanSize is the size of channel listening to TxPreEvent. + // txChanSize is the size of channel listening to NewTxsEvent. // The number is referenced from the size of tx pool. txChanSize = 4096 // chainHeadChanSize is the size of channel listening to ChainHeadEvent. @@ -57,9 +57,9 @@ const ( ) type txPool interface { - // SubscribeTxPreEvent should return an event subscription of - // TxPreEvent and send events to the given channel. - SubscribeTxPreEvent(chan<- core.TxPreEvent) event.Subscription + // SubscribeNewTxsEvent should return an event subscription of + // NewTxsEvent and send events to the given channel. + SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription } type blockChain interface { @@ -150,8 +150,8 @@ func (s *Service) loop() { headSub := blockchain.SubscribeChainHeadEvent(chainHeadCh) defer headSub.Unsubscribe() - txEventCh := make(chan core.TxPreEvent, txChanSize) - txSub := txpool.SubscribeTxPreEvent(txEventCh) + txEventCh := make(chan core.NewTxsEvent, txChanSize) + txSub := txpool.SubscribeNewTxsEvent(txEventCh) defer txSub.Unsubscribe() // Start a goroutine that exhausts the subsciptions to avoid events piling up @@ -689,7 +689,7 @@ func (s *Service) reportStats(conn *websocket.Conn) error { sync := s.eth.Downloader().Progress() syncing = s.eth.BlockChain().CurrentHeader().Number.Uint64() >= sync.HighestBlock - price, _ := s.eth.ApiBackend.SuggestPrice(context.Background()) + price, _ := s.eth.APIBackend.SuggestPrice(context.Background()) gasprice = int(price.Uint64()) } else { sync := s.les.Downloader().Progress() diff --git a/event/event.go b/event/event.go index 20d20d1f57..4232787314 100644 --- a/event/event.go +++ b/event/event.go @@ -180,6 +180,12 @@ func (s *TypeMuxSubscription) Unsubscribe() { s.closewait() } +func (s *TypeMuxSubscription) Closed() bool { + s.closeMu.Lock() + defer s.closeMu.Unlock() + return s.closed +} + func (s *TypeMuxSubscription) closewait() { s.closeMu.Lock() defer s.closeMu.Unlock() diff --git a/event/feed.go b/event/feed.go index 78fa3d98d8..f578f00c10 100644 --- a/event/feed.go +++ b/event/feed.go @@ -148,7 +148,9 @@ func (f *Feed) Send(value interface{}) (nsent int) { f.sendCases[i].Send = rvalue } - // Send until all channels except removeSub have been chosen. + // Send until all channels except removeSub have been chosen. 'cases' tracks a prefix + // of sendCases. When a send succeeds, the corresponding case moves to the end of + // 'cases' and it shrinks by one element. cases := f.sendCases for { // Fast path: try sending without blocking before adding to the select set. @@ -170,6 +172,7 @@ func (f *Feed) Send(value interface{}) (nsent int) { index := f.sendCases.find(recv.Interface()) f.sendCases = f.sendCases.delete(index) if index >= 0 && index < len(cases) { + // Shrink 'cases' too because the removed case was still active. cases = f.sendCases[:len(cases)-1] } } else { diff --git a/event/feed_test.go b/event/feed_test.go index a82c103033..be8876932c 100644 --- a/event/feed_test.go +++ b/event/feed_test.go @@ -235,6 +235,45 @@ func TestFeedUnsubscribeBlockedPost(t *testing.T) { wg.Wait() } +// Checks that unsubscribing a channel during Send works even if that +// channel has already been sent on. +func TestFeedUnsubscribeSentChan(t *testing.T) { + var ( + feed Feed + ch1 = make(chan int) + ch2 = make(chan int) + sub1 = feed.Subscribe(ch1) + sub2 = feed.Subscribe(ch2) + wg sync.WaitGroup + ) + defer sub2.Unsubscribe() + + wg.Add(1) + go func() { + feed.Send(0) + wg.Done() + }() + + // Wait for the value on ch1. + <-ch1 + // Unsubscribe ch1, removing it from the send cases. + sub1.Unsubscribe() + + // Receive ch2, finishing Send. + <-ch2 + wg.Wait() + + // Send again. This should send to ch2 only, so the wait group will unblock + // as soon as a value is received on ch2. + wg.Add(1) + go func() { + feed.Send(0) + wg.Done() + }() + <-ch2 + wg.Wait() +} + func TestFeedUnsubscribeFromInbox(t *testing.T) { var ( feed Feed diff --git a/event/filter/filter.go b/event/filter/filter.go index b1fbf30ee4..a6fe46d6a0 100644 --- a/event/filter/filter.go +++ b/event/filter/filter.go @@ -45,37 +45,37 @@ func New() *Filters { } } -func (self *Filters) Start() { - go self.loop() +func (f *Filters) Start() { + go f.loop() } -func (self *Filters) Stop() { - close(self.quit) +func (f *Filters) Stop() { + close(f.quit) } -func (self *Filters) Notify(filter Filter, data interface{}) { - self.ch <- FilterEvent{filter, data} +func (f *Filters) Notify(filter Filter, data interface{}) { + f.ch <- FilterEvent{filter, data} } -func (self *Filters) Install(watcher Filter) int { - self.watchers[self.id] = watcher - self.id++ +func (f *Filters) Install(watcher Filter) int { + f.watchers[f.id] = watcher + f.id++ - return self.id - 1 + return f.id - 1 } -func (self *Filters) Uninstall(id int) { - delete(self.watchers, id) +func (f *Filters) Uninstall(id int) { + delete(f.watchers, id) } -func (self *Filters) loop() { +func (f *Filters) loop() { out: for { select { - case <-self.quit: + case <-f.quit: break out - case event := <-self.ch: - for _, watcher := range self.watchers { + case event := <-f.ch: + for _, watcher := range f.watchers { if reflect.TypeOf(watcher) == reflect.TypeOf(event.filter) { if watcher.Compare(event.filter) { watcher.Trigger(event.data) @@ -86,10 +86,10 @@ out: } } -func (self *Filters) Match(a, b Filter) bool { +func (f *Filters) Match(a, b Filter) bool { return reflect.TypeOf(a) == reflect.TypeOf(b) && a.Compare(b) } -func (self *Filters) Get(i int) Filter { - return self.watchers[i] +func (f *Filters) Get(i int) Filter { + return f.watchers[i] } diff --git a/internal/build/pgp.go b/internal/build/pgp.go index 79ab9c06f1..c7d0d23397 100644 --- a/internal/build/pgp.go +++ b/internal/build/pgp.go @@ -57,3 +57,15 @@ func PGPSignFile(input string, output string, pgpkey string) error { // Generate the signature and return return openpgp.ArmoredDetachSign(out, keys[0], in, nil) } + +// PGPKeyID parses an armored key and returns the key ID. +func PGPKeyID(pgpkey string) (string, error) { + keys, err := openpgp.ReadArmoredKeyRing(bytes.NewBufferString(pgpkey)) + if err != nil { + return "", err + } + if len(keys) != 1 { + return "", fmt.Errorf("key count mismatch: have %d, want %d", len(keys), 1) + } + return keys[0].PrimaryKey.KeyIdString(), nil +} diff --git a/internal/debug/flags.go b/internal/debug/flags.go index 0ebd50da5e..4c6e65a5dd 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -28,10 +28,13 @@ import ( "github.com/EthereumCommonwealth/go-callisto/log/term" "github.com/EthereumCommonwealth/go-callisto/metrics" "github.com/EthereumCommonwealth/go-callisto/metrics/exp" + "github.com/fjl/memsize/memsizeui" colorable "github.com/mattn/go-colorable" "gopkg.in/urfave/cli.v1" ) +var Memsize memsizeui.Handler + var ( verbosityFlag = cli.IntFlag{ Name: "verbosity", @@ -129,21 +132,25 @@ func Setup(ctx *cli.Context) error { // pprof server if ctx.GlobalBool(pprofFlag.Name) { - // Hook go-metrics into expvar on any /debug/metrics request, load all vars - // from the registry into expvar, and execute regular expvar handler. - exp.Exp(metrics.DefaultRegistry) - address := fmt.Sprintf("%s:%d", ctx.GlobalString(pprofAddrFlag.Name), ctx.GlobalInt(pprofPortFlag.Name)) - go func() { - log.Info("Starting pprof server", "addr", fmt.Sprintf("http://%s/debug/pprof", address)) - if err := http.ListenAndServe(address, nil); err != nil { - log.Error("Failure in running pprof server", "err", err) - } - }() + StartPProf(address) } return nil } +func StartPProf(address string) { + // Hook go-metrics into expvar on any /debug/metrics request, load all vars + // from the registry into expvar, and execute regular expvar handler. + exp.Exp(metrics.DefaultRegistry) + http.Handle("/memsize/", http.StripPrefix("/memsize", &Memsize)) + log.Info("Starting pprof server", "addr", fmt.Sprintf("http://%s/debug/pprof", address)) + go func() { + if err := http.ListenAndServe(address, nil); err != nil { + log.Error("Failure in running pprof server", "err", err) + } + }() +} + // Exit stops all running profiles, flushing their output to the // respective file. func Exit() { diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index f27f08b0b9..e2abf9addd 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -32,6 +32,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common/math" "github.com/EthereumCommonwealth/go-callisto/consensus/ethash" "github.com/EthereumCommonwealth/go-callisto/core" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/core/vm" "github.com/EthereumCommonwealth/go-callisto/crypto" @@ -1006,7 +1007,7 @@ func (s *PublicTransactionPoolAPI) GetTransactionCount(ctx context.Context, addr // GetTransactionByHash returns the transaction for the given hash func (s *PublicTransactionPoolAPI) GetTransactionByHash(ctx context.Context, hash common.Hash) *RPCTransaction { // Try to return an already finalized transaction - if tx, blockHash, blockNumber, index := core.GetTransaction(s.b.ChainDb(), hash); tx != nil { + if tx, blockHash, blockNumber, index := rawdb.ReadTransaction(s.b.ChainDb(), hash); tx != nil { return newRPCTransaction(tx, blockHash, blockNumber, index) } // No finalized transaction, try to retrieve it from the pool @@ -1022,7 +1023,7 @@ func (s *PublicTransactionPoolAPI) GetRawTransactionByHash(ctx context.Context, var tx *types.Transaction // Retrieve a finalized transaction, or a pooled otherwise - if tx, _, _, _ = core.GetTransaction(s.b.ChainDb(), hash); tx == nil { + if tx, _, _, _ = rawdb.ReadTransaction(s.b.ChainDb(), hash); tx == nil { if tx = s.b.GetPoolTransaction(hash); tx == nil { // Transaction not found anywhere, abort return nil, nil @@ -1034,7 +1035,7 @@ func (s *PublicTransactionPoolAPI) GetRawTransactionByHash(ctx context.Context, // GetTransactionReceipt returns the transaction receipt for the given transaction hash. func (s *PublicTransactionPoolAPI) GetTransactionReceipt(ctx context.Context, hash common.Hash) (map[string]interface{}, error) { - tx, blockHash, blockNumber, index := core.GetTransaction(s.b.ChainDb(), hash) + tx, blockHash, blockNumber, index := rawdb.ReadTransaction(s.b.ChainDb(), hash) if tx == nil { //return nil, errors.New("unknown transaction") return nil, nil diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index bd2e6b5fb4..0e4ab6b2a5 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -65,7 +65,7 @@ type Backend interface { GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) Stats() (pending int, queued int) TxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions) - SubscribeTxPreEvent(chan<- core.TxPreEvent) event.Subscription + SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription ChainConfig() *params.ChainConfig CurrentBlock() *types.Block diff --git a/internal/jsre/jsre.go b/internal/jsre/jsre.go index 8916161492..0da845abd2 100644 --- a/internal/jsre/jsre.go +++ b/internal/jsre/jsre.go @@ -102,8 +102,8 @@ func randomSource() *rand.Rand { // call the functions of the otto vm directly to circumvent the queue. These // functions should be used if and only if running a routine that was already // called from JS through an RPC call. -func (self *JSRE) runEventLoop() { - defer close(self.closed) +func (re *JSRE) runEventLoop() { + defer close(re.closed) vm := otto.New() r := randomSource() @@ -202,14 +202,14 @@ loop: break loop } } - case req := <-self.evalQueue: + case req := <-re.evalQueue: // run the code, send the result back req.fn(vm) close(req.done) if waitForCallbacks && (len(registry) == 0) { break loop } - case waitForCallbacks = <-self.stopEventLoop: + case waitForCallbacks = <-re.stopEventLoop: if !waitForCallbacks || (len(registry) == 0) { break loop } @@ -223,31 +223,31 @@ loop: } // Do executes the given function on the JS event loop. -func (self *JSRE) Do(fn func(*otto.Otto)) { +func (re *JSRE) Do(fn func(*otto.Otto)) { done := make(chan bool) req := &evalReq{fn, done} - self.evalQueue <- req + re.evalQueue <- req <-done } // stops the event loop before exit, optionally waits for all timers to expire -func (self *JSRE) Stop(waitForCallbacks bool) { +func (re *JSRE) Stop(waitForCallbacks bool) { select { - case <-self.closed: - case self.stopEventLoop <- waitForCallbacks: - <-self.closed + case <-re.closed: + case re.stopEventLoop <- waitForCallbacks: + <-re.closed } } // Exec(file) loads and runs the contents of a file // if a relative path is given, the jsre's assetPath is used -func (self *JSRE) Exec(file string) error { - code, err := ioutil.ReadFile(common.AbsolutePath(self.assetPath, file)) +func (re *JSRE) Exec(file string) error { + code, err := ioutil.ReadFile(common.AbsolutePath(re.assetPath, file)) if err != nil { return err } var script *otto.Script - self.Do(func(vm *otto.Otto) { + re.Do(func(vm *otto.Otto) { script, err = vm.Compile(file, code) if err != nil { return @@ -259,36 +259,36 @@ func (self *JSRE) Exec(file string) error { // Bind assigns value v to a variable in the JS environment // This method is deprecated, use Set. -func (self *JSRE) Bind(name string, v interface{}) error { - return self.Set(name, v) +func (re *JSRE) Bind(name string, v interface{}) error { + return re.Set(name, v) } // Run runs a piece of JS code. -func (self *JSRE) Run(code string) (v otto.Value, err error) { - self.Do(func(vm *otto.Otto) { v, err = vm.Run(code) }) +func (re *JSRE) Run(code string) (v otto.Value, err error) { + re.Do(func(vm *otto.Otto) { v, err = vm.Run(code) }) return v, err } // Get returns the value of a variable in the JS environment. -func (self *JSRE) Get(ns string) (v otto.Value, err error) { - self.Do(func(vm *otto.Otto) { v, err = vm.Get(ns) }) +func (re *JSRE) Get(ns string) (v otto.Value, err error) { + re.Do(func(vm *otto.Otto) { v, err = vm.Get(ns) }) return v, err } // Set assigns value v to a variable in the JS environment. -func (self *JSRE) Set(ns string, v interface{}) (err error) { - self.Do(func(vm *otto.Otto) { err = vm.Set(ns, v) }) +func (re *JSRE) Set(ns string, v interface{}) (err error) { + re.Do(func(vm *otto.Otto) { err = vm.Set(ns, v) }) return err } // loadScript executes a JS script from inside the currently executing JS code. -func (self *JSRE) loadScript(call otto.FunctionCall) otto.Value { +func (re *JSRE) loadScript(call otto.FunctionCall) otto.Value { file, err := call.Argument(0).ToString() if err != nil { // TODO: throw exception return otto.FalseValue() } - file = common.AbsolutePath(self.assetPath, file) + file = common.AbsolutePath(re.assetPath, file) source, err := ioutil.ReadFile(file) if err != nil { // TODO: throw exception @@ -305,10 +305,10 @@ func (self *JSRE) loadScript(call otto.FunctionCall) otto.Value { // Evaluate executes code and pretty prints the result to the specified output // stream. -func (self *JSRE) Evaluate(code string, w io.Writer) error { +func (re *JSRE) Evaluate(code string, w io.Writer) error { var fail error - self.Do(func(vm *otto.Otto) { + re.Do(func(vm *otto.Otto) { val, err := vm.Run(code) if err != nil { prettyError(vm, err, w) @@ -321,8 +321,8 @@ func (self *JSRE) Evaluate(code string, w io.Writer) error { } // Compile compiles and then runs a piece of JS code. -func (self *JSRE) Compile(filename string, src interface{}) (err error) { - self.Do(func(vm *otto.Otto) { _, err = compileAndRun(vm, filename, src) }) +func (re *JSRE) Compile(filename string, src interface{}) (err error) { + re.Do(func(vm *otto.Otto) { _, err = compileAndRun(vm, filename, src) }) return err } diff --git a/les/api_backend.go b/les/api_backend.go index df57671839..9a8a30e0ff 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -25,6 +25,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common/math" "github.com/EthereumCommonwealth/go-callisto/core" "github.com/EthereumCommonwealth/go-callisto/core/bloombits" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/state" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/core/vm" @@ -83,16 +84,22 @@ func (b *LesApiBackend) GetBlock(ctx context.Context, blockHash common.Hash) (*t return b.eth.blockchain.GetBlockByHash(ctx, blockHash) } -func (b *LesApiBackend) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) { - return light.GetBlockReceipts(ctx, b.eth.odr, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash)) +func (b *LesApiBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { + if number := rawdb.ReadHeaderNumber(b.eth.chainDb, hash); number != nil { + return light.GetBlockReceipts(ctx, b.eth.odr, hash, *number) + } + return nil, nil } -func (b *LesApiBackend) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) { - return light.GetBlockLogs(ctx, b.eth.odr, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash)) +func (b *LesApiBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { + if number := rawdb.ReadHeaderNumber(b.eth.chainDb, hash); number != nil { + return light.GetBlockLogs(ctx, b.eth.odr, hash, *number) + } + return nil, nil } -func (b *LesApiBackend) GetTd(blockHash common.Hash) *big.Int { - return b.eth.blockchain.GetTdByHash(blockHash) +func (b *LesApiBackend) GetTd(hash common.Hash) *big.Int { + return b.eth.blockchain.GetTdByHash(hash) } func (b *LesApiBackend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) { @@ -129,8 +136,8 @@ func (b *LesApiBackend) TxPoolContent() (map[common.Address]types.Transactions, return b.eth.txPool.Content() } -func (b *LesApiBackend) SubscribeTxPreEvent(ch chan<- core.TxPreEvent) event.Subscription { - return b.eth.txPool.SubscribeTxPreEvent(ch) +func (b *LesApiBackend) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subscription { + return b.eth.txPool.SubscribeNewTxsEvent(ch) } func (b *LesApiBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { diff --git a/les/backend.go b/les/backend.go index 0d332f97b0..a57db92a0e 100644 --- a/les/backend.go +++ b/les/backend.go @@ -28,6 +28,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/consensus" "github.com/EthereumCommonwealth/go-callisto/core" "github.com/EthereumCommonwealth/go-callisto/core/bloombits" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/eth" "github.com/EthereumCommonwealth/go-callisto/eth/downloader" @@ -122,7 +123,7 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { if compat, ok := genesisErr.(*params.ConfigCompatError); ok { log.Warn("Rewinding chain to upgrade configuration", "err", compat) leth.blockchain.SetHead(compat.RewindTo) - core.WriteChainConfig(chainDb, genesisHash, chainConfig) + rawdb.WriteChainConfig(chainDb, genesisHash, chainConfig) } leth.txPool = light.NewTxPool(leth.chainConfig, leth.blockchain, leth.relay) diff --git a/les/distributor_test.go b/les/distributor_test.go index 55defb69be..2891bcab49 100644 --- a/les/distributor_test.go +++ b/les/distributor_test.go @@ -97,9 +97,8 @@ func (p *testDistPeer) waitBefore(cost uint64) (time.Duration, float64) { p.lock.RUnlock() if sumCost < testDistBufLimit { return 0, float64(testDistBufLimit-sumCost) / float64(testDistBufLimit) - } else { - return time.Duration(sumCost - testDistBufLimit), 0 } + return time.Duration(sumCost - testDistBufLimit), 0 } func (p *testDistPeer) canQueue() bool { diff --git a/les/fetcher.go b/les/fetcher.go index 546667e021..477cc5047d 100644 --- a/les/fetcher.go +++ b/les/fetcher.go @@ -25,7 +25,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common" "github.com/EthereumCommonwealth/go-callisto/common/mclock" "github.com/EthereumCommonwealth/go-callisto/consensus" - "github.com/EthereumCommonwealth/go-callisto/core" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/light" "github.com/EthereumCommonwealth/go-callisto/log" @@ -280,7 +280,7 @@ func (f *lightFetcher) announce(p *peer, head *announceData) { // if one of root's children is canonical, keep it, delete other branches and root itself var newRoot *fetcherTreeNode for i, nn := range fp.root.children { - if core.GetCanonicalHash(f.pm.chainDb, nn.number) == nn.hash { + if rawdb.ReadCanonicalHash(f.pm.chainDb, nn.number) == nn.hash { fp.root.children = append(fp.root.children[:i], fp.root.children[i+1:]...) nn.parent = nil newRoot = nn @@ -363,7 +363,7 @@ func (f *lightFetcher) peerHasBlock(p *peer, hash common.Hash, number uint64) bo // // when syncing, just check if it is part of the known chain, there is nothing better we // can do since we do not know the most recent block hash yet - return core.GetCanonicalHash(f.pm.chainDb, fp.root.number) == fp.root.hash && core.GetCanonicalHash(f.pm.chainDb, number) == hash + return rawdb.ReadCanonicalHash(f.pm.chainDb, fp.root.number) == fp.root.hash && rawdb.ReadCanonicalHash(f.pm.chainDb, number) == hash } // requestAmount calculates the amount of headers to be downloaded starting diff --git a/les/handler.go b/les/handler.go index 07c76bbcb0..f9ba194e9f 100644 --- a/les/handler.go +++ b/les/handler.go @@ -29,6 +29,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common" "github.com/EthereumCommonwealth/go-callisto/consensus" "github.com/EthereumCommonwealth/go-callisto/core" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/state" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/eth/downloader" @@ -528,9 +529,11 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { break } // Retrieve the requested block body, stopping if enough was found - if data := core.GetBodyRLP(pm.chainDb, hash, core.GetBlockNumber(pm.chainDb, hash)); len(data) != 0 { - bodies = append(bodies, data) - bytes += len(data) + if number := rawdb.ReadHeaderNumber(pm.chainDb, hash); number != nil { + if data := rawdb.ReadBodyRLP(pm.chainDb, hash, *number); len(data) != 0 { + bodies = append(bodies, data) + bytes += len(data) + } } } bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost) @@ -579,20 +582,22 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } for _, req := range req.Reqs { // Retrieve the requested state entry, stopping if enough was found - if header := core.GetHeader(pm.chainDb, req.BHash, core.GetBlockNumber(pm.chainDb, req.BHash)); header != nil { - statedb, err := pm.blockchain.State() - if err != nil { - continue - } - account, err := pm.getAccount(statedb, header.Root, common.BytesToHash(req.AccKey)) - if err != nil { - continue - } - code, _ := statedb.Database().TrieDB().Node(common.BytesToHash(account.CodeHash)) + if number := rawdb.ReadHeaderNumber(pm.chainDb, req.BHash); number != nil { + if header := rawdb.ReadHeader(pm.chainDb, req.BHash, *number); header != nil { + statedb, err := pm.blockchain.State() + if err != nil { + continue + } + account, err := pm.getAccount(statedb, header.Root, common.BytesToHash(req.AccKey)) + if err != nil { + continue + } + code, _ := statedb.Database().TrieDB().Node(common.BytesToHash(account.CodeHash)) - data = append(data, code) - if bytes += len(code); bytes >= softResponseLimit { - break + data = append(data, code) + if bytes += len(code); bytes >= softResponseLimit { + break + } } } } @@ -645,7 +650,10 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { break } // Retrieve the requested block's receipts, skipping if unknown to us - results := core.GetBlockReceipts(pm.chainDb, hash, core.GetBlockNumber(pm.chainDb, hash)) + var results types.Receipts + if number := rawdb.ReadHeaderNumber(pm.chainDb, hash); number != nil { + results = rawdb.ReadReceipts(pm.chainDb, hash, *number) + } if results == nil { if header := pm.blockchain.GetHeaderByHash(hash); header == nil || header.ReceiptHash != types.EmptyRootHash { continue @@ -705,28 +713,30 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } for _, req := range req.Reqs { // Retrieve the requested state entry, stopping if enough was found - if header := core.GetHeader(pm.chainDb, req.BHash, core.GetBlockNumber(pm.chainDb, req.BHash)); header != nil { - statedb, err := pm.blockchain.State() - if err != nil { - continue - } - var trie state.Trie - if len(req.AccKey) > 0 { - account, err := pm.getAccount(statedb, header.Root, common.BytesToHash(req.AccKey)) + if number := rawdb.ReadHeaderNumber(pm.chainDb, req.BHash); number != nil { + if header := rawdb.ReadHeader(pm.chainDb, req.BHash, *number); header != nil { + statedb, err := pm.blockchain.State() if err != nil { continue } - trie, _ = statedb.Database().OpenStorageTrie(common.BytesToHash(req.AccKey), account.Root) - } else { - trie, _ = statedb.Database().OpenTrie(header.Root) - } - if trie != nil { - var proof light.NodeList - trie.Prove(req.Key, 0, &proof) + var trie state.Trie + if len(req.AccKey) > 0 { + account, err := pm.getAccount(statedb, header.Root, common.BytesToHash(req.AccKey)) + if err != nil { + continue + } + trie, _ = statedb.Database().OpenStorageTrie(common.BytesToHash(req.AccKey), account.Root) + } else { + trie, _ = statedb.Database().OpenTrie(header.Root) + } + if trie != nil { + var proof light.NodeList + trie.Prove(req.Key, 0, &proof) - proofs = append(proofs, proof) - if bytes += proof.DataSize(); bytes >= softResponseLimit { - break + proofs = append(proofs, proof) + if bytes += proof.DataSize(); bytes >= softResponseLimit { + break + } } } } @@ -763,9 +773,11 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { if statedb == nil || req.BHash != lastBHash { statedb, root, lastBHash = nil, common.Hash{}, req.BHash - if header := core.GetHeader(pm.chainDb, req.BHash, core.GetBlockNumber(pm.chainDb, req.BHash)); header != nil { - statedb, _ = pm.blockchain.State() - root = header.Root + if number := rawdb.ReadHeaderNumber(pm.chainDb, req.BHash); number != nil { + if header := rawdb.ReadHeader(pm.chainDb, req.BHash, *number); header != nil { + statedb, _ = pm.blockchain.State() + root = header.Root + } } } if statedb == nil { @@ -859,7 +871,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { trieDb := trie.NewDatabase(ethdb.NewTable(pm.chainDb, light.ChtTablePrefix)) for _, req := range req.Reqs { if header := pm.blockchain.GetHeaderByNumber(req.BlockNum); header != nil { - sectionHead := core.GetCanonicalHash(pm.chainDb, req.ChtNum*light.CHTFrequencyServer-1) + sectionHead := rawdb.ReadCanonicalHash(pm.chainDb, req.ChtNum*light.CHTFrequencyServer-1) if root := light.GetChtRoot(pm.chainDb, req.ChtNum-1, sectionHead); root != (common.Hash{}) { trie, err := trie.New(root, trieDb) if err != nil { @@ -1114,10 +1126,10 @@ func (pm *ProtocolManager) getAccount(statedb *state.StateDB, root, hash common. func (pm *ProtocolManager) getHelperTrie(id uint, idx uint64) (common.Hash, string) { switch id { case htCanonical: - sectionHead := core.GetCanonicalHash(pm.chainDb, (idx+1)*light.CHTFrequencyClient-1) + sectionHead := rawdb.ReadCanonicalHash(pm.chainDb, (idx+1)*light.CHTFrequencyClient-1) return light.GetChtV2Root(pm.chainDb, idx, sectionHead), light.ChtTablePrefix case htBloomBits: - sectionHead := core.GetCanonicalHash(pm.chainDb, (idx+1)*light.BloomTrieFrequency-1) + sectionHead := rawdb.ReadCanonicalHash(pm.chainDb, (idx+1)*light.BloomTrieFrequency-1) return light.GetBloomTrieRoot(pm.chainDb, idx, sectionHead), light.BloomTrieTablePrefix } return common.Hash{}, "" @@ -1128,8 +1140,8 @@ func (pm *ProtocolManager) getHelperTrieAuxData(req HelperTrieReq) []byte { switch { case req.Type == htCanonical && req.AuxReq == auxHeader && len(req.Key) == 8: blockNum := binary.BigEndian.Uint64(req.Key) - hash := core.GetCanonicalHash(pm.chainDb, blockNum) - return core.GetHeaderRLP(pm.chainDb, hash, blockNum) + hash := rawdb.ReadCanonicalHash(pm.chainDb, blockNum) + return rawdb.ReadHeaderRLP(pm.chainDb, hash, blockNum) } return nil } @@ -1142,9 +1154,9 @@ func (pm *ProtocolManager) txStatus(hashes []common.Hash) []txStatus { // If the transaction is unknown to the pool, try looking it up locally if stat == core.TxStatusUnknown { - if block, number, index := core.GetTxLookupEntry(pm.chainDb, hashes[i]); block != (common.Hash{}) { + if block, number, index := rawdb.ReadTxLookupEntry(pm.chainDb, hashes[i]); block != (common.Hash{}) { stats[i].Status = core.TxStatusIncluded - stats[i].Lookup = &core.TxLookupEntry{BlockHash: block, BlockIndex: number, Index: index} + stats[i].Lookup = &rawdb.TxLookupEntry{BlockHash: block, BlockIndex: number, Index: index} } } } diff --git a/les/handler_test.go b/les/handler_test.go index 1627ac2c3e..e67119ff73 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -26,6 +26,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common" "github.com/EthereumCommonwealth/go-callisto/consensus/ethash" "github.com/EthereumCommonwealth/go-callisto/core" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/crypto" "github.com/EthereumCommonwealth/go-callisto/eth/downloader" @@ -50,8 +51,7 @@ func TestGetBlockHeadersLes1(t *testing.T) { testGetBlockHeaders(t, 1) } func TestGetBlockHeadersLes2(t *testing.T) { testGetBlockHeaders(t, 2) } func testGetBlockHeaders(t *testing.T, protocol int) { - db, _ := ethdb.NewMemDatabase() - pm := newTestProtocolManagerMust(t, false, downloader.MaxHashFetch+15, nil, nil, nil, db) + pm := newTestProtocolManagerMust(t, false, downloader.MaxHashFetch+15, nil, nil, nil, ethdb.NewMemDatabase()) bc := pm.blockchain.(*core.BlockChain) peer, _ := newTestPeer(t, "peer", protocol, pm, true) defer peer.close() @@ -180,8 +180,7 @@ func TestGetBlockBodiesLes1(t *testing.T) { testGetBlockBodies(t, 1) } func TestGetBlockBodiesLes2(t *testing.T) { testGetBlockBodies(t, 2) } func testGetBlockBodies(t *testing.T, protocol int) { - db, _ := ethdb.NewMemDatabase() - pm := newTestProtocolManagerMust(t, false, downloader.MaxBlockFetch+15, nil, nil, nil, db) + pm := newTestProtocolManagerMust(t, false, downloader.MaxBlockFetch+15, nil, nil, nil, ethdb.NewMemDatabase()) bc := pm.blockchain.(*core.BlockChain) peer, _ := newTestPeer(t, "peer", protocol, pm, true) defer peer.close() @@ -258,8 +257,7 @@ func TestGetCodeLes2(t *testing.T) { testGetCode(t, 2) } func testGetCode(t *testing.T, protocol int) { // Assemble the test environment - db, _ := ethdb.NewMemDatabase() - pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, db) + pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, ethdb.NewMemDatabase()) bc := pm.blockchain.(*core.BlockChain) peer, _ := newTestPeer(t, "peer", protocol, pm, true) defer peer.close() @@ -292,7 +290,7 @@ func TestGetReceiptLes2(t *testing.T) { testGetReceipt(t, 2) } func testGetReceipt(t *testing.T, protocol int) { // Assemble the test environment - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, db) bc := pm.blockchain.(*core.BlockChain) peer, _ := newTestPeer(t, "peer", protocol, pm, true) @@ -304,7 +302,7 @@ func testGetReceipt(t *testing.T, protocol int) { block := bc.GetBlockByNumber(i) hashes = append(hashes, block.Hash()) - receipts = append(receipts, core.GetBlockReceipts(db, block.Hash(), block.NumberU64())) + receipts = append(receipts, rawdb.ReadReceipts(db, block.Hash(), block.NumberU64())) } // Send the hash request and verify the response cost := peer.GetRequestCost(GetReceiptsMsg, len(hashes)) @@ -320,7 +318,7 @@ func TestGetProofsLes2(t *testing.T) { testGetProofs(t, 2) } func testGetProofs(t *testing.T, protocol int) { // Assemble the test environment - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, db) bc := pm.blockchain.(*core.BlockChain) peer, _ := newTestPeer(t, "peer", protocol, pm, true) @@ -383,7 +381,7 @@ func testGetCHTProofs(t *testing.T, protocol int) { frequency = uint64(light.CHTFrequencyServer) } // Assemble the test environment - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() pm := newTestProtocolManagerMust(t, false, int(frequency)+light.HelperTrieProcessConfirmations, testChainGen, nil, nil, db) bc := pm.blockchain.(*core.BlockChain) peer, _ := newTestPeer(t, "peer", protocol, pm, true) @@ -451,7 +449,7 @@ func testGetCHTProofs(t *testing.T, protocol int) { // Tests that bloombits proofs can be correctly retrieved. func TestGetBloombitsProofs(t *testing.T) { // Assemble the test environment - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() pm := newTestProtocolManagerMust(t, false, light.BloomTrieFrequency+256, testChainGen, nil, nil, db) bc := pm.blockchain.(*core.BlockChain) peer, _ := newTestPeer(t, "peer", 2, pm, true) @@ -490,7 +488,7 @@ func TestGetBloombitsProofs(t *testing.T) { } func TestTransactionStatusLes2(t *testing.T) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() pm := newTestProtocolManagerMust(t, false, 0, nil, nil, nil, db) chain := pm.blockchain.(*core.BlockChain) config := core.DefaultTxPoolConfig @@ -555,9 +553,9 @@ func TestTransactionStatusLes2(t *testing.T) { } // check if their status is included now - block1hash := core.GetCanonicalHash(db, 1) - test(tx1, false, txStatus{Status: core.TxStatusIncluded, Lookup: &core.TxLookupEntry{BlockHash: block1hash, BlockIndex: 1, Index: 0}}) - test(tx2, false, txStatus{Status: core.TxStatusIncluded, Lookup: &core.TxLookupEntry{BlockHash: block1hash, BlockIndex: 1, Index: 1}}) + block1hash := rawdb.ReadCanonicalHash(db, 1) + test(tx1, false, txStatus{Status: core.TxStatusIncluded, Lookup: &rawdb.TxLookupEntry{BlockHash: block1hash, BlockIndex: 1, Index: 0}}) + test(tx2, false, txStatus{Status: core.TxStatusIncluded, Lookup: &rawdb.TxLookupEntry{BlockHash: block1hash, BlockIndex: 1, Index: 1}}) // create a reorg that rolls them back gchain, _ = core.GenerateChain(params.TestChainConfig, chain.GetBlockByNumber(0), ethash.NewFaker(), db, 2, func(i int, block *core.BlockGen) {}) diff --git a/les/odr_requests.go b/les/odr_requests.go index 3bd4e4f867..ef12688eb3 100644 --- a/les/odr_requests.go +++ b/les/odr_requests.go @@ -24,7 +24,7 @@ import ( "fmt" "github.com/EthereumCommonwealth/go-callisto/common" - "github.com/EthereumCommonwealth/go-callisto/core" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/crypto" "github.com/EthereumCommonwealth/go-callisto/ethdb" @@ -110,7 +110,7 @@ func (r *BlockRequest) Validate(db ethdb.Database, msg *Msg) error { body := bodies[0] // Retrieve our stored header and validate block content against it - header := core.GetHeader(db, r.Hash, r.Number) + header := rawdb.ReadHeader(db, r.Hash, r.Number) if header == nil { return errHeaderUnavailable } @@ -166,7 +166,7 @@ func (r *ReceiptsRequest) Validate(db ethdb.Database, msg *Msg) error { receipt := receipts[0] // Retrieve our stored header and validate receipt content against it - header := core.GetHeader(db, r.Hash, r.Number) + header := rawdb.ReadHeader(db, r.Hash, r.Number) if header == nil { return errHeaderUnavailable } @@ -230,7 +230,7 @@ func (r *TrieRequest) Validate(db ethdb.Database, msg *Msg) error { } nodeSet := proofs[0].NodeSet() // Verify the proof and store if checks out - if _, err, _ := trie.VerifyProof(r.Id.Root, r.Key, nodeSet); err != nil { + if _, _, err := trie.VerifyProof(r.Id.Root, r.Key, nodeSet); err != nil { return fmt.Errorf("merkle proof verification failed: %v", err) } r.Proof = nodeSet @@ -241,7 +241,7 @@ func (r *TrieRequest) Validate(db ethdb.Database, msg *Msg) error { // Verify the proof and store if checks out nodeSet := proofs.NodeSet() reads := &readTraceDB{db: nodeSet} - if _, err, _ := trie.VerifyProof(r.Id.Root, r.Key, reads); err != nil { + if _, _, err := trie.VerifyProof(r.Id.Root, r.Key, reads); err != nil { return fmt.Errorf("merkle proof verification failed: %v", err) } // check if all nodes have been read by VerifyProof @@ -400,7 +400,7 @@ func (r *ChtRequest) Validate(db ethdb.Database, msg *Msg) error { var encNumber [8]byte binary.BigEndian.PutUint64(encNumber[:], r.BlockNum) - value, err, _ := trie.VerifyProof(r.ChtRoot, encNumber[:], light.NodeList(proof.Proof).NodeSet()) + value, _, err := trie.VerifyProof(r.ChtRoot, encNumber[:], light.NodeList(proof.Proof).NodeSet()) if err != nil { return err } @@ -435,7 +435,7 @@ func (r *ChtRequest) Validate(db ethdb.Database, msg *Msg) error { binary.BigEndian.PutUint64(encNumber[:], r.BlockNum) reads := &readTraceDB{db: nodeSet} - value, err, _ := trie.VerifyProof(r.ChtRoot, encNumber[:], reads) + value, _, err := trie.VerifyProof(r.ChtRoot, encNumber[:], reads) if err != nil { return fmt.Errorf("merkle proof verification failed: %v", err) } @@ -529,7 +529,7 @@ func (r *BloomRequest) Validate(db ethdb.Database, msg *Msg) error { for i, idx := range r.SectionIdxList { binary.BigEndian.PutUint64(encNumber[2:], idx) - value, err, _ := trie.VerifyProof(r.BloomTrieRoot, encNumber[:], reads) + value, _, err := trie.VerifyProof(r.BloomTrieRoot, encNumber[:], reads) if err != nil { return err } diff --git a/les/odr_test.go b/les/odr_test.go index a2cf8b4281..a944f368b5 100644 --- a/les/odr_test.go +++ b/les/odr_test.go @@ -26,6 +26,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common" "github.com/EthereumCommonwealth/go-callisto/common/math" "github.com/EthereumCommonwealth/go-callisto/core" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/state" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/core/vm" @@ -63,9 +64,13 @@ func TestOdrGetReceiptsLes2(t *testing.T) { testOdr(t, 2, 1, odrGetReceipts) } func odrGetReceipts(ctx context.Context, db ethdb.Database, config *params.ChainConfig, bc *core.BlockChain, lc *light.LightChain, bhash common.Hash) []byte { var receipts types.Receipts if bc != nil { - receipts = core.GetBlockReceipts(db, bhash, core.GetBlockNumber(db, bhash)) + if number := rawdb.ReadHeaderNumber(db, bhash); number != nil { + receipts = rawdb.ReadReceipts(db, bhash, *number) + } } else { - receipts, _ = light.GetBlockReceipts(ctx, lc.Odr(), bhash, core.GetBlockNumber(db, bhash)) + if number := rawdb.ReadHeaderNumber(db, bhash); number != nil { + receipts, _ = light.GetBlockReceipts(ctx, lc.Odr(), bhash, *number) + } } if receipts == nil { return nil @@ -160,8 +165,8 @@ func testOdr(t *testing.T, protocol int, expFail uint64, fn odrTestFn) { peers := newPeerSet() dist := newRequestDistributor(peers, make(chan struct{})) rm := newRetrieveManager(peers, dist, nil) - db, _ := ethdb.NewMemDatabase() - ldb, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() + ldb := ethdb.NewMemDatabase() odr := NewLesOdr(ldb, light.NewChtIndexer(db, true), light.NewBloomTrieIndexer(db, true), eth.NewBloomIndexer(db, light.BloomTrieFrequency), rm) pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, db) lpm := newTestProtocolManagerMust(t, true, 0, nil, peers, odr, ldb) @@ -178,7 +183,7 @@ func testOdr(t *testing.T, protocol int, expFail uint64, fn odrTestFn) { test := func(expFail uint64) { for i := uint64(0); i <= pm.blockchain.CurrentHeader().Number.Uint64(); i++ { - bhash := core.GetCanonicalHash(db, i) + bhash := rawdb.ReadCanonicalHash(db, i) b1 := fn(light.NoOdr, db, pm.chainConfig, pm.blockchain.(*core.BlockChain), nil, bhash) ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) diff --git a/les/protocol.go b/les/protocol.go index b91917bb37..6974a71209 100644 --- a/les/protocol.go +++ b/les/protocol.go @@ -28,6 +28,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common" "github.com/EthereumCommonwealth/go-callisto/core" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/crypto" "github.com/EthereumCommonwealth/go-callisto/crypto/secp256k1" "github.com/EthereumCommonwealth/go-callisto/rlp" @@ -160,9 +161,8 @@ func (a *announceData) checkSignature(pubKey *ecdsa.PublicKey) error { pbytes := elliptic.Marshal(pubKey.Curve, pubKey.X, pubKey.Y) if bytes.Equal(pbytes, recPubkey) { return nil - } else { - return errors.New("Wrong signature") } + return errors.New("Wrong signature") } type blockInfo struct { @@ -224,6 +224,6 @@ type proofsData [][]rlp.RawValue type txStatus struct { Status core.TxStatus - Lookup *core.TxLookupEntry `rlp:"nil"` + Lookup *rawdb.TxLookupEntry `rlp:"nil"` Error string } diff --git a/les/randselect.go b/les/randselect.go index 1a9d0695bd..1cc1d3d3e0 100644 --- a/les/randselect.go +++ b/les/randselect.go @@ -118,17 +118,16 @@ func (n *wrsNode) insert(item wrsItem, weight int64) int { if n.level == 0 { n.items[branch] = item return branch - } else { - var subNode *wrsNode - if n.items[branch] == nil { - subNode = &wrsNode{maxItems: n.maxItems / wrsBranches, level: n.level - 1} - n.items[branch] = subNode - } else { - subNode = n.items[branch].(*wrsNode) - } - subIdx := subNode.insert(item, weight) - return subNode.maxItems*branch + subIdx } + var subNode *wrsNode + if n.items[branch] == nil { + subNode = &wrsNode{maxItems: n.maxItems / wrsBranches, level: n.level - 1} + n.items[branch] = subNode + } else { + subNode = n.items[branch].(*wrsNode) + } + subIdx := subNode.insert(item, weight) + return subNode.maxItems*branch + subIdx } // setWeight updates the weight of a certain item (which should exist) and returns @@ -162,12 +161,10 @@ func (n *wrsNode) choose(val int64) (wrsItem, int64) { if val < w { if n.level == 0 { return n.items[i].(wrsItem), n.weights[i] - } else { - return n.items[i].(*wrsNode).choose(val) } - } else { - val -= w + return n.items[i].(*wrsNode).choose(val) } + val -= w } panic(nil) } diff --git a/les/request_test.go b/les/request_test.go index 2ee4f0e50e..8621595b1b 100644 --- a/les/request_test.go +++ b/les/request_test.go @@ -22,7 +22,7 @@ import ( "time" "github.com/EthereumCommonwealth/go-callisto/common" - "github.com/EthereumCommonwealth/go-callisto/core" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/crypto" "github.com/EthereumCommonwealth/go-callisto/eth" "github.com/EthereumCommonwealth/go-callisto/ethdb" @@ -58,15 +58,22 @@ func TestTrieEntryAccessLes1(t *testing.T) { testAccess(t, 1, tfTrieEntryAccess) func TestTrieEntryAccessLes2(t *testing.T) { testAccess(t, 2, tfTrieEntryAccess) } func tfTrieEntryAccess(db ethdb.Database, bhash common.Hash, number uint64) light.OdrRequest { - return &light.TrieRequest{Id: light.StateTrieID(core.GetHeader(db, bhash, core.GetBlockNumber(db, bhash))), Key: testBankSecureTrieKey} + if number := rawdb.ReadHeaderNumber(db, bhash); number != nil { + return &light.TrieRequest{Id: light.StateTrieID(rawdb.ReadHeader(db, bhash, *number)), Key: testBankSecureTrieKey} + } + return nil } func TestCodeAccessLes1(t *testing.T) { testAccess(t, 1, tfCodeAccess) } func TestCodeAccessLes2(t *testing.T) { testAccess(t, 2, tfCodeAccess) } -func tfCodeAccess(db ethdb.Database, bhash common.Hash, number uint64) light.OdrRequest { - header := core.GetHeader(db, bhash, core.GetBlockNumber(db, bhash)) +func tfCodeAccess(db ethdb.Database, bhash common.Hash, num uint64) light.OdrRequest { + number := rawdb.ReadHeaderNumber(db, bhash) + if number != nil { + return nil + } + header := rawdb.ReadHeader(db, bhash, *number) if header.Number.Uint64() < testContractDeployed { return nil } @@ -80,8 +87,8 @@ func testAccess(t *testing.T, protocol int, fn accessTestFn) { peers := newPeerSet() dist := newRequestDistributor(peers, make(chan struct{})) rm := newRetrieveManager(peers, dist, nil) - db, _ := ethdb.NewMemDatabase() - ldb, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() + ldb := ethdb.NewMemDatabase() odr := NewLesOdr(ldb, light.NewChtIndexer(db, true), light.NewBloomTrieIndexer(db, true), eth.NewBloomIndexer(db, light.BloomTrieFrequency), rm) pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, db) @@ -99,7 +106,7 @@ func testAccess(t *testing.T, protocol int, fn accessTestFn) { test := func(expFail uint64) { for i := uint64(0); i <= pm.blockchain.CurrentHeader().Number.Uint64(); i++ { - bhash := core.GetCanonicalHash(db, i) + bhash := rawdb.ReadCanonicalHash(db, i) if req := fn(ldb, bhash, i); req != nil { ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) defer cancel() diff --git a/les/server.go b/les/server.go index c51b4ee43b..b5b7c193cb 100644 --- a/les/server.go +++ b/les/server.go @@ -25,6 +25,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common" "github.com/EthereumCommonwealth/go-callisto/core" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/eth" "github.com/EthereumCommonwealth/go-callisto/ethdb" @@ -329,11 +330,11 @@ func (pm *ProtocolManager) blockLoop() { header := ev.Block.Header() hash := header.Hash() number := header.Number.Uint64() - td := core.GetTd(pm.chainDb, hash, number) + td := rawdb.ReadTd(pm.chainDb, hash, number) if td != nil && td.Cmp(lastBroadcastTd) > 0 { var reorg uint64 if lastHead != nil { - reorg = lastHead.Number.Uint64() - core.FindCommonAncestor(pm.chainDb, header, lastHead).Number.Uint64() + reorg = lastHead.Number.Uint64() - rawdb.FindCommonAncestor(pm.chainDb, header, lastHead).Number.Uint64() } lastHead = header lastBroadcastTd = td diff --git a/les/serverpool.go b/les/serverpool.go index 2954f44594..9c86d3d2f1 100644 --- a/les/serverpool.go +++ b/les/serverpool.go @@ -601,9 +601,8 @@ func (e *discoveredEntry) Weight() int64 { t := time.Duration(mclock.Now() - e.lastDiscovered) if t <= discoverExpireStart { return 1000000000 - } else { - return int64(1000000000 * math.Exp(-float64(t-discoverExpireStart)/float64(discoverExpireConst))) } + return int64(1000000000 * math.Exp(-float64(t-discoverExpireStart)/float64(discoverExpireConst))) } // knownEntry implements wrsItem diff --git a/les/sync.go b/les/sync.go index 1e13f3f90f..08a6ddae0f 100644 --- a/les/sync.go +++ b/les/sync.go @@ -20,7 +20,7 @@ import ( "context" "time" - "github.com/EthereumCommonwealth/go-callisto/core" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/eth/downloader" "github.com/EthereumCommonwealth/go-callisto/light" ) @@ -56,7 +56,7 @@ func (pm *ProtocolManager) syncer() { func (pm *ProtocolManager) needToSync(peerHead blockInfo) bool { head := pm.blockchain.CurrentHeader() - currentTd := core.GetTd(pm.chainDb, head.Hash(), head.Number.Uint64()) + currentTd := rawdb.ReadTd(pm.chainDb, head.Hash(), head.Number.Uint64()) return currentTd != nil && peerHead.Td.Cmp(currentTd) > 0 } diff --git a/light/lightchain.go b/light/lightchain.go index e4c532aa51..0a0ef2359a 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -27,6 +27,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common" "github.com/EthereumCommonwealth/go-callisto/consensus" "github.com/EthereumCommonwealth/go-callisto/core" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/state" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/ethdb" @@ -142,7 +143,7 @@ func (self *LightChain) Odr() OdrBackend { // loadLastState loads the last known chain state from the database. This method // assumes that the chain manager mutex is held. func (self *LightChain) loadLastState() error { - if head := core.GetHeadHeaderHash(self.chainDb); head == (common.Hash{}) { + if head := rawdb.ReadHeadHeaderHash(self.chainDb); head == (common.Hash{}) { // Corrupt or empty database, init from scratch self.Reset() } else { @@ -189,12 +190,9 @@ func (bc *LightChain) ResetWithGenesisBlock(genesis *types.Block) { defer bc.mu.Unlock() // Prepare the genesis block and reinitialise the chain - if err := core.WriteTd(bc.chainDb, genesis.Hash(), genesis.NumberU64(), genesis.Difficulty()); err != nil { - log.Crit("Failed to write genesis block TD", "err", err) - } - if err := core.WriteBlock(bc.chainDb, genesis); err != nil { - log.Crit("Failed to write genesis block", "err", err) - } + rawdb.WriteTd(bc.chainDb, genesis.Hash(), genesis.NumberU64(), genesis.Difficulty()) + rawdb.WriteBlock(bc.chainDb, genesis) + bc.genesisBlock = genesis bc.hc.SetGenesis(bc.genesisBlock.Header()) bc.hc.SetCurrentHeader(bc.genesisBlock.Header()) @@ -223,7 +221,11 @@ func (self *LightChain) GetBody(ctx context.Context, hash common.Hash) (*types.B body := cached.(*types.Body) return body, nil } - body, err := GetBody(ctx, self.odr, hash, self.hc.GetBlockNumber(hash)) + number := self.hc.GetBlockNumber(hash) + if number == nil { + return nil, errors.New("unknown block") + } + body, err := GetBody(ctx, self.odr, hash, *number) if err != nil { return nil, err } @@ -239,7 +241,11 @@ func (self *LightChain) GetBodyRLP(ctx context.Context, hash common.Hash) (rlp.R if cached, ok := self.bodyRLPCache.Get(hash); ok { return cached.(rlp.RawValue), nil } - body, err := GetBodyRLP(ctx, self.odr, hash, self.hc.GetBlockNumber(hash)) + number := self.hc.GetBlockNumber(hash) + if number == nil { + return nil, errors.New("unknown block") + } + body, err := GetBodyRLP(ctx, self.odr, hash, *number) if err != nil { return nil, err } @@ -274,7 +280,11 @@ func (self *LightChain) GetBlock(ctx context.Context, hash common.Hash, number u // GetBlockByHash retrieves a block from the database or ODR service by hash, // caching it if found. func (self *LightChain) GetBlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) { - return self.GetBlock(ctx, hash, self.hc.GetBlockNumber(hash)) + number := self.hc.GetBlockNumber(hash) + if number == nil { + return nil, errors.New("unknown block") + } + return self.GetBlock(ctx, hash, *number) } // GetBlockByNumber retrieves a block from the database or ODR service by diff --git a/light/lightchain_test.go b/light/lightchain_test.go index 6eb95ed41a..b8bf467384 100644 --- a/light/lightchain_test.go +++ b/light/lightchain_test.go @@ -24,6 +24,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common" "github.com/EthereumCommonwealth/go-callisto/consensus/ethash" "github.com/EthereumCommonwealth/go-callisto/core" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/ethdb" "github.com/EthereumCommonwealth/go-callisto/params" @@ -51,7 +52,7 @@ func makeHeaderChain(parent *types.Header, n int, db ethdb.Database, seed int) [ // chain. Depending on the full flag, if creates either a full block chain or a // header only chain. func newCanonical(n int) (ethdb.Database, *LightChain, error) { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() gspec := core.Genesis{Config: params.TestChainConfig} genesis := gspec.MustCommit(db) blockchain, _ := NewLightChain(&dummyOdr{db: db}, gspec.Config, ethash.NewFaker()) @@ -68,7 +69,7 @@ func newCanonical(n int) (ethdb.Database, *LightChain, error) { // newTestLightChain creates a LightChain that doesn't validate anything. func newTestLightChain() *LightChain { - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() gspec := &core.Genesis{ Difficulty: big.NewInt(1), Config: params.TestChainConfig, @@ -122,8 +123,8 @@ func testHeaderChainImport(chain []*types.Header, lightchain *LightChain) error } // Manually insert the header into the database, but don't reorganize (allows subsequent testing) lightchain.mu.Lock() - core.WriteTd(lightchain.chainDb, header.Hash(), header.Number.Uint64(), new(big.Int).Add(header.Difficulty, lightchain.GetTdByHash(header.ParentHash))) - core.WriteHeader(lightchain.chainDb, header) + rawdb.WriteTd(lightchain.chainDb, header.Hash(), header.Number.Uint64(), new(big.Int).Add(header.Difficulty, lightchain.GetTdByHash(header.ParentHash))) + rawdb.WriteHeader(lightchain.chainDb, header) lightchain.mu.Unlock() } return nil diff --git a/light/odr.go b/light/odr.go index 2c69573e8a..189373c204 100644 --- a/light/odr.go +++ b/light/odr.go @@ -24,6 +24,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common" "github.com/EthereumCommonwealth/go-callisto/core" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/ethdb" ) @@ -112,7 +113,7 @@ type BlockRequest struct { // StoreResult stores the retrieved data in local database func (req *BlockRequest) StoreResult(db ethdb.Database) { - core.WriteBodyRLP(db, req.Hash, req.Number, req.Rlp) + rawdb.WriteBodyRLP(db, req.Hash, req.Number, req.Rlp) } // ReceiptsRequest is the ODR request type for retrieving block bodies @@ -125,7 +126,7 @@ type ReceiptsRequest struct { // StoreResult stores the retrieved data in local database func (req *ReceiptsRequest) StoreResult(db ethdb.Database) { - core.WriteBlockReceipts(db, req.Hash, req.Number, req.Receipts) + rawdb.WriteReceipts(db, req.Hash, req.Number, req.Receipts) } // ChtRequest is the ODR request type for state/storage trie entries @@ -140,11 +141,11 @@ type ChtRequest struct { // StoreResult stores the retrieved data in local database func (req *ChtRequest) StoreResult(db ethdb.Database) { - // if there is a canonical hash, there is a header too - core.WriteHeader(db, req.Header) hash, num := req.Header.Hash(), req.Header.Number.Uint64() - core.WriteTd(db, hash, num, req.Td) - core.WriteCanonicalHash(db, hash, num) + + rawdb.WriteHeader(db, req.Header) + rawdb.WriteTd(db, hash, num, req.Td) + rawdb.WriteCanonicalHash(db, hash, num) } // BloomRequest is the ODR request type for retrieving bloom filters from a CHT structure @@ -161,11 +162,11 @@ type BloomRequest struct { // StoreResult stores the retrieved data in local database func (req *BloomRequest) StoreResult(db ethdb.Database) { for i, sectionIdx := range req.SectionIdxList { - sectionHead := core.GetCanonicalHash(db, (sectionIdx+1)*BloomTrieFrequency-1) + sectionHead := rawdb.ReadCanonicalHash(db, (sectionIdx+1)*BloomTrieFrequency-1) // if we don't have the canonical hash stored for this section head number, we'll still store it under // a key with a zero sectionHead. GetBloomBits will look there too if we still don't have the canonical // hash. In the unlikely case we've retrieved the section head hash since then, we'll just retrieve the // bit vector again from the network. - core.WriteBloomBits(db, req.BitIdx, sectionIdx, sectionHead, req.BloomBits[i]) + rawdb.WriteBloomBits(db, req.BitIdx, sectionIdx, sectionHead, req.BloomBits[i]) } } diff --git a/light/odr_test.go b/light/odr_test.go index 5953bc5ae0..e4b0d673be 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -28,6 +28,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common/math" "github.com/EthereumCommonwealth/go-callisto/consensus/ethash" "github.com/EthereumCommonwealth/go-callisto/core" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/state" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/core/vm" @@ -70,9 +71,15 @@ func (odr *testOdr) Retrieve(ctx context.Context, req OdrRequest) error { } switch req := req.(type) { case *BlockRequest: - req.Rlp = core.GetBodyRLP(odr.sdb, req.Hash, core.GetBlockNumber(odr.sdb, req.Hash)) + number := rawdb.ReadHeaderNumber(odr.sdb, req.Hash) + if number != nil { + req.Rlp = rawdb.ReadBodyRLP(odr.sdb, req.Hash, *number) + } case *ReceiptsRequest: - req.Receipts = core.GetBlockReceipts(odr.sdb, req.Hash, core.GetBlockNumber(odr.sdb, req.Hash)) + number := rawdb.ReadHeaderNumber(odr.sdb, req.Hash) + if number != nil { + req.Receipts = rawdb.ReadReceipts(odr.sdb, req.Hash, *number) + } case *TrieRequest: t, _ := trie.New(req.Id.Root, trie.NewDatabase(odr.sdb)) nodes := NewNodeSet() @@ -108,9 +115,15 @@ func TestOdrGetReceiptsLes1(t *testing.T) { testChainOdr(t, 1, odrGetReceipts) } func odrGetReceipts(ctx context.Context, db ethdb.Database, bc *core.BlockChain, lc *LightChain, bhash common.Hash) ([]byte, error) { var receipts types.Receipts if bc != nil { - receipts = core.GetBlockReceipts(db, bhash, core.GetBlockNumber(db, bhash)) + number := rawdb.ReadHeaderNumber(db, bhash) + if number != nil { + receipts = rawdb.ReadReceipts(db, bhash, *number) + } } else { - receipts, _ = GetBlockReceipts(ctx, lc.Odr(), bhash, core.GetBlockNumber(db, bhash)) + number := rawdb.ReadHeaderNumber(db, bhash) + if number != nil { + receipts, _ = GetBlockReceipts(ctx, lc.Odr(), bhash, *number) + } } if receipts == nil { return nil, nil @@ -232,8 +245,8 @@ func testChainGen(i int, block *core.BlockGen) { func testChainOdr(t *testing.T, protocol int, fn odrTestFn) { var ( - sdb, _ = ethdb.NewMemDatabase() - ldb, _ = ethdb.NewMemDatabase() + sdb = ethdb.NewMemDatabase() + ldb = ethdb.NewMemDatabase() gspec = core.Genesis{Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}} genesis = gspec.MustCommit(sdb) ) @@ -260,7 +273,7 @@ func testChainOdr(t *testing.T, protocol int, fn odrTestFn) { test := func(expFail int) { for i := uint64(0); i <= blockchain.CurrentHeader().Number.Uint64(); i++ { - bhash := core.GetCanonicalHash(sdb, i) + bhash := rawdb.ReadCanonicalHash(sdb, i) b1, err := fn(NoOdr, sdb, blockchain, nil, bhash) if err != nil { t.Fatalf("error in full-node test for block %d: %v", i, err) diff --git a/light/odr_util.go b/light/odr_util.go index 79a305daaa..722ef2c57d 100644 --- a/light/odr_util.go +++ b/light/odr_util.go @@ -22,6 +22,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common" "github.com/EthereumCommonwealth/go-callisto/core" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/crypto" "github.com/EthereumCommonwealth/go-callisto/rlp" @@ -31,10 +32,10 @@ var sha3_nil = crypto.Keccak256Hash(nil) func GetHeaderByNumber(ctx context.Context, odr OdrBackend, number uint64) (*types.Header, error) { db := odr.Database() - hash := core.GetCanonicalHash(db, number) + hash := rawdb.ReadCanonicalHash(db, number) if (hash != common.Hash{}) { // if there is a canonical hash, there is a header too - header := core.GetHeader(db, hash, number) + header := rawdb.ReadHeader(db, hash, number) if header == nil { panic("Canonical hash present but header not found") } @@ -47,14 +48,14 @@ func GetHeaderByNumber(ctx context.Context, odr OdrBackend, number uint64) (*typ ) if odr.ChtIndexer() != nil { chtCount, sectionHeadNum, sectionHead = odr.ChtIndexer().Sections() - canonicalHash := core.GetCanonicalHash(db, sectionHeadNum) + canonicalHash := rawdb.ReadCanonicalHash(db, sectionHeadNum) // if the CHT was injected as a trusted checkpoint, we have no canonical hash yet so we accept zero hash too for chtCount > 0 && canonicalHash != sectionHead && canonicalHash != (common.Hash{}) { chtCount-- if chtCount > 0 { sectionHeadNum = chtCount*CHTFrequencyClient - 1 sectionHead = odr.ChtIndexer().SectionHead(chtCount - 1) - canonicalHash = core.GetCanonicalHash(db, sectionHeadNum) + canonicalHash = rawdb.ReadCanonicalHash(db, sectionHeadNum) } } } @@ -69,7 +70,7 @@ func GetHeaderByNumber(ctx context.Context, odr OdrBackend, number uint64) (*typ } func GetCanonicalHash(ctx context.Context, odr OdrBackend, number uint64) (common.Hash, error) { - hash := core.GetCanonicalHash(odr.Database(), number) + hash := rawdb.ReadCanonicalHash(odr.Database(), number) if (hash != common.Hash{}) { return hash, nil } @@ -82,7 +83,7 @@ func GetCanonicalHash(ctx context.Context, odr OdrBackend, number uint64) (commo // GetBodyRLP retrieves the block body (transactions and uncles) in RLP encoding. func GetBodyRLP(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64) (rlp.RawValue, error) { - if data := core.GetBodyRLP(odr.Database(), hash, number); data != nil { + if data := rawdb.ReadBodyRLP(odr.Database(), hash, number); data != nil { return data, nil } r := &BlockRequest{Hash: hash, Number: number} @@ -111,7 +112,7 @@ func GetBody(ctx context.Context, odr OdrBackend, hash common.Hash, number uint6 // back from the stored header and body. func GetBlock(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64) (*types.Block, error) { // Retrieve the block header and body contents - header := core.GetHeader(odr.Database(), hash, number) + header := rawdb.ReadHeader(odr.Database(), hash, number) if header == nil { return nil, ErrNoHeader } @@ -127,7 +128,7 @@ func GetBlock(ctx context.Context, odr OdrBackend, hash common.Hash, number uint // in a block given by its hash. func GetBlockReceipts(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64) (types.Receipts, error) { // Retrieve the potentially incomplete receipts from disk or network - receipts := core.GetBlockReceipts(odr.Database(), hash, number) + receipts := rawdb.ReadReceipts(odr.Database(), hash, number) if receipts == nil { r := &ReceiptsRequest{Hash: hash, Number: number} if err := odr.Retrieve(ctx, r); err != nil { @@ -141,13 +142,13 @@ func GetBlockReceipts(ctx context.Context, odr OdrBackend, hash common.Hash, num if err != nil { return nil, err } - genesis := core.GetCanonicalHash(odr.Database(), 0) - config, _ := core.GetChainConfig(odr.Database(), genesis) + genesis := rawdb.ReadCanonicalHash(odr.Database(), 0) + config := rawdb.ReadChainConfig(odr.Database(), genesis) if err := core.SetReceiptsData(config, block, receipts); err != nil { return nil, err } - core.WriteBlockReceipts(odr.Database(), hash, number, receipts) + rawdb.WriteReceipts(odr.Database(), hash, number, receipts) } return receipts, nil } @@ -156,7 +157,7 @@ func GetBlockReceipts(ctx context.Context, odr OdrBackend, hash common.Hash, num // block given by its hash. func GetBlockLogs(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64) ([][]*types.Log, error) { // Retrieve the potentially incomplete receipts from disk or network - receipts := core.GetBlockReceipts(odr.Database(), hash, number) + receipts := rawdb.ReadReceipts(odr.Database(), hash, number) if receipts == nil { r := &ReceiptsRequest{Hash: hash, Number: number} if err := odr.Retrieve(ctx, r); err != nil { @@ -187,24 +188,24 @@ func GetBloomBits(ctx context.Context, odr OdrBackend, bitIdx uint, sectionIdxLi ) if odr.BloomTrieIndexer() != nil { bloomTrieCount, sectionHeadNum, sectionHead = odr.BloomTrieIndexer().Sections() - canonicalHash := core.GetCanonicalHash(db, sectionHeadNum) + canonicalHash := rawdb.ReadCanonicalHash(db, sectionHeadNum) // if the BloomTrie was injected as a trusted checkpoint, we have no canonical hash yet so we accept zero hash too for bloomTrieCount > 0 && canonicalHash != sectionHead && canonicalHash != (common.Hash{}) { bloomTrieCount-- if bloomTrieCount > 0 { sectionHeadNum = bloomTrieCount*BloomTrieFrequency - 1 sectionHead = odr.BloomTrieIndexer().SectionHead(bloomTrieCount - 1) - canonicalHash = core.GetCanonicalHash(db, sectionHeadNum) + canonicalHash = rawdb.ReadCanonicalHash(db, sectionHeadNum) } } } for i, sectionIdx := range sectionIdxList { - sectionHead := core.GetCanonicalHash(db, (sectionIdx+1)*BloomTrieFrequency-1) + sectionHead := rawdb.ReadCanonicalHash(db, (sectionIdx+1)*BloomTrieFrequency-1) // if we don't have the canonical hash stored for this section head number, we'll still look for // an entry with a zero sectionHead (we store it with zero section head too if we don't know it // at the time of the retrieval) - bloomBits, err := core.GetBloomBits(db, bitIdx, sectionIdx, sectionHead) + bloomBits, err := rawdb.ReadBloomBits(db, bitIdx, sectionIdx, sectionHead) if err == nil { result[i] = bloomBits } else { diff --git a/light/postprocess.go b/light/postprocess.go index 4c778ae711..eeb9a9646e 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -25,6 +25,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common" "github.com/EthereumCommonwealth/go-callisto/common/bitutil" "github.com/EthereumCommonwealth/go-callisto/core" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/ethdb" "github.com/EthereumCommonwealth/go-callisto/log" @@ -58,18 +59,18 @@ type trustedCheckpoint struct { var ( mainnetCheckpoint = trustedCheckpoint{ name: "mainnet", - sectionIdx: 165, - sectionHead: common.HexToHash("21028acf9cd9ce80257221adc437c3c58ce046c4d43c21c3e9b1d1349059ec73"), - chtRoot: common.HexToHash("26b2458cb7d0080d3a39311c914be92c368777a65ec074e1893b8bdc79e3910a"), - bloomTrieRoot: common.HexToHash("5d06908769179186165a72db7fc3473b25c28ed27efe78a392a9ff2c3fa67f84"), + sectionIdx: 170, + sectionHead: common.HexToHash("3bb2c28bcce463d57968f14f56cdb3fbf35349ab7a701f44c1afb57349c9a356"), + chtRoot: common.HexToHash("d92b6d0853455f8439086292338e87f69781921680dd7aa072fb71547b87415e"), + bloomTrieRoot: common.HexToHash("e4e8250a2fefddead7ae42daecd848cbf9b66d748a8270f8bbd4370b764bb9e9"), } ropstenCheckpoint = trustedCheckpoint{ name: "ropsten", - sectionIdx: 92, - sectionHead: common.HexToHash("21a158f9cc643da13a237cafceb37381072649f7278cf98c5820bfbced7cfcec"), - chtRoot: common.HexToHash("1a8ddb8b086d7a33ca90eea90730225948fa504ae0283b15aff3c15c0e089bf9"), - bloomTrieRoot: common.HexToHash("fd192f92afbcdd0020c81ca0625116b5995509659653b10123bd986fe5129cc1"), + sectionIdx: 97, + sectionHead: common.HexToHash("719448c67c01eb5b9f27833a36a4e34612f66801316d7ff37daf9e77fb4cd095"), + chtRoot: common.HexToHash("a7857afc15930ca6e583b6c3d563a025144011655843d52d28e2fdaadd417bea"), + bloomTrieRoot: common.HexToHash("9c71d4b50cbec86dfeaa8e08992de8a4667b81d13c54d6522b17ce2fc5d36416"), } ) @@ -161,7 +162,7 @@ func (c *ChtIndexerBackend) Process(header *types.Header) { hash, num := header.Hash(), header.Number.Uint64() c.lastHash = hash - td := core.GetTd(c.diskdb, hash, num) + td := rawdb.ReadTd(c.diskdb, hash, num) if td == nil { panic(nil) } @@ -272,7 +273,7 @@ func (b *BloomTrieIndexerBackend) Commit() error { binary.BigEndian.PutUint64(encKey[2:10], b.section) var decomp []byte for j := uint64(0); j < b.bloomTrieRatio; j++ { - data, err := core.GetBloomBits(b.diskdb, i, b.section*b.bloomTrieRatio+j, b.sectionHeads[j]) + data, err := rawdb.ReadBloomBits(b.diskdb, i, b.section*b.bloomTrieRatio+j, b.sectionHeads[j]) if err != nil { return err } diff --git a/light/trie_test.go b/light/trie_test.go index dfcdd9b871..8df93db131 100644 --- a/light/trie_test.go +++ b/light/trie_test.go @@ -34,10 +34,10 @@ import ( func TestNodeIterator(t *testing.T) { var ( - fulldb, _ = ethdb.NewMemDatabase() - lightdb, _ = ethdb.NewMemDatabase() - gspec = core.Genesis{Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}} - genesis = gspec.MustCommit(fulldb) + fulldb = ethdb.NewMemDatabase() + lightdb = ethdb.NewMemDatabase() + gspec = core.Genesis{Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}} + genesis = gspec.MustCommit(fulldb) ) gspec.MustCommit(lightdb) blockchain, _ := core.NewBlockChain(fulldb, nil, params.TestChainConfig, ethash.NewFullFaker(), vm.Config{}) diff --git a/light/txpool.go b/light/txpool.go index ffad896d5e..2bf7fe7297 100644 --- a/light/txpool.go +++ b/light/txpool.go @@ -24,6 +24,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common" "github.com/EthereumCommonwealth/go-callisto/core" + "github.com/EthereumCommonwealth/go-callisto/core/rawdb" "github.com/EthereumCommonwealth/go-callisto/core/state" "github.com/EthereumCommonwealth/go-callisto/core/types" "github.com/EthereumCommonwealth/go-callisto/ethdb" @@ -183,9 +184,8 @@ func (pool *TxPool) checkMinedTxs(ctx context.Context, hash common.Hash, number if _, err := GetBlockReceipts(ctx, pool.odr, hash, number); err != nil { // ODR caches, ignore results return err } - if err := core.WriteTxLookupEntries(pool.chainDb, block); err != nil { - return err - } + rawdb.WriteTxLookupEntries(pool.chainDb, block) + // Update the transaction pool's state for _, tx := range list { delete(pool.pending, tx.Hash()) @@ -202,7 +202,7 @@ func (pool *TxPool) rollbackTxs(hash common.Hash, txc txStateChanges) { if list, ok := pool.mined[hash]; ok { for _, tx := range list { txHash := tx.Hash() - core.DeleteTxLookupEntry(pool.chainDb, txHash) + rawdb.DeleteTxLookupEntry(pool.chainDb, txHash) pool.pending[txHash] = tx txc.setState(txHash, false) } @@ -258,7 +258,7 @@ func (pool *TxPool) reorgOnNewHead(ctx context.Context, newHeader *types.Header) idx2 := idx - txPermanent if len(pool.mined) > 0 { for i := pool.clearIdx; i < idx2; i++ { - hash := core.GetCanonicalHash(pool.chainDb, i) + hash := rawdb.ReadCanonicalHash(pool.chainDb, i) if list, ok := pool.mined[hash]; ok { hashes := make([]common.Hash, len(list)) for i, tx := range list { @@ -321,9 +321,9 @@ func (pool *TxPool) Stop() { log.Info("Transaction pool stopped") } -// SubscribeTxPreEvent registers a subscription of core.TxPreEvent and +// SubscribeNewTxsEvent registers a subscription of core.NewTxsEvent and // starts sending event to the given channel. -func (pool *TxPool) SubscribeTxPreEvent(ch chan<- core.TxPreEvent) event.Subscription { +func (pool *TxPool) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subscription { return pool.scope.Track(pool.txFeed.Subscribe(ch)) } @@ -412,7 +412,7 @@ func (self *TxPool) add(ctx context.Context, tx *types.Transaction) error { // Notify the subscribers. This event is posted in a goroutine // because it's possible that somewhere during the post "Remove transaction" // gets called which will then wait for the global tx pool lock and deadlock. - go self.txFeed.Send(core.TxPreEvent{Tx: tx}) + go self.txFeed.Send(core.NewTxsEvent{Txs: types.Transactions{tx}}) } // Print a log message if low enough level is set diff --git a/light/txpool_test.go b/light/txpool_test.go index fdc5d05ca6..acef3d0fc6 100644 --- a/light/txpool_test.go +++ b/light/txpool_test.go @@ -81,8 +81,8 @@ func TestTxPool(t *testing.T) { } var ( - sdb, _ = ethdb.NewMemDatabase() - ldb, _ = ethdb.NewMemDatabase() + sdb = ethdb.NewMemDatabase() + ldb = ethdb.NewMemDatabase() gspec = core.Genesis{Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}} genesis = gspec.MustCommit(sdb) ) diff --git a/log/README.md b/log/README.md index 0951b21cb5..b4476577b6 100644 --- a/log/README.md +++ b/log/README.md @@ -45,7 +45,7 @@ srvlog.SetHandler(log.MultiHandler( log.StreamHandler(os.Stderr, log.LogfmtFormat()), log.LvlFilterHandler( log.LvlError, - log.Must.FileHandler("errors.json", log.JsonFormat())))) + log.Must.FileHandler("errors.json", log.JSONFormat())))) ``` Will result in output that looks like this: diff --git a/log/doc.go b/log/doc.go index 83ad8c54f6..bff2f4966a 100644 --- a/log/doc.go +++ b/log/doc.go @@ -86,7 +86,7 @@ from the rpc package in logfmt to standard out. The other prints records at Erro or above in JSON formatted output to the file /var/log/service.json handler := log.MultiHandler( - log.LvlFilterHandler(log.LvlError, log.Must.FileHandler("/var/log/service.json", log.JsonFormat())), + log.LvlFilterHandler(log.LvlError, log.Must.FileHandler("/var/log/service.json", log.JSONFormat())), log.MatchFilterHandler("pkg", "app/rpc" log.StdoutHandler()) ) @@ -304,8 +304,8 @@ For all Handler functions which can return an error, there is a version of that function which will return no error but panics on failure. They are all available on the Must object. For example: - log.Must.FileHandler("/path", log.JsonFormat) - log.Must.NetHandler("tcp", ":1234", log.JsonFormat) + log.Must.FileHandler("/path", log.JSONFormat) + log.Must.NetHandler("tcp", ":1234", log.JSONFormat) Inspiration and Credit diff --git a/log/format.go b/log/format.go index 7151dc12ea..36afc948ea 100644 --- a/log/format.go +++ b/log/format.go @@ -196,16 +196,16 @@ func logfmt(buf *bytes.Buffer, ctx []interface{}, color int, term bool) { buf.WriteByte('\n') } -// JsonFormat formats log records as JSON objects separated by newlines. -// It is the equivalent of JsonFormatEx(false, true). -func JsonFormat() Format { - return JsonFormatEx(false, true) +// JSONFormat formats log records as JSON objects separated by newlines. +// It is the equivalent of JSONFormatEx(false, true). +func JSONFormat() Format { + return JSONFormatEx(false, true) } -// JsonFormatEx formats log records as JSON objects. If pretty is true, +// JSONFormatEx formats log records as JSON objects. If pretty is true, // records will be pretty-printed. If lineSeparated is true, records // will be logged with a new line between each record. -func JsonFormatEx(pretty, lineSeparated bool) Format { +func JSONFormatEx(pretty, lineSeparated bool) Format { jsonMarshal := json.Marshal if pretty { jsonMarshal = func(v interface{}) ([]byte, error) { @@ -225,7 +225,7 @@ func JsonFormatEx(pretty, lineSeparated bool) Format { if !ok { props[errorKey] = fmt.Sprintf("%+v is not a string key", r.Ctx[i]) } - props[k] = formatJsonValue(r.Ctx[i+1]) + props[k] = formatJSONValue(r.Ctx[i+1]) } b, err := jsonMarshal(props) @@ -270,7 +270,7 @@ func formatShared(value interface{}) (result interface{}) { } } -func formatJsonValue(value interface{}) interface{} { +func formatJSONValue(value interface{}) interface{} { value = formatShared(value) switch value.(type) { case int, int8, int16, int32, int64, float32, float64, uint, uint8, uint16, uint32, uint64, string: diff --git a/log/handler.go b/log/handler.go index d5594b853d..3c99114dcb 100644 --- a/log/handler.go +++ b/log/handler.go @@ -11,8 +11,8 @@ import ( "github.com/go-stack/stack" ) +// Handler defines where and how log records are written. // A Logger prints its log records by writing to a Handler. -// The Handler interface defines where and how log records are written. // Handlers are composable, providing you great flexibility in combining // them to achieve the logging structure that suits your applications. type Handler interface { @@ -193,7 +193,7 @@ func LvlFilterHandler(maxLvl Lvl, h Handler) Handler { }, h) } -// A MultiHandler dispatches any write to each of its handlers. +// MultiHandler dispatches any write to each of its handlers. // This is useful for writing different types of log information // to different locations. For example, to log to a file and // standard error: @@ -212,7 +212,7 @@ func MultiHandler(hs ...Handler) Handler { }) } -// A FailoverHandler writes all log records to the first handler +// FailoverHandler writes all log records to the first handler // specified, but will failover and write to the second handler if // the first handler has failed, and so on for all handlers specified. // For example you might want to log to a network socket, but failover @@ -220,7 +220,7 @@ func MultiHandler(hs ...Handler) Handler { // standard out if the file write fails: // // log.FailoverHandler( -// log.Must.NetHandler("tcp", ":9090", log.JsonFormat()), +// log.Must.NetHandler("tcp", ":9090", log.JSONFormat()), // log.Must.FileHandler("/var/log/app.log", log.LogfmtFormat()), // log.StdoutHandler) // @@ -234,9 +234,8 @@ func FailoverHandler(hs ...Handler) Handler { err = h.Log(r) if err == nil { return nil - } else { - r.Ctx = append(r.Ctx, fmt.Sprintf("failover_err_%d", i), err) } + r.Ctx = append(r.Ctx, fmt.Sprintf("failover_err_%d", i), err) } return err @@ -320,13 +319,12 @@ func evaluateLazy(lz Lazy) (interface{}, error) { results := value.Call([]reflect.Value{}) if len(results) == 1 { return results[0].Interface(), nil - } else { - values := make([]interface{}, len(results)) - for i, v := range results { - values[i] = v.Interface() - } - return values, nil } + values := make([]interface{}, len(results)) + for i, v := range results { + values[i] = v.Interface() + } + return values, nil } // DiscardHandler reports success for all writes but does nothing. @@ -338,7 +336,7 @@ func DiscardHandler() Handler { }) } -// The Must object provides the following Handler creation functions +// Must provides the following Handler creation functions // which instead of returning an error parameter only return a Handler // and panic on failure: FileHandler, NetHandler, SyslogHandler, SyslogNetHandler var Must muster diff --git a/log/logger.go b/log/logger.go index 15c83a9b25..a2fe6dc580 100644 --- a/log/logger.go +++ b/log/logger.go @@ -24,7 +24,7 @@ const ( LvlTrace ) -// Aligned returns a 5-character string containing the name of a Lvl. +// AlignedString returns a 5-character string containing the name of a Lvl. func (l Lvl) AlignedString() string { switch l { case LvlTrace: @@ -64,7 +64,7 @@ func (l Lvl) String() string { } } -// Returns the appropriate Lvl from a string name. +// LvlFromString returns the appropriate Lvl from a string name. // Useful for parsing command line args and configuration files. func LvlFromString(lvlString string) (Lvl, error) { switch lvlString { @@ -95,6 +95,7 @@ type Record struct { KeyNames RecordKeyNames } +// RecordKeyNames gets stored in a Record when the write function is executed. type RecordKeyNames struct { Time string Msg string diff --git a/metrics/librato/client.go b/metrics/librato/client.go index 8c0c850e38..1f8920cb1a 100644 --- a/metrics/librato/client.go +++ b/metrics/librato/client.go @@ -65,7 +65,7 @@ type Batch struct { Source string `json:"source"` } -func (self *LibratoClient) PostMetrics(batch Batch) (err error) { +func (c *LibratoClient) PostMetrics(batch Batch) (err error) { var ( js []byte req *http.Request @@ -85,7 +85,7 @@ func (self *LibratoClient) PostMetrics(batch Batch) (err error) { } req.Header.Set("Content-Type", "application/json") - req.SetBasicAuth(self.Email, self.Token) + req.SetBasicAuth(c.Email, c.Token) if resp, err = http.DefaultClient.Do(req); err != nil { return diff --git a/metrics/librato/librato.go b/metrics/librato/librato.go index e7ef254fd1..c9ad4068f7 100644 --- a/metrics/librato/librato.go +++ b/metrics/librato/librato.go @@ -40,14 +40,14 @@ func Librato(r metrics.Registry, d time.Duration, e string, t string, s string, NewReporter(r, d, e, t, s, p, u).Run() } -func (self *Reporter) Run() { +func (rep *Reporter) Run() { log.Printf("WARNING: This client has been DEPRECATED! It has been moved to https://github.com/mihasya/go-metrics-librato and will be removed from rcrowley/go-metrics on August 5th 2015") - ticker := time.Tick(self.Interval) - metricsApi := &LibratoClient{self.Email, self.Token} + ticker := time.Tick(rep.Interval) + metricsApi := &LibratoClient{rep.Email, rep.Token} for now := range ticker { var metrics Batch var err error - if metrics, err = self.BuildRequest(now, self.Registry); err != nil { + if metrics, err = rep.BuildRequest(now, rep.Registry); err != nil { log.Printf("ERROR constructing librato request body %s", err) continue } @@ -79,21 +79,21 @@ func sumSquaresTimer(t metrics.Timer) float64 { return sumSquares } -func (self *Reporter) BuildRequest(now time.Time, r metrics.Registry) (snapshot Batch, err error) { +func (rep *Reporter) BuildRequest(now time.Time, r metrics.Registry) (snapshot Batch, err error) { snapshot = Batch{ // coerce timestamps to a stepping fn so that they line up in Librato graphs - MeasureTime: (now.Unix() / self.intervalSec) * self.intervalSec, - Source: self.Source, + MeasureTime: (now.Unix() / rep.intervalSec) * rep.intervalSec, + Source: rep.Source, } snapshot.Gauges = make([]Measurement, 0) snapshot.Counters = make([]Measurement, 0) - histogramGaugeCount := 1 + len(self.Percentiles) + histogramGaugeCount := 1 + len(rep.Percentiles) r.Each(func(name string, metric interface{}) { - if self.Namespace != "" { - name = fmt.Sprintf("%s.%s", self.Namespace, name) + if rep.Namespace != "" { + name = fmt.Sprintf("%s.%s", rep.Namespace, name) } measurement := Measurement{} - measurement[Period] = self.Interval.Seconds() + measurement[Period] = rep.Interval.Seconds() switch m := metric.(type) { case metrics.Counter: if m.Count() > 0 { @@ -125,7 +125,7 @@ func (self *Reporter) BuildRequest(now time.Time, r metrics.Registry) (snapshot measurement[Sum] = float64(s.Sum()) measurement[SumSquares] = sumSquares(s) gauges[0] = measurement - for i, p := range self.Percentiles { + for i, p := range rep.Percentiles { gauges[i+1] = Measurement{ Name: fmt.Sprintf("%s.%.2f", measurement[Name], p), Value: s.Percentile(p), @@ -142,7 +142,7 @@ func (self *Reporter) BuildRequest(now time.Time, r metrics.Registry) (snapshot Measurement{ Name: fmt.Sprintf("%s.%s", name, "1min"), Value: m.Rate1(), - Period: int64(self.Interval.Seconds()), + Period: int64(rep.Interval.Seconds()), Attributes: map[string]interface{}{ DisplayUnitsLong: Operations, DisplayUnitsShort: OperationsShort, @@ -152,7 +152,7 @@ func (self *Reporter) BuildRequest(now time.Time, r metrics.Registry) (snapshot Measurement{ Name: fmt.Sprintf("%s.%s", name, "5min"), Value: m.Rate5(), - Period: int64(self.Interval.Seconds()), + Period: int64(rep.Interval.Seconds()), Attributes: map[string]interface{}{ DisplayUnitsLong: Operations, DisplayUnitsShort: OperationsShort, @@ -162,7 +162,7 @@ func (self *Reporter) BuildRequest(now time.Time, r metrics.Registry) (snapshot Measurement{ Name: fmt.Sprintf("%s.%s", name, "15min"), Value: m.Rate15(), - Period: int64(self.Interval.Seconds()), + Period: int64(rep.Interval.Seconds()), Attributes: map[string]interface{}{ DisplayUnitsLong: Operations, DisplayUnitsShort: OperationsShort, @@ -184,15 +184,15 @@ func (self *Reporter) BuildRequest(now time.Time, r metrics.Registry) (snapshot Max: float64(m.Max()), Min: float64(m.Min()), SumSquares: sumSquaresTimer(m), - Period: int64(self.Interval.Seconds()), - Attributes: self.TimerAttributes, + Period: int64(rep.Interval.Seconds()), + Attributes: rep.TimerAttributes, } - for i, p := range self.Percentiles { + for i, p := range rep.Percentiles { gauges[i+1] = Measurement{ Name: fmt.Sprintf("%s.timer.%2.0f", name, p*100), Value: m.Percentile(p), - Period: int64(self.Interval.Seconds()), - Attributes: self.TimerAttributes, + Period: int64(rep.Interval.Seconds()), + Attributes: rep.TimerAttributes, } } snapshot.Gauges = append(snapshot.Gauges, gauges...) @@ -200,7 +200,7 @@ func (self *Reporter) BuildRequest(now time.Time, r metrics.Registry) (snapshot Measurement{ Name: fmt.Sprintf("%s.%s", name, "rate.1min"), Value: m.Rate1(), - Period: int64(self.Interval.Seconds()), + Period: int64(rep.Interval.Seconds()), Attributes: map[string]interface{}{ DisplayUnitsLong: Operations, DisplayUnitsShort: OperationsShort, @@ -210,7 +210,7 @@ func (self *Reporter) BuildRequest(now time.Time, r metrics.Registry) (snapshot Measurement{ Name: fmt.Sprintf("%s.%s", name, "rate.5min"), Value: m.Rate5(), - Period: int64(self.Interval.Seconds()), + Period: int64(rep.Interval.Seconds()), Attributes: map[string]interface{}{ DisplayUnitsLong: Operations, DisplayUnitsShort: OperationsShort, @@ -220,7 +220,7 @@ func (self *Reporter) BuildRequest(now time.Time, r metrics.Registry) (snapshot Measurement{ Name: fmt.Sprintf("%s.%s", name, "rate.15min"), Value: m.Rate15(), - Period: int64(self.Interval.Seconds()), + Period: int64(rep.Interval.Seconds()), Attributes: map[string]interface{}{ DisplayUnitsLong: Operations, DisplayUnitsShort: OperationsShort, diff --git a/miner/worker.go b/miner/worker.go index 4a68936a7f..5b5290b0ba 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -42,7 +42,7 @@ const ( resultQueueSize = 10 miningLogAtDepth = 5 - // txChanSize is the size of channel listening to TxPreEvent. + // txChanSize is the size of channel listening to NewTxsEvent. // The number is referenced from the size of tx pool. txChanSize = 4096 // chainHeadChanSize is the size of channel listening to ChainHeadEvent. @@ -71,6 +71,7 @@ type Work struct { family *set.Set // family set (used for checking uncle invalidity) uncles *set.Set // uncle set tcount int // tx count in cycle + gasPool *core.GasPool // available gas used to pack transactions Block *types.Block // the new block @@ -95,8 +96,8 @@ type worker struct { // update loop mux *event.TypeMux - txCh chan core.TxPreEvent - txSub event.Subscription + txsCh chan core.NewTxsEvent + txsSub event.Subscription chainHeadCh chan core.ChainHeadEvent chainHeadSub event.Subscription chainSideCh chan core.ChainSideEvent @@ -137,7 +138,7 @@ func newWorker(config *params.ChainConfig, engine consensus.Engine, coinbase com engine: engine, eth: eth, mux: mux, - txCh: make(chan core.TxPreEvent, txChanSize), + txsCh: make(chan core.NewTxsEvent, txChanSize), chainHeadCh: make(chan core.ChainHeadEvent, chainHeadChanSize), chainSideCh: make(chan core.ChainSideEvent, chainSideChanSize), chainDb: eth.ChainDb(), @@ -149,8 +150,8 @@ func newWorker(config *params.ChainConfig, engine consensus.Engine, coinbase com agents: make(map[Agent]struct{}), unconfirmed: newUnconfirmedBlocks(eth.BlockChain(), miningLogAtDepth), } - // Subscribe TxPreEvent for tx pool - worker.txSub = eth.TxPool().SubscribeTxPreEvent(worker.txCh) + // Subscribe NewTxsEvent for tx pool + worker.txsSub = eth.TxPool().SubscribeNewTxsEvent(worker.txsCh) // Subscribe events for blockchain worker.chainHeadSub = eth.BlockChain().SubscribeChainHeadEvent(worker.chainHeadCh) worker.chainSideSub = eth.BlockChain().SubscribeChainSideEvent(worker.chainSideCh) @@ -241,7 +242,7 @@ func (self *worker) unregister(agent Agent) { } func (self *worker) update() { - defer self.txSub.Unsubscribe() + defer self.txsSub.Unsubscribe() defer self.chainHeadSub.Unsubscribe() defer self.chainSideSub.Unsubscribe() @@ -258,15 +259,21 @@ func (self *worker) update() { self.possibleUncles[ev.Block.Hash()] = ev.Block self.uncleMu.Unlock() - // Handle TxPreEvent - case ev := <-self.txCh: - // Apply transaction to the pending state if we're not mining + // Handle NewTxsEvent + case ev := <-self.txsCh: + // Apply transactions to the pending state if we're not mining. + // + // Note all transactions received may not be continuous with transactions + // already included in the current mining block. These transactions will + // be automatically eliminated. if atomic.LoadInt32(&self.mining) == 0 { self.currentMu.Lock() - acc, _ := types.Sender(self.current.signer, ev.Tx) - txs := map[common.Address]types.Transactions{acc: {ev.Tx}} + txs := make(map[common.Address]types.Transactions) + for _, tx := range ev.Txs { + acc, _ := types.Sender(self.current.signer, tx) + txs[acc] = append(txs[acc], tx) + } txset := types.NewTransactionsByPriceAndNonce(self.current.signer, txs) - self.current.commitTransactions(self.mux, txset, self.chain, self.coinbase) self.updateSnapshot() self.currentMu.Unlock() @@ -278,7 +285,7 @@ func (self *worker) update() { } // System stopped - case <-self.txSub.Err(): + case <-self.txsSub.Err(): return case <-self.chainHeadSub.Err(): return @@ -522,14 +529,16 @@ func (self *worker) updateSnapshot() { } func (env *Work) commitTransactions(mux *event.TypeMux, txs *types.TransactionsByPriceAndNonce, bc *core.BlockChain, coinbase common.Address) { - gp := new(core.GasPool).AddGas(env.header.GasLimit) + if env.gasPool == nil { + env.gasPool = new(core.GasPool).AddGas(env.header.GasLimit) + } var coalescedLogs []*types.Log for { // If we don't have enough gas for any further transactions then we're done - if gp.Gas() < params.TxGas { - log.Trace("Not enough gas for further transactions", "gp", gp) + if env.gasPool.Gas() < params.TxGas { + log.Trace("Not enough gas for further transactions", "have", env.gasPool, "want", params.TxGas) break } // Retrieve the next transaction and abort if all done @@ -553,7 +562,7 @@ func (env *Work) commitTransactions(mux *event.TypeMux, txs *types.TransactionsB // Start executing the transaction env.state.Prepare(tx.Hash(), common.Hash{}, env.tcount) - err, logs := env.commitTransaction(tx, bc, coinbase, gp) + err, logs := env.commitTransaction(tx, bc, coinbase, env.gasPool) switch err { case core.ErrGasLimitReached: // Pop the current out-of-gas transaction without shifting in the next from the account diff --git a/mobile/geth.go b/mobile/geth.go index 75b4847a2a..daad0eecc6 100644 --- a/mobile/geth.go +++ b/mobile/geth.go @@ -29,6 +29,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/eth/downloader" "github.com/EthereumCommonwealth/go-callisto/ethclient" "github.com/EthereumCommonwealth/go-callisto/ethstats" + "github.com/EthereumCommonwealth/go-callisto/internal/debug" "github.com/EthereumCommonwealth/go-callisto/les" "github.com/EthereumCommonwealth/go-callisto/node" "github.com/EthereumCommonwealth/go-callisto/p2p" @@ -72,6 +73,9 @@ type NodeConfig struct { // WhisperEnabled specifies whether the node should run the Whisper protocol. WhisperEnabled bool + + // Listening address of pprof server. + PprofAddress string } // defaultNodeConfig contains the default node configuration values to use if all @@ -107,6 +111,11 @@ func NewNode(datadir string, config *NodeConfig) (stack *Node, _ error) { if config.BootstrapNodes == nil || config.BootstrapNodes.Size() == 0 { config.BootstrapNodes = defaultNodeConfig.BootstrapNodes } + + if config.PprofAddress != "" { + debug.StartPProf(config.PprofAddress) + } + // Create the empty networking stack nodeConf := &node.Config{ Name: clientIdentifier, @@ -127,6 +136,8 @@ func NewNode(datadir string, config *NodeConfig) (stack *Node, _ error) { return nil, err } + debug.Memsize.Add("node", rawStack) + var genesis *core.Genesis if config.EthereumGenesis != "" { // Parse the user supplied genesis spec if not mainnet diff --git a/mobile/types.go b/mobile/types.go index f571f61048..3ec45602ee 100644 --- a/mobile/types.go +++ b/mobile/types.go @@ -329,6 +329,7 @@ func (r *Receipt) EncodeJSON() (string, error) { return string(data), err } +func (r *Receipt) GetStatus() int { return int(r.receipt.Status) } func (r *Receipt) GetPostState() []byte { return r.receipt.PostState } func (r *Receipt) GetCumulativeGasUsed() int64 { return int64(r.receipt.CumulativeGasUsed) } func (r *Receipt) GetBloom() *Bloom { return &Bloom{r.receipt.Bloom} } diff --git a/node/api.go b/node/api.go index b4dd862ae6..175654bdc2 100644 --- a/node/api.go +++ b/node/api.go @@ -217,7 +217,7 @@ func (api *PrivateAdminAPI) StartWS(host *string, port *int, allowedOrigins *str return true, nil } -// StopRPC terminates an already running websocket RPC API endpoint. +// StopWS terminates an already running websocket RPC API endpoint. func (api *PrivateAdminAPI) StopWS() (bool, error) { api.node.lock.Lock() defer api.node.lock.Unlock() diff --git a/node/doc.go b/node/doc.go index d9688e0a12..41a88c19a1 100644 --- a/node/doc.go +++ b/node/doc.go @@ -59,7 +59,7 @@ using the same data directory will store this information in different subdirect the data directory. LevelDB databases are also stored within the instance subdirectory. If multiple node -instances use the same data directory, openening the databases with identical names will +instances use the same data directory, opening the databases with identical names will create one database for each instance. The account key store is shared among all node instances using the same data directory @@ -84,7 +84,7 @@ directory. Mode instance A opens the database "db", node instance B opens the da static-nodes.json -- devp2p static node list of instance B db/ -- LevelDB content for "db" db-2/ -- LevelDB content for "db-2" - B.ipc -- JSON-RPC UNIX domain socket endpoint of instance A + B.ipc -- JSON-RPC UNIX domain socket endpoint of instance B keystore/ -- account key store, used by both instances */ package node diff --git a/node/node.go b/node/node.go index f7c212c3f7..5a4ac768c2 100644 --- a/node/node.go +++ b/node/node.go @@ -568,7 +568,7 @@ func (n *Node) EventMux() *event.TypeMux { // ephemeral, a memory database is returned. func (n *Node) OpenDatabase(name string, cache, handles int) (ethdb.Database, error) { if n.config.DataDir == "" { - return ethdb.NewMemDatabase() + return ethdb.NewMemDatabase(), nil } return ethdb.NewLDBDatabase(n.config.resolvePath(name), cache, handles) } diff --git a/node/node_test.go b/node/node_test.go index 588e149ef2..75ca9264dc 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -507,8 +507,8 @@ func TestAPIGather(t *testing.T) { } // Register a batch of services with some configured APIs calls := make(chan string, 1) - makeAPI := func(result string) *OneMethodApi { - return &OneMethodApi{fun: func() { calls <- result }} + makeAPI := func(result string) *OneMethodAPI { + return &OneMethodAPI{fun: func() { calls <- result }} } services := map[string]struct { APIs []rpc.API diff --git a/node/service.go b/node/service.go index e62a9b0d0e..ccef8b337c 100644 --- a/node/service.go +++ b/node/service.go @@ -41,7 +41,7 @@ type ServiceContext struct { // node is an ephemeral one, a memory database is returned. func (ctx *ServiceContext) OpenDatabase(name string, cache int, handles int) (ethdb.Database, error) { if ctx.config.DataDir == "" { - return ethdb.NewMemDatabase() + return ethdb.NewMemDatabase(), nil } db, err := ethdb.NewLDBDatabase(ctx.config.resolvePath(name), cache, handles) if err != nil { diff --git a/node/utils_test.go b/node/utils_test.go index 4120484924..f019e8dca7 100644 --- a/node/utils_test.go +++ b/node/utils_test.go @@ -121,12 +121,12 @@ func InstrumentedServiceMakerC(base ServiceConstructor) ServiceConstructor { return InstrumentingWrapperMaker(base, reflect.TypeOf(InstrumentedServiceC{})) } -// OneMethodApi is a single-method API handler to be returned by test services. -type OneMethodApi struct { +// OneMethodAPI is a single-method API handler to be returned by test services. +type OneMethodAPI struct { fun func() } -func (api *OneMethodApi) TheOneMethod() { +func (api *OneMethodAPI) TheOneMethod() { if api.fun != nil { api.fun() } diff --git a/p2p/discover/table_test.go b/p2p/discover/table_test.go index c8807a0bc1..d3d6aeec64 100644 --- a/p2p/discover/table_test.go +++ b/p2p/discover/table_test.go @@ -582,26 +582,26 @@ func (*preminedTestnet) ping(toid NodeID, toaddr *net.UDPAddr) error { return ni // mine generates a testnet struct literal with nodes at // various distances to the given target. -func (n *preminedTestnet) mine(target NodeID) { - n.target = target - n.targetSha = crypto.Keccak256Hash(n.target[:]) +func (tn *preminedTestnet) mine(target NodeID) { + tn.target = target + tn.targetSha = crypto.Keccak256Hash(tn.target[:]) found := 0 for found < bucketSize*10 { k := newkey() id := PubkeyID(&k.PublicKey) sha := crypto.Keccak256Hash(id[:]) - ld := logdist(n.targetSha, sha) - if len(n.dists[ld]) < bucketSize { - n.dists[ld] = append(n.dists[ld], id) + ld := logdist(tn.targetSha, sha) + if len(tn.dists[ld]) < bucketSize { + tn.dists[ld] = append(tn.dists[ld], id) fmt.Println("found ID with ld", ld) found++ } } fmt.Println("&preminedTestnet{") - fmt.Printf(" target: %#v,\n", n.target) - fmt.Printf(" targetSha: %#v,\n", n.targetSha) - fmt.Printf(" dists: [%d][]NodeID{\n", len(n.dists)) - for ld, ns := range n.dists { + fmt.Printf(" target: %#v,\n", tn.target) + fmt.Printf(" targetSha: %#v,\n", tn.targetSha) + fmt.Printf(" dists: [%d][]NodeID{\n", len(tn.dists)) + for ld, ns := range tn.dists { if len(ns) == 0 { continue } diff --git a/p2p/discv5/net_test.go b/p2p/discv5/net_test.go index b74d259a47..9bdd9ad149 100644 --- a/p2p/discv5/net_test.go +++ b/p2p/discv5/net_test.go @@ -336,26 +336,26 @@ func (*preminedTestnet) localAddr() *net.UDPAddr { // mine generates a testnet struct literal with nodes at // various distances to the given target. -func (n *preminedTestnet) mine(target NodeID) { - n.target = target - n.targetSha = crypto.Keccak256Hash(n.target[:]) +func (tn *preminedTestnet) mine(target NodeID) { + tn.target = target + tn.targetSha = crypto.Keccak256Hash(tn.target[:]) found := 0 for found < bucketSize*10 { k := newkey() id := PubkeyID(&k.PublicKey) sha := crypto.Keccak256Hash(id[:]) - ld := logdist(n.targetSha, sha) - if len(n.dists[ld]) < bucketSize { - n.dists[ld] = append(n.dists[ld], id) + ld := logdist(tn.targetSha, sha) + if len(tn.dists[ld]) < bucketSize { + tn.dists[ld] = append(tn.dists[ld], id) fmt.Println("found ID with ld", ld) found++ } } fmt.Println("&preminedTestnet{") - fmt.Printf(" target: %#v,\n", n.target) - fmt.Printf(" targetSha: %#v,\n", n.targetSha) - fmt.Printf(" dists: [%d][]NodeID{\n", len(n.dists)) - for ld, ns := range n.dists { + fmt.Printf(" target: %#v,\n", tn.target) + fmt.Printf(" targetSha: %#v,\n", tn.targetSha) + fmt.Printf(" dists: [%d][]NodeID{\n", len(tn.dists)) + for ld, ns := range tn.dists { if len(ns) == 0 { continue } diff --git a/p2p/discv5/node.go b/p2p/discv5/node.go index 09d2ade3f7..ae433f4c8f 100644 --- a/p2p/discv5/node.go +++ b/p2p/discv5/node.go @@ -315,11 +315,11 @@ func PubkeyID(pub *ecdsa.PublicKey) NodeID { // Pubkey returns the public key represented by the node ID. // It returns an error if the ID is not a point on the curve. -func (id NodeID) Pubkey() (*ecdsa.PublicKey, error) { +func (n NodeID) Pubkey() (*ecdsa.PublicKey, error) { p := &ecdsa.PublicKey{Curve: crypto.S256(), X: new(big.Int), Y: new(big.Int)} - half := len(id) / 2 - p.X.SetBytes(id[:half]) - p.Y.SetBytes(id[half:]) + half := len(n) / 2 + p.X.SetBytes(n[:half]) + p.Y.SetBytes(n[half:]) if !p.Curve.IsOnCurve(p.X, p.Y) { return nil, errors.New("id is invalid secp256k1 curve point") } diff --git a/p2p/discv5/ticket.go b/p2p/discv5/ticket.go index ccadfd3a0b..09055d13bb 100644 --- a/p2p/discv5/ticket.go +++ b/p2p/discv5/ticket.go @@ -304,8 +304,8 @@ func (s ticketRefByWaitTime) Len() int { return len(s) } -func (r ticketRef) waitTime() mclock.AbsTime { - return r.t.regTime[r.idx] - r.t.issueTime +func (ref ticketRef) waitTime() mclock.AbsTime { + return ref.t.regTime[ref.idx] - ref.t.issueTime } // Less reports whether the element with diff --git a/p2p/discv5/topic.go b/p2p/discv5/topic.go index adb8395797..494116cc9a 100644 --- a/p2p/discv5/topic.go +++ b/p2p/discv5/topic.go @@ -271,15 +271,15 @@ func (t *topicTable) useTicket(node *Node, serialNo uint32, topics []Topic, idx return false } -func (topictab *topicTable) getTicket(node *Node, topics []Topic) *ticket { - topictab.collectGarbage() +func (t *topicTable) getTicket(node *Node, topics []Topic) *ticket { + t.collectGarbage() now := mclock.Now() - n := topictab.getOrNewNode(node) + n := t.getOrNewNode(node) n.lastIssuedTicket++ - topictab.storeTicketCounters(node) + t.storeTicketCounters(node) - t := &ticket{ + tic := &ticket{ issueTime: now, topics: topics, serial: n.lastIssuedTicket, @@ -287,15 +287,15 @@ func (topictab *topicTable) getTicket(node *Node, topics []Topic) *ticket { } for i, topic := range topics { var waitPeriod time.Duration - if topic := topictab.topics[topic]; topic != nil { + if topic := t.topics[topic]; topic != nil { waitPeriod = topic.wcl.waitPeriod } else { waitPeriod = minWaitPeriod } - t.regTime[i] = now + mclock.AbsTime(waitPeriod) + tic.regTime[i] = now + mclock.AbsTime(waitPeriod) } - return t + return tic } const gcInterval = time.Minute diff --git a/p2p/enr/enr.go b/p2p/enr/enr.go index b284b67ae2..f191547a24 100644 --- a/p2p/enr/enr.go +++ b/p2p/enr/enr.go @@ -29,21 +29,16 @@ package enr import ( "bytes" - "crypto/ecdsa" "errors" "fmt" "io" "sort" - "github.com/EthereumCommonwealth/go-callisto/crypto" - "github.com/EthereumCommonwealth/go-callisto/crypto/sha3" "github.com/EthereumCommonwealth/go-callisto/rlp" ) const SizeLimit = 300 // maximum encoded size of a node record in bytes -const ID_SECP256k1_KECCAK = ID("secp256k1-keccak") // the default identity scheme - var ( errNoID = errors.New("unknown or unspecified identity scheme") errInvalidSig = errors.New("invalid signature") @@ -80,8 +75,8 @@ func (r *Record) Seq() uint64 { } // SetSeq updates the record sequence number. This invalidates any signature on the record. -// Calling SetSeq is usually not required because signing the redord increments the -// sequence number. +// Calling SetSeq is usually not required because setting any key in a signed record +// increments the sequence number. func (r *Record) SetSeq(s uint64) { r.signature = nil r.raw = nil @@ -104,33 +99,42 @@ func (r *Record) Load(e Entry) error { return &KeyError{Key: e.ENRKey(), Err: errNotFound} } -// Set adds or updates the given entry in the record. -// It panics if the value can't be encoded. +// Set adds or updates the given entry in the record. It panics if the value can't be +// encoded. If the record is signed, Set increments the sequence number and invalidates +// the sequence number. func (r *Record) Set(e Entry) { - r.signature = nil - r.raw = nil blob, err := rlp.EncodeToBytes(e) if err != nil { panic(fmt.Errorf("enr: can't encode %s: %v", e.ENRKey(), err)) } + r.invalidate() - i := sort.Search(len(r.pairs), func(i int) bool { return r.pairs[i].k >= e.ENRKey() }) - - if i < len(r.pairs) && r.pairs[i].k == e.ENRKey() { + pairs := make([]pair, len(r.pairs)) + copy(pairs, r.pairs) + i := sort.Search(len(pairs), func(i int) bool { return pairs[i].k >= e.ENRKey() }) + switch { + case i < len(pairs) && pairs[i].k == e.ENRKey(): // element is present at r.pairs[i] - r.pairs[i].v = blob - return - } else if i < len(r.pairs) { + pairs[i].v = blob + case i < len(r.pairs): // insert pair before i-th elem el := pair{e.ENRKey(), blob} - r.pairs = append(r.pairs, pair{}) - copy(r.pairs[i+1:], r.pairs[i:]) - r.pairs[i] = el - return + pairs = append(pairs, pair{}) + copy(pairs[i+1:], pairs[i:]) + pairs[i] = el + default: + // element should be placed at the end of r.pairs + pairs = append(pairs, pair{e.ENRKey(), blob}) } + r.pairs = pairs +} - // element should be placed at the end of r.pairs - r.pairs = append(r.pairs, pair{e.ENRKey(), blob}) +func (r *Record) invalidate() { + if r.signature == nil { + r.seq++ + } + r.signature = nil + r.raw = nil } // EncodeRLP implements rlp.Encoder. Encoding fails if @@ -196,39 +200,55 @@ func (r *Record) DecodeRLP(s *rlp.Stream) error { return err } - // Verify signature. - if err = dec.verifySignature(); err != nil { + _, scheme := dec.idScheme() + if scheme == nil { + return errNoID + } + if err := scheme.Verify(&dec, dec.signature); err != nil { return err } *r = dec return nil } -type s256raw []byte - -func (s256raw) ENRKey() string { return "secp256k1" } - // NodeAddr returns the node address. The return value will be nil if the record is -// unsigned. +// unsigned or uses an unknown identity scheme. func (r *Record) NodeAddr() []byte { - var entry s256raw - if r.Load(&entry) != nil { + _, scheme := r.idScheme() + if scheme == nil { return nil } - return crypto.Keccak256(entry) + return scheme.NodeAddr(r) } -// Sign signs the record with the given private key. It updates the record's identity -// scheme, public key and increments the sequence number. Sign returns an error if the -// encoded record is larger than the size limit. -func (r *Record) Sign(privkey *ecdsa.PrivateKey) error { - r.seq = r.seq + 1 - r.Set(ID_SECP256k1_KECCAK) - r.Set(Secp256k1(privkey.PublicKey)) - return r.signAndEncode(privkey) +// SetSig sets the record signature. It returns an error if the encoded record is larger +// than the size limit or if the signature is invalid according to the passed scheme. +func (r *Record) SetSig(idscheme string, sig []byte) error { + // Check that "id" is set and matches the given scheme. This panics because + // inconsitencies here are always implementation bugs in the signing function calling + // this method. + id, s := r.idScheme() + if s == nil { + panic(errNoID) + } + if id != idscheme { + panic(fmt.Errorf("identity scheme mismatch in Sign: record has %s, want %s", id, idscheme)) + } + + // Verify against the scheme. + if err := s.Verify(r, sig); err != nil { + return err + } + raw, err := r.encode(sig) + if err != nil { + return err + } + r.signature, r.raw = sig, raw + return nil } -func (r *Record) appendPairs(list []interface{}) []interface{} { +// AppendElements appends the sequence number and entries to the given slice. +func (r *Record) AppendElements(list []interface{}) []interface{} { list = append(list, r.seq) for _, p := range r.pairs { list = append(list, p.k, p.v) @@ -236,54 +256,23 @@ func (r *Record) appendPairs(list []interface{}) []interface{} { return list } -func (r *Record) signAndEncode(privkey *ecdsa.PrivateKey) error { - // Put record elements into a flat list. Leave room for the signature. - list := make([]interface{}, 1, len(r.pairs)*2+2) - list = r.appendPairs(list) - - // Sign the tail of the list. - h := sha3.NewKeccak256() - rlp.Encode(h, list[1:]) - sig, err := crypto.Sign(h.Sum(nil), privkey) - if err != nil { - return err +func (r *Record) encode(sig []byte) (raw []byte, err error) { + list := make([]interface{}, 1, 2*len(r.pairs)+1) + list[0] = sig + list = r.AppendElements(list) + if raw, err = rlp.EncodeToBytes(list); err != nil { + return nil, err } - sig = sig[:len(sig)-1] // remove v - - // Put signature in front. - r.signature, list[0] = sig, sig - r.raw, err = rlp.EncodeToBytes(list) - if err != nil { - return err + if len(raw) > SizeLimit { + return nil, errTooBig } - if len(r.raw) > SizeLimit { - return errTooBig - } - return nil + return raw, nil } -func (r *Record) verifySignature() error { - // Get identity scheme, public key, signature. +func (r *Record) idScheme() (string, IdentityScheme) { var id ID - var entry s256raw if err := r.Load(&id); err != nil { - return err - } else if id != ID_SECP256k1_KECCAK { - return errNoID + return "", nil } - if err := r.Load(&entry); err != nil { - return err - } else if len(entry) != 33 { - return fmt.Errorf("invalid public key") - } - - // Verify the signature. - list := make([]interface{}, 0, len(r.pairs)*2+1) - list = r.appendPairs(list) - h := sha3.NewKeccak256() - rlp.Encode(h, list) - if !crypto.VerifySignature(entry, h.Sum(nil), r.signature) { - return errInvalidSig - } - return nil + return string(id), FindIdentityScheme(string(id)) } diff --git a/p2p/enr/enr_test.go b/p2p/enr/enr_test.go index 040f90597a..f66b7eb054 100644 --- a/p2p/enr/enr_test.go +++ b/p2p/enr/enr_test.go @@ -54,35 +54,35 @@ func TestGetSetID(t *testing.T) { assert.Equal(t, id, id2) } -// TestGetSetIP4 tests encoding/decoding and setting/getting of the IP4 key. +// TestGetSetIP4 tests encoding/decoding and setting/getting of the IP key. func TestGetSetIP4(t *testing.T) { - ip := IP4{192, 168, 0, 3} + ip := IP{192, 168, 0, 3} var r Record r.Set(ip) - var ip2 IP4 + var ip2 IP require.NoError(t, r.Load(&ip2)) assert.Equal(t, ip, ip2) } -// TestGetSetIP6 tests encoding/decoding and setting/getting of the IP6 key. +// TestGetSetIP6 tests encoding/decoding and setting/getting of the IP key. func TestGetSetIP6(t *testing.T) { - ip := IP6{0x20, 0x01, 0x48, 0x60, 0, 0, 0x20, 0x01, 0, 0, 0, 0, 0, 0, 0x00, 0x68} + ip := IP{0x20, 0x01, 0x48, 0x60, 0, 0, 0x20, 0x01, 0, 0, 0, 0, 0, 0, 0x00, 0x68} var r Record r.Set(ip) - var ip2 IP6 + var ip2 IP require.NoError(t, r.Load(&ip2)) assert.Equal(t, ip, ip2) } // TestGetSetDiscPort tests encoding/decoding and setting/getting of the DiscPort key. -func TestGetSetDiscPort(t *testing.T) { - port := DiscPort(30309) +func TestGetSetUDP(t *testing.T) { + port := UDP(30309) var r Record r.Set(port) - var port2 DiscPort + var port2 UDP require.NoError(t, r.Load(&port2)) assert.Equal(t, port, port2) } @@ -90,7 +90,7 @@ func TestGetSetDiscPort(t *testing.T) { // TestGetSetSecp256k1 tests encoding/decoding and setting/getting of the Secp256k1 key. func TestGetSetSecp256k1(t *testing.T) { var r Record - if err := r.Sign(privkey); err != nil { + if err := SignV4(&r, privkey); err != nil { t.Fatal(err) } @@ -101,16 +101,16 @@ func TestGetSetSecp256k1(t *testing.T) { func TestLoadErrors(t *testing.T) { var r Record - ip4 := IP4{127, 0, 0, 1} + ip4 := IP{127, 0, 0, 1} r.Set(ip4) // Check error for missing keys. - var ip6 IP6 - err := r.Load(&ip6) + var udp UDP + err := r.Load(&udp) if !IsNotFound(err) { t.Error("IsNotFound should return true for missing key") } - assert.Equal(t, &KeyError{Key: ip6.ENRKey(), Err: errNotFound}, err) + assert.Equal(t, &KeyError{Key: udp.ENRKey(), Err: errNotFound}, err) // Check error for invalid keys. var list []uint @@ -174,7 +174,7 @@ func TestDirty(t *testing.T) { t.Errorf("expected errEncodeUnsigned, got %#v", err) } - require.NoError(t, r.Sign(privkey)) + require.NoError(t, SignV4(&r, privkey)) if !r.Signed() { t.Error("Signed return false for signed record") } @@ -194,13 +194,13 @@ func TestDirty(t *testing.T) { func TestGetSetOverwrite(t *testing.T) { var r Record - ip := IP4{192, 168, 0, 3} + ip := IP{192, 168, 0, 3} r.Set(ip) - ip2 := IP4{192, 168, 0, 4} + ip2 := IP{192, 168, 0, 4} r.Set(ip2) - var ip3 IP4 + var ip3 IP require.NoError(t, r.Load(&ip3)) assert.Equal(t, ip2, ip3) } @@ -208,9 +208,9 @@ func TestGetSetOverwrite(t *testing.T) { // TestSignEncodeAndDecode tests signing, RLP encoding and RLP decoding of a record. func TestSignEncodeAndDecode(t *testing.T) { var r Record - r.Set(DiscPort(30303)) - r.Set(IP4{127, 0, 0, 1}) - require.NoError(t, r.Sign(privkey)) + r.Set(UDP(30303)) + r.Set(IP{127, 0, 0, 1}) + require.NoError(t, SignV4(&r, privkey)) blob, err := rlp.EncodeToBytes(r) require.NoError(t, err) @@ -230,12 +230,12 @@ func TestNodeAddr(t *testing.T) { t.Errorf("wrong address on empty record: got %v, want %v", addr, nil) } - require.NoError(t, r.Sign(privkey)) - expected := "caaa1485d83b18b32ed9ad666026151bf0cae8a0a88c857ae2d4c5be2daa6726" + require.NoError(t, SignV4(&r, privkey)) + expected := "a448f24c6d18e575453db13171562b71999873db5b286df957af199ec94617f7" assert.Equal(t, expected, hex.EncodeToString(r.NodeAddr())) } -var pyRecord, _ = hex.DecodeString("f896b840954dc36583c1f4b69ab59b1375f362f06ee99f3723cd77e64b6de6d211c27d7870642a79d4516997f94091325d2a7ca6215376971455fb221d34f35b277149a1018664697363763582765f82696490736563703235366b312d6b656363616b83697034847f00000189736563703235366b31a103ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd3138") +var pyRecord, _ = hex.DecodeString("f884b8407098ad865b00a582051940cb9cf36836572411a47278783077011599ed5cd16b76f2635f4e234738f30813a89eb9137e3e3df5266e3a1f11df72ecf1145ccb9c01826964827634826970847f00000189736563703235366b31a103ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd31388375647082765f") // TestPythonInterop checks that we can decode and verify a record produced by the Python // implementation. @@ -246,10 +246,10 @@ func TestPythonInterop(t *testing.T) { } var ( - wantAddr, _ = hex.DecodeString("caaa1485d83b18b32ed9ad666026151bf0cae8a0a88c857ae2d4c5be2daa6726") - wantSeq = uint64(1) - wantIP = IP4{127, 0, 0, 1} - wantDiscport = DiscPort(30303) + wantAddr, _ = hex.DecodeString("a448f24c6d18e575453db13171562b71999873db5b286df957af199ec94617f7") + wantSeq = uint64(1) + wantIP = IP{127, 0, 0, 1} + wantUDP = UDP(30303) ) if r.Seq() != wantSeq { t.Errorf("wrong seq: got %d, want %d", r.Seq(), wantSeq) @@ -257,7 +257,7 @@ func TestPythonInterop(t *testing.T) { if addr := r.NodeAddr(); !bytes.Equal(addr, wantAddr) { t.Errorf("wrong addr: got %x, want %x", addr, wantAddr) } - want := map[Entry]interface{}{new(IP4): &wantIP, new(DiscPort): &wantDiscport} + want := map[Entry]interface{}{new(IP): &wantIP, new(UDP): &wantUDP} for k, v := range want { desc := fmt.Sprintf("loading key %q", k.ENRKey()) if assert.NoError(t, r.Load(k), desc) { @@ -272,14 +272,14 @@ func TestRecordTooBig(t *testing.T) { key := randomString(10) // set a big value for random key, expect error - r.Set(WithEntry(key, randomString(300))) - if err := r.Sign(privkey); err != errTooBig { + r.Set(WithEntry(key, randomString(SizeLimit))) + if err := SignV4(&r, privkey); err != errTooBig { t.Fatalf("expected to get errTooBig, got %#v", err) } // set an acceptable value for random key, expect no error r.Set(WithEntry(key, randomString(100))) - require.NoError(t, r.Sign(privkey)) + require.NoError(t, SignV4(&r, privkey)) } // TestSignEncodeAndDecodeRandom tests encoding/decoding of records containing random key/value pairs. @@ -295,7 +295,7 @@ func TestSignEncodeAndDecodeRandom(t *testing.T) { r.Set(WithEntry(key, &value)) } - require.NoError(t, r.Sign(privkey)) + require.NoError(t, SignV4(&r, privkey)) _, err := rlp.EncodeToBytes(r) require.NoError(t, err) diff --git a/p2p/enr/entries.go b/p2p/enr/entries.go index 4f97b0b955..a3aa8807fa 100644 --- a/p2p/enr/entries.go +++ b/p2p/enr/entries.go @@ -57,59 +57,43 @@ func WithEntry(k string, v interface{}) Entry { return &generic{key: k, value: v} } -// DiscPort is the "discv5" key, which holds the UDP port for discovery v5. -type DiscPort uint16 +// TCP is the "tcp" key, which holds the TCP port of the node. +type TCP uint16 -func (v DiscPort) ENRKey() string { return "discv5" } +func (v TCP) ENRKey() string { return "tcp" } + +// UDP is the "udp" key, which holds the UDP port of the node. +type UDP uint16 + +func (v UDP) ENRKey() string { return "udp" } // ID is the "id" key, which holds the name of the identity scheme. type ID string +const IDv4 = ID("v4") // the default identity scheme + func (v ID) ENRKey() string { return "id" } -// IP4 is the "ip4" key, which holds a 4-byte IPv4 address. -type IP4 net.IP +// IP is the "ip" key, which holds the IP address of the node. +type IP net.IP -func (v IP4) ENRKey() string { return "ip4" } +func (v IP) ENRKey() string { return "ip" } // EncodeRLP implements rlp.Encoder. -func (v IP4) EncodeRLP(w io.Writer) error { - ip4 := net.IP(v).To4() - if ip4 == nil { - return fmt.Errorf("invalid IPv4 address: %v", v) +func (v IP) EncodeRLP(w io.Writer) error { + if ip4 := net.IP(v).To4(); ip4 != nil { + return rlp.Encode(w, ip4) } - return rlp.Encode(w, ip4) + return rlp.Encode(w, net.IP(v)) } // DecodeRLP implements rlp.Decoder. -func (v *IP4) DecodeRLP(s *rlp.Stream) error { +func (v *IP) DecodeRLP(s *rlp.Stream) error { if err := s.Decode((*net.IP)(v)); err != nil { return err } - if len(*v) != 4 { - return fmt.Errorf("invalid IPv4 address, want 4 bytes: %v", *v) - } - return nil -} - -// IP6 is the "ip6" key, which holds a 16-byte IPv6 address. -type IP6 net.IP - -func (v IP6) ENRKey() string { return "ip6" } - -// EncodeRLP implements rlp.Encoder. -func (v IP6) EncodeRLP(w io.Writer) error { - ip6 := net.IP(v) - return rlp.Encode(w, ip6) -} - -// DecodeRLP implements rlp.Decoder. -func (v *IP6) DecodeRLP(s *rlp.Stream) error { - if err := s.Decode((*net.IP)(v)); err != nil { - return err - } - if len(*v) != 16 { - return fmt.Errorf("invalid IPv6 address, want 16 bytes: %v", *v) + if len(*v) != 4 && len(*v) != 16 { + return fmt.Errorf("invalid IP address, want 4 or 16 bytes: %v", *v) } return nil } diff --git a/p2p/enr/idscheme.go b/p2p/enr/idscheme.go new file mode 100644 index 0000000000..98a3c13232 --- /dev/null +++ b/p2p/enr/idscheme.go @@ -0,0 +1,114 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package enr + +import ( + "crypto/ecdsa" + "fmt" + "sync" + + "github.com/EthereumCommonwealth/go-callisto/common/math" + "github.com/EthereumCommonwealth/go-callisto/crypto" + "github.com/EthereumCommonwealth/go-callisto/crypto/sha3" + "github.com/EthereumCommonwealth/go-callisto/rlp" +) + +// Registry of known identity schemes. +var schemes sync.Map + +// An IdentityScheme is capable of verifying record signatures and +// deriving node addresses. +type IdentityScheme interface { + Verify(r *Record, sig []byte) error + NodeAddr(r *Record) []byte +} + +// RegisterIdentityScheme adds an identity scheme to the global registry. +func RegisterIdentityScheme(name string, scheme IdentityScheme) { + if _, loaded := schemes.LoadOrStore(name, scheme); loaded { + panic("identity scheme " + name + " already registered") + } +} + +// FindIdentityScheme resolves name to an identity scheme in the global registry. +func FindIdentityScheme(name string) IdentityScheme { + s, ok := schemes.Load(name) + if !ok { + return nil + } + return s.(IdentityScheme) +} + +// v4ID is the "v4" identity scheme. +type v4ID struct{} + +func init() { + RegisterIdentityScheme("v4", v4ID{}) +} + +// SignV4 signs a record using the v4 scheme. +func SignV4(r *Record, privkey *ecdsa.PrivateKey) error { + // Copy r to avoid modifying it if signing fails. + cpy := *r + cpy.Set(ID("v4")) + cpy.Set(Secp256k1(privkey.PublicKey)) + + h := sha3.NewKeccak256() + rlp.Encode(h, cpy.AppendElements(nil)) + sig, err := crypto.Sign(h.Sum(nil), privkey) + if err != nil { + return err + } + sig = sig[:len(sig)-1] // remove v + if err = cpy.SetSig("v4", sig); err == nil { + *r = cpy + } + return err +} + +// s256raw is an unparsed secp256k1 public key entry. +type s256raw []byte + +func (s256raw) ENRKey() string { return "secp256k1" } + +func (v4ID) Verify(r *Record, sig []byte) error { + var entry s256raw + if err := r.Load(&entry); err != nil { + return err + } else if len(entry) != 33 { + return fmt.Errorf("invalid public key") + } + + h := sha3.NewKeccak256() + rlp.Encode(h, r.AppendElements(nil)) + if !crypto.VerifySignature(entry, h.Sum(nil), sig) { + return errInvalidSig + } + return nil +} + +func (v4ID) NodeAddr(r *Record) []byte { + var pubkey Secp256k1 + err := r.Load(&pubkey) + if err != nil { + return nil + } + buf := make([]byte, 64) + math.ReadBits(pubkey.X, buf[:32]) + math.ReadBits(pubkey.Y, buf[32:]) + return crypto.Keccak256(buf) +} diff --git a/p2p/enr/idscheme_test.go b/p2p/enr/idscheme_test.go new file mode 100644 index 0000000000..d790e12f14 --- /dev/null +++ b/p2p/enr/idscheme_test.go @@ -0,0 +1,36 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package enr + +import ( + "crypto/ecdsa" + "math/big" + "testing" +) + +// Checks that failure to sign leaves the record unmodified. +func TestSignError(t *testing.T) { + invalidKey := &ecdsa.PrivateKey{D: new(big.Int), PublicKey: *pubkey} + + var r Record + if err := SignV4(&r, invalidKey); err == nil { + t.Fatal("expected error from SignV4") + } + if len(r.pairs) > 0 { + t.Fatal("expected empty record, have", r.pairs) + } +} diff --git a/p2p/message.go b/p2p/message.go index 846b8a0de5..283d952464 100644 --- a/p2p/message.go +++ b/p2p/message.go @@ -229,21 +229,20 @@ func ExpectMsg(r MsgReader, code uint64, content interface{}) error { } if content == nil { return msg.Discard() - } else { - contentEnc, err := rlp.EncodeToBytes(content) - if err != nil { - panic("content encode error: " + err.Error()) - } - if int(msg.Size) != len(contentEnc) { - return fmt.Errorf("message size mismatch: got %d, want %d", msg.Size, len(contentEnc)) - } - actualContent, err := ioutil.ReadAll(msg.Payload) - if err != nil { - return err - } - if !bytes.Equal(actualContent, contentEnc) { - return fmt.Errorf("message payload mismatch:\ngot: %x\nwant: %x", actualContent, contentEnc) - } + } + contentEnc, err := rlp.EncodeToBytes(content) + if err != nil { + panic("content encode error: " + err.Error()) + } + if int(msg.Size) != len(contentEnc) { + return fmt.Errorf("message size mismatch: got %d, want %d", msg.Size, len(contentEnc)) + } + actualContent, err := ioutil.ReadAll(msg.Payload) + if err != nil { + return err + } + if !bytes.Equal(actualContent, contentEnc) { + return fmt.Errorf("message payload mismatch:\ngot: %x\nwant: %x", actualContent, contentEnc) } return nil } @@ -271,15 +270,15 @@ func newMsgEventer(rw MsgReadWriter, feed *event.Feed, peerID discover.NodeID, p // ReadMsg reads a message from the underlying MsgReadWriter and emits a // "message received" event -func (self *msgEventer) ReadMsg() (Msg, error) { - msg, err := self.MsgReadWriter.ReadMsg() +func (ev *msgEventer) ReadMsg() (Msg, error) { + msg, err := ev.MsgReadWriter.ReadMsg() if err != nil { return msg, err } - self.feed.Send(&PeerEvent{ + ev.feed.Send(&PeerEvent{ Type: PeerEventTypeMsgRecv, - Peer: self.peerID, - Protocol: self.Protocol, + Peer: ev.peerID, + Protocol: ev.Protocol, MsgCode: &msg.Code, MsgSize: &msg.Size, }) @@ -288,15 +287,15 @@ func (self *msgEventer) ReadMsg() (Msg, error) { // WriteMsg writes a message to the underlying MsgReadWriter and emits a // "message sent" event -func (self *msgEventer) WriteMsg(msg Msg) error { - err := self.MsgReadWriter.WriteMsg(msg) +func (ev *msgEventer) WriteMsg(msg Msg) error { + err := ev.MsgReadWriter.WriteMsg(msg) if err != nil { return err } - self.feed.Send(&PeerEvent{ + ev.feed.Send(&PeerEvent{ Type: PeerEventTypeMsgSend, - Peer: self.peerID, - Protocol: self.Protocol, + Peer: ev.peerID, + Protocol: ev.Protocol, MsgCode: &msg.Code, MsgSize: &msg.Size, }) @@ -305,8 +304,8 @@ func (self *msgEventer) WriteMsg(msg Msg) error { // Close closes the underlying MsgReadWriter if it implements the io.Closer // interface -func (self *msgEventer) Close() error { - if v, ok := self.MsgReadWriter.(io.Closer); ok { +func (ev *msgEventer) Close() error { + if v, ok := ev.MsgReadWriter.(io.Closer); ok { return v.Close() } return nil diff --git a/p2p/peer.go b/p2p/peer.go index 81561e616c..640a4cd0eb 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -220,6 +220,7 @@ loop: reason = discReasonForError(err) break loop case err = <-p.disc: + reason = discReasonForError(err) break loop } } diff --git a/p2p/peer_error.go b/p2p/peer_error.go index a1cddb707b..ab61bfef06 100644 --- a/p2p/peer_error.go +++ b/p2p/peer_error.go @@ -48,8 +48,8 @@ func newPeerError(code int, format string, v ...interface{}) *peerError { return err } -func (self *peerError) Error() string { - return self.message +func (pe *peerError) Error() string { + return pe.message } var errProtocolReturned = errors.New("protocol returned") diff --git a/p2p/simulations/adapters/exec.go b/p2p/simulations/adapters/exec.go index b6875932ae..0eae4e1a98 100644 --- a/p2p/simulations/adapters/exec.go +++ b/p2p/simulations/adapters/exec.go @@ -17,7 +17,6 @@ package adapters import ( - "bufio" "context" "crypto/ecdsa" "encoding/json" @@ -29,7 +28,6 @@ import ( "os/exec" "os/signal" "path/filepath" - "regexp" "strings" "sync" "syscall" @@ -150,10 +148,6 @@ func (n *ExecNode) Client() (*rpc.Client, error) { return n.client, nil } -// wsAddrPattern is a regex used to read the WebSocket address from the node's -// log -var wsAddrPattern = regexp.MustCompile(`ws://[\d.:]+`) - // Start exec's the node passing the ID and service as command line arguments // and the node config encoded as JSON in the _P2P_NODE_CONFIG environment // variable @@ -196,23 +190,9 @@ func (n *ExecNode) Start(snapshots map[string][]byte) (err error) { n.Cmd = cmd // read the WebSocket address from the stderr logs - var wsAddr string - wsAddrC := make(chan string) - go func() { - s := bufio.NewScanner(stderrR) - for s.Scan() { - if strings.Contains(s.Text(), "WebSocket endpoint opened:") { - wsAddrC <- wsAddrPattern.FindString(s.Text()) - } - } - }() - select { - case wsAddr = <-wsAddrC: - if wsAddr == "" { - return errors.New("failed to read WebSocket address from stderr") - } - case <-time.After(10 * time.Second): - return errors.New("timed out waiting for WebSocket address on stderr") + wsAddr, err := findWSAddr(stderrR, 10*time.Second) + if err != nil { + return fmt.Errorf("error getting WebSocket address: %s", err) } // create the RPC client and load the node info diff --git a/p2p/simulations/adapters/inproc.go b/p2p/simulations/adapters/inproc.go index fef00ee91e..81271d449f 100644 --- a/p2p/simulations/adapters/inproc.go +++ b/p2p/simulations/adapters/inproc.go @@ -154,30 +154,30 @@ type SimNode struct { } // Addr returns the node's discovery address -func (self *SimNode) Addr() []byte { - return []byte(self.Node().String()) +func (sn *SimNode) Addr() []byte { + return []byte(sn.Node().String()) } // Node returns a discover.Node representing the SimNode -func (self *SimNode) Node() *discover.Node { - return discover.NewNode(self.ID, net.IP{127, 0, 0, 1}, 30303, 30303) +func (sn *SimNode) Node() *discover.Node { + return discover.NewNode(sn.ID, net.IP{127, 0, 0, 1}, 30303, 30303) } // Client returns an rpc.Client which can be used to communicate with the // underlying services (it is set once the node has started) -func (self *SimNode) Client() (*rpc.Client, error) { - self.lock.RLock() - defer self.lock.RUnlock() - if self.client == nil { +func (sn *SimNode) Client() (*rpc.Client, error) { + sn.lock.RLock() + defer sn.lock.RUnlock() + if sn.client == nil { return nil, errors.New("node not started") } - return self.client, nil + return sn.client, nil } // ServeRPC serves RPC requests over the given connection by creating an // in-memory client to the node's RPC server -func (self *SimNode) ServeRPC(conn net.Conn) error { - handler, err := self.node.RPCHandler() +func (sn *SimNode) ServeRPC(conn net.Conn) error { + handler, err := sn.node.RPCHandler() if err != nil { return err } @@ -187,13 +187,13 @@ func (self *SimNode) ServeRPC(conn net.Conn) error { // Snapshots creates snapshots of the services by calling the // simulation_snapshot RPC method -func (self *SimNode) Snapshots() (map[string][]byte, error) { - self.lock.RLock() - services := make(map[string]node.Service, len(self.running)) - for name, service := range self.running { +func (sn *SimNode) Snapshots() (map[string][]byte, error) { + sn.lock.RLock() + services := make(map[string]node.Service, len(sn.running)) + for name, service := range sn.running { services[name] = service } - self.lock.RUnlock() + sn.lock.RUnlock() if len(services) == 0 { return nil, errors.New("no running services") } @@ -213,23 +213,23 @@ func (self *SimNode) Snapshots() (map[string][]byte, error) { } // Start registers the services and starts the underlying devp2p node -func (self *SimNode) Start(snapshots map[string][]byte) error { +func (sn *SimNode) Start(snapshots map[string][]byte) error { newService := func(name string) func(ctx *node.ServiceContext) (node.Service, error) { return func(nodeCtx *node.ServiceContext) (node.Service, error) { ctx := &ServiceContext{ - RPCDialer: self.adapter, + RPCDialer: sn.adapter, NodeContext: nodeCtx, - Config: self.config, + Config: sn.config, } if snapshots != nil { ctx.Snapshot = snapshots[name] } - serviceFunc := self.adapter.services[name] + serviceFunc := sn.adapter.services[name] service, err := serviceFunc(ctx) if err != nil { return nil, err } - self.running[name] = service + sn.running[name] = service return service, nil } } @@ -237,9 +237,9 @@ func (self *SimNode) Start(snapshots map[string][]byte) error { // ensure we only register the services once in the case of the node // being stopped and then started again var regErr error - self.registerOnce.Do(func() { - for _, name := range self.config.Services { - if err := self.node.Register(newService(name)); err != nil { + sn.registerOnce.Do(func() { + for _, name := range sn.config.Services { + if err := sn.node.Register(newService(name)); err != nil { regErr = err return } @@ -249,54 +249,54 @@ func (self *SimNode) Start(snapshots map[string][]byte) error { return regErr } - if err := self.node.Start(); err != nil { + if err := sn.node.Start(); err != nil { return err } // create an in-process RPC client - handler, err := self.node.RPCHandler() + handler, err := sn.node.RPCHandler() if err != nil { return err } - self.lock.Lock() - self.client = rpc.DialInProc(handler) - self.lock.Unlock() + sn.lock.Lock() + sn.client = rpc.DialInProc(handler) + sn.lock.Unlock() return nil } // Stop closes the RPC client and stops the underlying devp2p node -func (self *SimNode) Stop() error { - self.lock.Lock() - if self.client != nil { - self.client.Close() - self.client = nil +func (sn *SimNode) Stop() error { + sn.lock.Lock() + if sn.client != nil { + sn.client.Close() + sn.client = nil } - self.lock.Unlock() - return self.node.Stop() + sn.lock.Unlock() + return sn.node.Stop() } // Services returns a copy of the underlying services -func (self *SimNode) Services() []node.Service { - self.lock.RLock() - defer self.lock.RUnlock() - services := make([]node.Service, 0, len(self.running)) - for _, service := range self.running { +func (sn *SimNode) Services() []node.Service { + sn.lock.RLock() + defer sn.lock.RUnlock() + services := make([]node.Service, 0, len(sn.running)) + for _, service := range sn.running { services = append(services, service) } return services } // Server returns the underlying p2p.Server -func (self *SimNode) Server() *p2p.Server { - return self.node.Server() +func (sn *SimNode) Server() *p2p.Server { + return sn.node.Server() } // SubscribeEvents subscribes the given channel to peer events from the // underlying p2p.Server -func (self *SimNode) SubscribeEvents(ch chan *p2p.PeerEvent) event.Subscription { - srv := self.Server() +func (sn *SimNode) SubscribeEvents(ch chan *p2p.PeerEvent) event.Subscription { + srv := sn.Server() if srv == nil { panic("node not running") } @@ -304,12 +304,12 @@ func (self *SimNode) SubscribeEvents(ch chan *p2p.PeerEvent) event.Subscription } // NodeInfo returns information about the node -func (self *SimNode) NodeInfo() *p2p.NodeInfo { - server := self.Server() +func (sn *SimNode) NodeInfo() *p2p.NodeInfo { + server := sn.Server() if server == nil { return &p2p.NodeInfo{ - ID: self.ID.String(), - Enode: self.Node().String(), + ID: sn.ID.String(), + Enode: sn.Node().String(), } } return server.NodeInfo() diff --git a/p2p/simulations/adapters/state.go b/p2p/simulations/adapters/state.go index 0d4ecfb0ff..78dfb11f95 100644 --- a/p2p/simulations/adapters/state.go +++ b/p2p/simulations/adapters/state.go @@ -20,12 +20,12 @@ type SimStateStore struct { m map[string][]byte } -func (self *SimStateStore) Load(s string) ([]byte, error) { - return self.m[s], nil +func (st *SimStateStore) Load(s string) ([]byte, error) { + return st.m[s], nil } -func (self *SimStateStore) Save(s string, data []byte) error { - self.m[s] = data +func (st *SimStateStore) Save(s string, data []byte) error { + st.m[s] = data return nil } diff --git a/p2p/simulations/adapters/ws.go b/p2p/simulations/adapters/ws.go new file mode 100644 index 0000000000..979a21709e --- /dev/null +++ b/p2p/simulations/adapters/ws.go @@ -0,0 +1,51 @@ +package adapters + +import ( + "bufio" + "errors" + "io" + "regexp" + "strings" + "time" +) + +// wsAddrPattern is a regex used to read the WebSocket address from the node's +// log +var wsAddrPattern = regexp.MustCompile(`ws://[\d.:]+`) + +func matchWSAddr(str string) (string, bool) { + if !strings.Contains(str, "WebSocket endpoint opened") { + return "", false + } + + return wsAddrPattern.FindString(str), true +} + +// findWSAddr scans through reader r, looking for the log entry with +// WebSocket address information. +func findWSAddr(r io.Reader, timeout time.Duration) (string, error) { + ch := make(chan string) + + go func() { + s := bufio.NewScanner(r) + for s.Scan() { + addr, ok := matchWSAddr(s.Text()) + if ok { + ch <- addr + } + } + close(ch) + }() + + var wsAddr string + select { + case wsAddr = <-ch: + if wsAddr == "" { + return "", errors.New("empty result") + } + case <-time.After(timeout): + return "", errors.New("timed out") + } + + return wsAddr, nil +} diff --git a/p2p/simulations/adapters/ws_test.go b/p2p/simulations/adapters/ws_test.go new file mode 100644 index 0000000000..0bb9ed2b2b --- /dev/null +++ b/p2p/simulations/adapters/ws_test.go @@ -0,0 +1,21 @@ +package adapters + +import ( + "bytes" + "testing" + "time" +) + +func TestFindWSAddr(t *testing.T) { + line := `t=2018-05-02T19:00:45+0200 lvl=info msg="WebSocket endpoint opened" node.id=26c65a606d1125a44695bc08573190d047152b6b9a776ccbbe593e90f91444d9c1ebdadac6a775ad9fdd0923468a1d698ed3a842c1fb89c1bc0f9d4801f8c39c url=ws://127.0.0.1:59975` + buf := bytes.NewBufferString(line) + got, err := findWSAddr(buf, 10*time.Second) + if err != nil { + t.Fatalf("Failed to find addr: %v", err) + } + expected := `ws://127.0.0.1:59975` + + if got != expected { + t.Fatalf("Expected to get '%s', but got '%s'", expected, got) + } +} diff --git a/p2p/simulations/network.go b/p2p/simulations/network.go index 9c09436e25..4c44207385 100644 --- a/p2p/simulations/network.go +++ b/p2p/simulations/network.go @@ -74,22 +74,22 @@ func NewNetwork(nodeAdapter adapters.NodeAdapter, conf *NetworkConfig) *Network } // Events returns the output event feed of the Network. -func (self *Network) Events() *event.Feed { - return &self.events +func (net *Network) Events() *event.Feed { + return &net.events } // NewNode adds a new node to the network with a random ID -func (self *Network) NewNode() (*Node, error) { +func (net *Network) NewNode() (*Node, error) { conf := adapters.RandomNodeConfig() - conf.Services = []string{self.DefaultService} - return self.NewNodeWithConfig(conf) + conf.Services = []string{net.DefaultService} + return net.NewNodeWithConfig(conf) } // NewNodeWithConfig adds a new node to the network with the given config, // returning an error if a node with the same ID or name already exists -func (self *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error) { - self.lock.Lock() - defer self.lock.Unlock() +func (net *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error) { + net.lock.Lock() + defer net.lock.Unlock() // create a random ID and PrivateKey if not set if conf.ID == (discover.NodeID{}) { @@ -100,31 +100,31 @@ func (self *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error) id := conf.ID if conf.Reachable == nil { conf.Reachable = func(otherID discover.NodeID) bool { - _, err := self.InitConn(conf.ID, otherID) + _, err := net.InitConn(conf.ID, otherID) return err == nil } } // assign a name to the node if not set if conf.Name == "" { - conf.Name = fmt.Sprintf("node%02d", len(self.Nodes)+1) + conf.Name = fmt.Sprintf("node%02d", len(net.Nodes)+1) } // check the node doesn't already exist - if node := self.getNode(id); node != nil { + if node := net.getNode(id); node != nil { return nil, fmt.Errorf("node with ID %q already exists", id) } - if node := self.getNodeByName(conf.Name); node != nil { + if node := net.getNodeByName(conf.Name); node != nil { return nil, fmt.Errorf("node with name %q already exists", conf.Name) } // if no services are configured, use the default service if len(conf.Services) == 0 { - conf.Services = []string{self.DefaultService} + conf.Services = []string{net.DefaultService} } // use the NodeAdapter to create the node - adapterNode, err := self.nodeAdapter.NewNode(conf) + adapterNode, err := net.nodeAdapter.NewNode(conf) if err != nil { return nil, err } @@ -133,27 +133,27 @@ func (self *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error) Config: conf, } log.Trace(fmt.Sprintf("node %v created", id)) - self.nodeMap[id] = len(self.Nodes) - self.Nodes = append(self.Nodes, node) + net.nodeMap[id] = len(net.Nodes) + net.Nodes = append(net.Nodes, node) // emit a "control" event - self.events.Send(ControlEvent(node)) + net.events.Send(ControlEvent(node)) return node, nil } // Config returns the network configuration -func (self *Network) Config() *NetworkConfig { - return &self.NetworkConfig +func (net *Network) Config() *NetworkConfig { + return &net.NetworkConfig } // StartAll starts all nodes in the network -func (self *Network) StartAll() error { - for _, node := range self.Nodes { +func (net *Network) StartAll() error { + for _, node := range net.Nodes { if node.Up { continue } - if err := self.Start(node.ID()); err != nil { + if err := net.Start(node.ID()); err != nil { return err } } @@ -161,12 +161,12 @@ func (self *Network) StartAll() error { } // StopAll stops all nodes in the network -func (self *Network) StopAll() error { - for _, node := range self.Nodes { +func (net *Network) StopAll() error { + for _, node := range net.Nodes { if !node.Up { continue } - if err := self.Stop(node.ID()); err != nil { + if err := net.Stop(node.ID()); err != nil { return err } } @@ -174,21 +174,21 @@ func (self *Network) StopAll() error { } // Start starts the node with the given ID -func (self *Network) Start(id discover.NodeID) error { - return self.startWithSnapshots(id, nil) +func (net *Network) Start(id discover.NodeID) error { + return net.startWithSnapshots(id, nil) } // startWithSnapshots starts the node with the given ID using the give // snapshots -func (self *Network) startWithSnapshots(id discover.NodeID, snapshots map[string][]byte) error { - node := self.GetNode(id) +func (net *Network) startWithSnapshots(id discover.NodeID, snapshots map[string][]byte) error { + node := net.GetNode(id) if node == nil { return fmt.Errorf("node %v does not exist", id) } if node.Up { return fmt.Errorf("node %v already up", id) } - log.Trace(fmt.Sprintf("starting node %v: %v using %v", id, node.Up, self.nodeAdapter.Name())) + log.Trace(fmt.Sprintf("starting node %v: %v using %v", id, node.Up, net.nodeAdapter.Name())) if err := node.Start(snapshots); err != nil { log.Warn(fmt.Sprintf("start up failed: %v", err)) return err @@ -196,7 +196,7 @@ func (self *Network) startWithSnapshots(id discover.NodeID, snapshots map[string node.Up = true log.Info(fmt.Sprintf("started node %v: %v", id, node.Up)) - self.events.Send(NewEvent(node)) + net.events.Send(NewEvent(node)) // subscribe to peer events client, err := node.Client() @@ -208,22 +208,22 @@ func (self *Network) startWithSnapshots(id discover.NodeID, snapshots map[string if err != nil { return fmt.Errorf("error getting peer events for node %v: %s", id, err) } - go self.watchPeerEvents(id, events, sub) + go net.watchPeerEvents(id, events, sub) return nil } // watchPeerEvents reads peer events from the given channel and emits // corresponding network events -func (self *Network) watchPeerEvents(id discover.NodeID, events chan *p2p.PeerEvent, sub event.Subscription) { +func (net *Network) watchPeerEvents(id discover.NodeID, events chan *p2p.PeerEvent, sub event.Subscription) { defer func() { sub.Unsubscribe() // assume the node is now down - self.lock.Lock() - node := self.getNode(id) + net.lock.Lock() + node := net.getNode(id) node.Up = false - self.lock.Unlock() - self.events.Send(NewEvent(node)) + net.lock.Unlock() + net.events.Send(NewEvent(node)) }() for { select { @@ -235,16 +235,16 @@ func (self *Network) watchPeerEvents(id discover.NodeID, events chan *p2p.PeerEv switch event.Type { case p2p.PeerEventTypeAdd: - self.DidConnect(id, peer) + net.DidConnect(id, peer) case p2p.PeerEventTypeDrop: - self.DidDisconnect(id, peer) + net.DidDisconnect(id, peer) case p2p.PeerEventTypeMsgSend: - self.DidSend(id, peer, event.Protocol, *event.MsgCode) + net.DidSend(id, peer, event.Protocol, *event.MsgCode) case p2p.PeerEventTypeMsgRecv: - self.DidReceive(peer, id, event.Protocol, *event.MsgCode) + net.DidReceive(peer, id, event.Protocol, *event.MsgCode) } @@ -258,8 +258,8 @@ func (self *Network) watchPeerEvents(id discover.NodeID, events chan *p2p.PeerEv } // Stop stops the node with the given ID -func (self *Network) Stop(id discover.NodeID) error { - node := self.GetNode(id) +func (net *Network) Stop(id discover.NodeID) error { + node := net.GetNode(id) if node == nil { return fmt.Errorf("node %v does not exist", id) } @@ -272,15 +272,15 @@ func (self *Network) Stop(id discover.NodeID) error { node.Up = false log.Info(fmt.Sprintf("stop node %v: %v", id, node.Up)) - self.events.Send(ControlEvent(node)) + net.events.Send(ControlEvent(node)) return nil } // Connect connects two nodes together by calling the "admin_addPeer" RPC // method on the "one" node so that it connects to the "other" node -func (self *Network) Connect(oneID, otherID discover.NodeID) error { +func (net *Network) Connect(oneID, otherID discover.NodeID) error { log.Debug(fmt.Sprintf("connecting %s to %s", oneID, otherID)) - conn, err := self.InitConn(oneID, otherID) + conn, err := net.InitConn(oneID, otherID) if err != nil { return err } @@ -288,14 +288,14 @@ func (self *Network) Connect(oneID, otherID discover.NodeID) error { if err != nil { return err } - self.events.Send(ControlEvent(conn)) + net.events.Send(ControlEvent(conn)) return client.Call(nil, "admin_addPeer", string(conn.other.Addr())) } // Disconnect disconnects two nodes by calling the "admin_removePeer" RPC // method on the "one" node so that it disconnects from the "other" node -func (self *Network) Disconnect(oneID, otherID discover.NodeID) error { - conn := self.GetConn(oneID, otherID) +func (net *Network) Disconnect(oneID, otherID discover.NodeID) error { + conn := net.GetConn(oneID, otherID) if conn == nil { return fmt.Errorf("connection between %v and %v does not exist", oneID, otherID) } @@ -306,13 +306,13 @@ func (self *Network) Disconnect(oneID, otherID discover.NodeID) error { if err != nil { return err } - self.events.Send(ControlEvent(conn)) + net.events.Send(ControlEvent(conn)) return client.Call(nil, "admin_removePeer", string(conn.other.Addr())) } // DidConnect tracks the fact that the "one" node connected to the "other" node -func (self *Network) DidConnect(one, other discover.NodeID) error { - conn, err := self.GetOrCreateConn(one, other) +func (net *Network) DidConnect(one, other discover.NodeID) error { + conn, err := net.GetOrCreateConn(one, other) if err != nil { return fmt.Errorf("connection between %v and %v does not exist", one, other) } @@ -320,14 +320,14 @@ func (self *Network) DidConnect(one, other discover.NodeID) error { return fmt.Errorf("%v and %v already connected", one, other) } conn.Up = true - self.events.Send(NewEvent(conn)) + net.events.Send(NewEvent(conn)) return nil } // DidDisconnect tracks the fact that the "one" node disconnected from the // "other" node -func (self *Network) DidDisconnect(one, other discover.NodeID) error { - conn := self.GetConn(one, other) +func (net *Network) DidDisconnect(one, other discover.NodeID) error { + conn := net.GetConn(one, other) if conn == nil { return fmt.Errorf("connection between %v and %v does not exist", one, other) } @@ -336,12 +336,12 @@ func (self *Network) DidDisconnect(one, other discover.NodeID) error { } conn.Up = false conn.initiated = time.Now().Add(-dialBanTimeout) - self.events.Send(NewEvent(conn)) + net.events.Send(NewEvent(conn)) return nil } // DidSend tracks the fact that "sender" sent a message to "receiver" -func (self *Network) DidSend(sender, receiver discover.NodeID, proto string, code uint64) error { +func (net *Network) DidSend(sender, receiver discover.NodeID, proto string, code uint64) error { msg := &Msg{ One: sender, Other: receiver, @@ -349,12 +349,12 @@ func (self *Network) DidSend(sender, receiver discover.NodeID, proto string, cod Code: code, Received: false, } - self.events.Send(NewEvent(msg)) + net.events.Send(NewEvent(msg)) return nil } // DidReceive tracks the fact that "receiver" received a message from "sender" -func (self *Network) DidReceive(sender, receiver discover.NodeID, proto string, code uint64) error { +func (net *Network) DidReceive(sender, receiver discover.NodeID, proto string, code uint64) error { msg := &Msg{ One: sender, Other: receiver, @@ -362,36 +362,36 @@ func (self *Network) DidReceive(sender, receiver discover.NodeID, proto string, Code: code, Received: true, } - self.events.Send(NewEvent(msg)) + net.events.Send(NewEvent(msg)) return nil } // GetNode gets the node with the given ID, returning nil if the node does not // exist -func (self *Network) GetNode(id discover.NodeID) *Node { - self.lock.Lock() - defer self.lock.Unlock() - return self.getNode(id) +func (net *Network) GetNode(id discover.NodeID) *Node { + net.lock.Lock() + defer net.lock.Unlock() + return net.getNode(id) } // GetNode gets the node with the given name, returning nil if the node does // not exist -func (self *Network) GetNodeByName(name string) *Node { - self.lock.Lock() - defer self.lock.Unlock() - return self.getNodeByName(name) +func (net *Network) GetNodeByName(name string) *Node { + net.lock.Lock() + defer net.lock.Unlock() + return net.getNodeByName(name) } -func (self *Network) getNode(id discover.NodeID) *Node { - i, found := self.nodeMap[id] +func (net *Network) getNode(id discover.NodeID) *Node { + i, found := net.nodeMap[id] if !found { return nil } - return self.Nodes[i] + return net.Nodes[i] } -func (self *Network) getNodeByName(name string) *Node { - for _, node := range self.Nodes { +func (net *Network) getNodeByName(name string) *Node { + for _, node := range net.Nodes { if node.Config.Name == name { return node } @@ -400,40 +400,40 @@ func (self *Network) getNodeByName(name string) *Node { } // GetNodes returns the existing nodes -func (self *Network) GetNodes() (nodes []*Node) { - self.lock.Lock() - defer self.lock.Unlock() +func (net *Network) GetNodes() (nodes []*Node) { + net.lock.Lock() + defer net.lock.Unlock() - nodes = append(nodes, self.Nodes...) + nodes = append(nodes, net.Nodes...) return nodes } // GetConn returns the connection which exists between "one" and "other" // regardless of which node initiated the connection -func (self *Network) GetConn(oneID, otherID discover.NodeID) *Conn { - self.lock.Lock() - defer self.lock.Unlock() - return self.getConn(oneID, otherID) +func (net *Network) GetConn(oneID, otherID discover.NodeID) *Conn { + net.lock.Lock() + defer net.lock.Unlock() + return net.getConn(oneID, otherID) } // GetOrCreateConn is like GetConn but creates the connection if it doesn't // already exist -func (self *Network) GetOrCreateConn(oneID, otherID discover.NodeID) (*Conn, error) { - self.lock.Lock() - defer self.lock.Unlock() - return self.getOrCreateConn(oneID, otherID) +func (net *Network) GetOrCreateConn(oneID, otherID discover.NodeID) (*Conn, error) { + net.lock.Lock() + defer net.lock.Unlock() + return net.getOrCreateConn(oneID, otherID) } -func (self *Network) getOrCreateConn(oneID, otherID discover.NodeID) (*Conn, error) { - if conn := self.getConn(oneID, otherID); conn != nil { +func (net *Network) getOrCreateConn(oneID, otherID discover.NodeID) (*Conn, error) { + if conn := net.getConn(oneID, otherID); conn != nil { return conn, nil } - one := self.getNode(oneID) + one := net.getNode(oneID) if one == nil { return nil, fmt.Errorf("node %v does not exist", oneID) } - other := self.getNode(otherID) + other := net.getNode(otherID) if other == nil { return nil, fmt.Errorf("node %v does not exist", otherID) } @@ -444,18 +444,18 @@ func (self *Network) getOrCreateConn(oneID, otherID discover.NodeID) (*Conn, err other: other, } label := ConnLabel(oneID, otherID) - self.connMap[label] = len(self.Conns) - self.Conns = append(self.Conns, conn) + net.connMap[label] = len(net.Conns) + net.Conns = append(net.Conns, conn) return conn, nil } -func (self *Network) getConn(oneID, otherID discover.NodeID) *Conn { +func (net *Network) getConn(oneID, otherID discover.NodeID) *Conn { label := ConnLabel(oneID, otherID) - i, found := self.connMap[label] + i, found := net.connMap[label] if !found { return nil } - return self.Conns[i] + return net.Conns[i] } // InitConn(one, other) retrieves the connectiton model for the connection between @@ -466,13 +466,13 @@ func (self *Network) getConn(oneID, otherID discover.NodeID) *Conn { // it also checks whether there has been recent attempt to connect the peers // this is cheating as the simulation is used as an oracle and know about // remote peers attempt to connect to a node which will then not initiate the connection -func (self *Network) InitConn(oneID, otherID discover.NodeID) (*Conn, error) { - self.lock.Lock() - defer self.lock.Unlock() +func (net *Network) InitConn(oneID, otherID discover.NodeID) (*Conn, error) { + net.lock.Lock() + defer net.lock.Unlock() if oneID == otherID { return nil, fmt.Errorf("refusing to connect to self %v", oneID) } - conn, err := self.getOrCreateConn(oneID, otherID) + conn, err := net.getOrCreateConn(oneID, otherID) if err != nil { return nil, err } @@ -491,28 +491,28 @@ func (self *Network) InitConn(oneID, otherID discover.NodeID) (*Conn, error) { } // Shutdown stops all nodes in the network and closes the quit channel -func (self *Network) Shutdown() { - for _, node := range self.Nodes { +func (net *Network) Shutdown() { + for _, node := range net.Nodes { log.Debug(fmt.Sprintf("stopping node %s", node.ID().TerminalString())) if err := node.Stop(); err != nil { log.Warn(fmt.Sprintf("error stopping node %s", node.ID().TerminalString()), "err", err) } } - close(self.quitc) + close(net.quitc) } //Reset resets all network properties: //emtpies the nodes and the connection list -func (self *Network) Reset() { - self.lock.Lock() - defer self.lock.Unlock() +func (net *Network) Reset() { + net.lock.Lock() + defer net.lock.Unlock() //re-initialize the maps - self.connMap = make(map[string]int) - self.nodeMap = make(map[discover.NodeID]int) + net.connMap = make(map[string]int) + net.nodeMap = make(map[discover.NodeID]int) - self.Nodes = nil - self.Conns = nil + net.Nodes = nil + net.Conns = nil } // Node is a wrapper around adapters.Node which is used to track the status @@ -528,37 +528,37 @@ type Node struct { } // ID returns the ID of the node -func (self *Node) ID() discover.NodeID { - return self.Config.ID +func (n *Node) ID() discover.NodeID { + return n.Config.ID } // String returns a log-friendly string -func (self *Node) String() string { - return fmt.Sprintf("Node %v", self.ID().TerminalString()) +func (n *Node) String() string { + return fmt.Sprintf("Node %v", n.ID().TerminalString()) } // NodeInfo returns information about the node -func (self *Node) NodeInfo() *p2p.NodeInfo { +func (n *Node) NodeInfo() *p2p.NodeInfo { // avoid a panic if the node is not started yet - if self.Node == nil { + if n.Node == nil { return nil } - info := self.Node.NodeInfo() - info.Name = self.Config.Name + info := n.Node.NodeInfo() + info.Name = n.Config.Name return info } // MarshalJSON implements the json.Marshaler interface so that the encoded // JSON includes the NodeInfo -func (self *Node) MarshalJSON() ([]byte, error) { +func (n *Node) MarshalJSON() ([]byte, error) { return json.Marshal(struct { Info *p2p.NodeInfo `json:"info,omitempty"` Config *adapters.NodeConfig `json:"config,omitempty"` Up bool `json:"up"` }{ - Info: self.NodeInfo(), - Config: self.Config, - Up: self.Up, + Info: n.NodeInfo(), + Config: n.Config, + Up: n.Up, }) } @@ -580,19 +580,19 @@ type Conn struct { } // nodesUp returns whether both nodes are currently up -func (self *Conn) nodesUp() error { - if !self.one.Up { - return fmt.Errorf("one %v is not up", self.One) +func (c *Conn) nodesUp() error { + if !c.one.Up { + return fmt.Errorf("one %v is not up", c.One) } - if !self.other.Up { - return fmt.Errorf("other %v is not up", self.Other) + if !c.other.Up { + return fmt.Errorf("other %v is not up", c.Other) } return nil } // String returns a log-friendly string -func (self *Conn) String() string { - return fmt.Sprintf("Conn %v->%v", self.One.TerminalString(), self.Other.TerminalString()) +func (c *Conn) String() string { + return fmt.Sprintf("Conn %v->%v", c.One.TerminalString(), c.Other.TerminalString()) } // Msg represents a p2p message sent between two nodes in the network @@ -605,8 +605,8 @@ type Msg struct { } // String returns a log-friendly string -func (self *Msg) String() string { - return fmt.Sprintf("Msg(%d) %v->%v", self.Code, self.One.TerminalString(), self.Other.TerminalString()) +func (m *Msg) String() string { + return fmt.Sprintf("Msg(%d) %v->%v", m.Code, m.One.TerminalString(), m.Other.TerminalString()) } // ConnLabel generates a deterministic string which represents a connection @@ -640,14 +640,14 @@ type NodeSnapshot struct { } // Snapshot creates a network snapshot -func (self *Network) Snapshot() (*Snapshot, error) { - self.lock.Lock() - defer self.lock.Unlock() +func (net *Network) Snapshot() (*Snapshot, error) { + net.lock.Lock() + defer net.lock.Unlock() snap := &Snapshot{ - Nodes: make([]NodeSnapshot, len(self.Nodes)), - Conns: make([]Conn, len(self.Conns)), + Nodes: make([]NodeSnapshot, len(net.Nodes)), + Conns: make([]Conn, len(net.Conns)), } - for i, node := range self.Nodes { + for i, node := range net.Nodes { snap.Nodes[i] = NodeSnapshot{Node: *node} if !node.Up { continue @@ -658,33 +658,33 @@ func (self *Network) Snapshot() (*Snapshot, error) { } snap.Nodes[i].Snapshots = snapshots } - for i, conn := range self.Conns { + for i, conn := range net.Conns { snap.Conns[i] = *conn } return snap, nil } // Load loads a network snapshot -func (self *Network) Load(snap *Snapshot) error { +func (net *Network) Load(snap *Snapshot) error { for _, n := range snap.Nodes { - if _, err := self.NewNodeWithConfig(n.Node.Config); err != nil { + if _, err := net.NewNodeWithConfig(n.Node.Config); err != nil { return err } if !n.Node.Up { continue } - if err := self.startWithSnapshots(n.Node.Config.ID, n.Snapshots); err != nil { + if err := net.startWithSnapshots(n.Node.Config.ID, n.Snapshots); err != nil { return err } } for _, conn := range snap.Conns { - if !self.GetNode(conn.One).Up || !self.GetNode(conn.Other).Up { + if !net.GetNode(conn.One).Up || !net.GetNode(conn.Other).Up { //in this case, at least one of the nodes of a connection is not up, //so it would result in the snapshot `Load` to fail continue } - if err := self.Connect(conn.One, conn.Other); err != nil { + if err := net.Connect(conn.One, conn.Other); err != nil { return err } } @@ -692,7 +692,7 @@ func (self *Network) Load(snap *Snapshot) error { } // Subscribe reads control events from a channel and executes them -func (self *Network) Subscribe(events chan *Event) { +func (net *Network) Subscribe(events chan *Event) { for { select { case event, ok := <-events: @@ -700,23 +700,23 @@ func (self *Network) Subscribe(events chan *Event) { return } if event.Control { - self.executeControlEvent(event) + net.executeControlEvent(event) } - case <-self.quitc: + case <-net.quitc: return } } } -func (self *Network) executeControlEvent(event *Event) { +func (net *Network) executeControlEvent(event *Event) { log.Trace("execute control event", "type", event.Type, "event", event) switch event.Type { case EventTypeNode: - if err := self.executeNodeEvent(event); err != nil { + if err := net.executeNodeEvent(event); err != nil { log.Error("error executing node event", "event", event, "err", err) } case EventTypeConn: - if err := self.executeConnEvent(event); err != nil { + if err := net.executeConnEvent(event); err != nil { log.Error("error executing conn event", "event", event, "err", err) } case EventTypeMsg: @@ -724,21 +724,21 @@ func (self *Network) executeControlEvent(event *Event) { } } -func (self *Network) executeNodeEvent(e *Event) error { +func (net *Network) executeNodeEvent(e *Event) error { if !e.Node.Up { - return self.Stop(e.Node.ID()) + return net.Stop(e.Node.ID()) } - if _, err := self.NewNodeWithConfig(e.Node.Config); err != nil { + if _, err := net.NewNodeWithConfig(e.Node.Config); err != nil { return err } - return self.Start(e.Node.ID()) + return net.Start(e.Node.ID()) } -func (self *Network) executeConnEvent(e *Event) error { +func (net *Network) executeConnEvent(e *Event) error { if e.Conn.Up { - return self.Connect(e.Conn.One, e.Conn.Other) + return net.Connect(e.Conn.One, e.Conn.Other) } else { - return self.Disconnect(e.Conn.One, e.Conn.Other) + return net.Disconnect(e.Conn.One, e.Conn.Other) } } diff --git a/p2p/testing/peerpool.go b/p2p/testing/peerpool.go index c13dfa4b15..6f6b6c10c2 100644 --- a/p2p/testing/peerpool.go +++ b/p2p/testing/peerpool.go @@ -39,29 +39,29 @@ func NewTestPeerPool() *TestPeerPool { return &TestPeerPool{peers: make(map[discover.NodeID]TestPeer)} } -func (self *TestPeerPool) Add(p TestPeer) { - self.lock.Lock() - defer self.lock.Unlock() - log.Trace(fmt.Sprintf("pp add peer %v", p.ID())) - self.peers[p.ID()] = p +func (p *TestPeerPool) Add(peer TestPeer) { + p.lock.Lock() + defer p.lock.Unlock() + log.Trace(fmt.Sprintf("pp add peer %v", peer.ID())) + p.peers[peer.ID()] = peer } -func (self *TestPeerPool) Remove(p TestPeer) { - self.lock.Lock() - defer self.lock.Unlock() - delete(self.peers, p.ID()) +func (p *TestPeerPool) Remove(peer TestPeer) { + p.lock.Lock() + defer p.lock.Unlock() + delete(p.peers, peer.ID()) } -func (self *TestPeerPool) Has(id discover.NodeID) bool { - self.lock.Lock() - defer self.lock.Unlock() - _, ok := self.peers[id] +func (p *TestPeerPool) Has(id discover.NodeID) bool { + p.lock.Lock() + defer p.lock.Unlock() + _, ok := p.peers[id] return ok } -func (self *TestPeerPool) Get(id discover.NodeID) TestPeer { - self.lock.Lock() - defer self.lock.Unlock() - return self.peers[id] +func (p *TestPeerPool) Get(id discover.NodeID) TestPeer { + p.lock.Lock() + defer p.lock.Unlock() + return p.peers[id] } diff --git a/p2p/testing/protocolsession.go b/p2p/testing/protocolsession.go index 398f27040d..7cb59a3e1e 100644 --- a/p2p/testing/protocolsession.go +++ b/p2p/testing/protocolsession.go @@ -78,10 +78,10 @@ type Disconnect struct { } // trigger sends messages from peers -func (self *ProtocolSession) trigger(trig Trigger) error { - simNode, ok := self.adapter.GetNode(trig.Peer) +func (s *ProtocolSession) trigger(trig Trigger) error { + simNode, ok := s.adapter.GetNode(trig.Peer) if !ok { - return fmt.Errorf("trigger: peer %v does not exist (1- %v)", trig.Peer, len(self.IDs)) + return fmt.Errorf("trigger: peer %v does not exist (1- %v)", trig.Peer, len(s.IDs)) } mockNode, ok := simNode.Services()[0].(*mockNode) if !ok { @@ -107,7 +107,7 @@ func (self *ProtocolSession) trigger(trig Trigger) error { } // expect checks an expectation of a message sent out by the pivot node -func (self *ProtocolSession) expect(exps []Expect) error { +func (s *ProtocolSession) expect(exps []Expect) error { // construct a map of expectations for each node peerExpects := make(map[discover.NodeID][]Expect) for _, exp := range exps { @@ -120,9 +120,9 @@ func (self *ProtocolSession) expect(exps []Expect) error { // construct a map of mockNodes for each node mockNodes := make(map[discover.NodeID]*mockNode) for nodeID := range peerExpects { - simNode, ok := self.adapter.GetNode(nodeID) + simNode, ok := s.adapter.GetNode(nodeID) if !ok { - return fmt.Errorf("trigger: peer %v does not exist (1- %v)", nodeID, len(self.IDs)) + return fmt.Errorf("trigger: peer %v does not exist (1- %v)", nodeID, len(s.IDs)) } mockNode, ok := simNode.Services()[0].(*mockNode) if !ok { @@ -202,9 +202,9 @@ func (self *ProtocolSession) expect(exps []Expect) error { } // TestExchanges tests a series of exchanges against the session -func (self *ProtocolSession) TestExchanges(exchanges ...Exchange) error { +func (s *ProtocolSession) TestExchanges(exchanges ...Exchange) error { for i, e := range exchanges { - if err := self.testExchange(e); err != nil { + if err := s.testExchange(e); err != nil { return fmt.Errorf("exchange #%d %q: %v", i, e.Label, err) } log.Trace(fmt.Sprintf("exchange #%d %q: run successfully", i, e.Label)) @@ -214,14 +214,14 @@ func (self *ProtocolSession) TestExchanges(exchanges ...Exchange) error { // testExchange tests a single Exchange. // Default timeout value is 2 seconds. -func (self *ProtocolSession) testExchange(e Exchange) error { +func (s *ProtocolSession) testExchange(e Exchange) error { errc := make(chan error) done := make(chan struct{}) defer close(done) go func() { for _, trig := range e.Triggers { - err := self.trigger(trig) + err := s.trigger(trig) if err != nil { errc <- err return @@ -229,7 +229,7 @@ func (self *ProtocolSession) testExchange(e Exchange) error { } select { - case errc <- self.expect(e.Expects): + case errc <- s.expect(e.Expects): case <-done: } }() @@ -250,7 +250,7 @@ func (self *ProtocolSession) testExchange(e Exchange) error { // TestDisconnected tests the disconnections given as arguments // the disconnect structs describe what disconnect error is expected on which peer -func (self *ProtocolSession) TestDisconnected(disconnects ...*Disconnect) error { +func (s *ProtocolSession) TestDisconnected(disconnects ...*Disconnect) error { expects := make(map[discover.NodeID]error) for _, disconnect := range disconnects { expects[disconnect.Peer] = disconnect.Error @@ -259,7 +259,7 @@ func (self *ProtocolSession) TestDisconnected(disconnects ...*Disconnect) error timeout := time.After(time.Second) for len(expects) > 0 { select { - case event := <-self.events: + case event := <-s.events: if event.Type != p2p.PeerEventTypeDrop { continue } diff --git a/p2p/testing/protocoltester.go b/p2p/testing/protocoltester.go index 52bca5f1e6..d6b75de02a 100644 --- a/p2p/testing/protocoltester.go +++ b/p2p/testing/protocoltester.go @@ -101,24 +101,24 @@ func NewProtocolTester(t *testing.T, id discover.NodeID, n int, run func(*p2p.Pe } // Stop stops the p2p server -func (self *ProtocolTester) Stop() error { - self.Server.Stop() +func (t *ProtocolTester) Stop() error { + t.Server.Stop() return nil } // Connect brings up the remote peer node and connects it using the // p2p/simulations network connection with the in memory network adapter -func (self *ProtocolTester) Connect(selfID discover.NodeID, peers ...*adapters.NodeConfig) { +func (t *ProtocolTester) Connect(selfID discover.NodeID, peers ...*adapters.NodeConfig) { for _, peer := range peers { log.Trace(fmt.Sprintf("start node %v", peer.ID)) - if _, err := self.network.NewNodeWithConfig(peer); err != nil { + if _, err := t.network.NewNodeWithConfig(peer); err != nil { panic(fmt.Sprintf("error starting peer %v: %v", peer.ID, err)) } - if err := self.network.Start(peer.ID); err != nil { + if err := t.network.Start(peer.ID); err != nil { panic(fmt.Sprintf("error starting peer %v: %v", peer.ID, err)) } log.Trace(fmt.Sprintf("connect to %v", peer.ID)) - if err := self.network.Connect(selfID, peer.ID); err != nil { + if err := t.network.Connect(selfID, peer.ID); err != nil { panic(fmt.Sprintf("error connecting to peer %v: %v", peer.ID, err)) } } diff --git a/params/version.go b/params/version.go index 689777b592..f237a0847b 100644 --- a/params/version.go +++ b/params/version.go @@ -23,7 +23,7 @@ import ( const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 8 // Minor version component of the current release - VersionPatch = 6 // Patch version component of the current release + VersionPatch = 9 // Patch version component of the current release VersionMeta = "CLO stable" // Version metadata to append to the version string ) diff --git a/rlp/decode.go b/rlp/decode.go index 60d9dab2b5..dbbe599597 100644 --- a/rlp/decode.go +++ b/rlp/decode.go @@ -29,6 +29,23 @@ import ( ) var ( + // EOL is returned when the end of the current list + // has been reached during streaming. + EOL = errors.New("rlp: end of list") + + // Actual Errors + ErrExpectedString = errors.New("rlp: expected String or Byte") + ErrExpectedList = errors.New("rlp: expected List") + ErrCanonInt = errors.New("rlp: non-canonical integer format") + ErrCanonSize = errors.New("rlp: non-canonical size information") + ErrElemTooLarge = errors.New("rlp: element is larger than containing list") + ErrValueTooLarge = errors.New("rlp: value size exceeds available input length") + ErrMoreThanOneValue = errors.New("rlp: input contains more than one value") + + // internal errors + errNotInList = errors.New("rlp: call of ListEnd outside of any list") + errNotAtEOL = errors.New("rlp: call of ListEnd not positioned at EOL") + errUintOverflow = errors.New("rlp: uint overflow") errNoPointer = errors.New("rlp: interface given to Decode must be a pointer") errDecodeIntoNil = errors.New("rlp: pointer given to Decode must not be nil") ) @@ -274,9 +291,8 @@ func makeListDecoder(typ reflect.Type, tag tags) (decoder, error) { if etype.Kind() == reflect.Uint8 && !reflect.PtrTo(etype).Implements(decoderInterface) { if typ.Kind() == reflect.Array { return decodeByteArray, nil - } else { - return decodeByteSlice, nil } + return decodeByteSlice, nil } etypeinfo, err := cachedTypeInfo1(etype, tags{}) if err != nil { @@ -555,29 +571,6 @@ func (k Kind) String() string { } } -var ( - // EOL is returned when the end of the current list - // has been reached during streaming. - EOL = errors.New("rlp: end of list") - - // Actual Errors - ErrExpectedString = errors.New("rlp: expected String or Byte") - ErrExpectedList = errors.New("rlp: expected List") - ErrCanonInt = errors.New("rlp: non-canonical integer format") - ErrCanonSize = errors.New("rlp: non-canonical size information") - ErrElemTooLarge = errors.New("rlp: element is larger than containing list") - ErrValueTooLarge = errors.New("rlp: value size exceeds available input length") - - // This error is reported by DecodeBytes if the slice contains - // additional data after the first RLP value. - ErrMoreThanOneValue = errors.New("rlp: input contains more than one value") - - // internal errors - errNotInList = errors.New("rlp: call of ListEnd outside of any list") - errNotAtEOL = errors.New("rlp: call of ListEnd not positioned at EOL") - errUintOverflow = errors.New("rlp: uint overflow") -) - // ByteReader must be implemented by any input reader for a Stream. It // is implemented by e.g. bufio.Reader and bytes.Reader. type ByteReader interface { diff --git a/rlp/encode.go b/rlp/encode.go index 44592c2f53..445b4b5b21 100644 --- a/rlp/encode.go +++ b/rlp/encode.go @@ -92,7 +92,7 @@ func Encode(w io.Writer, val interface{}) error { return eb.toWriter(w) } -// EncodeBytes returns the RLP encoding of val. +// EncodeToBytes returns the RLP encoding of val. // Please see the documentation of Encode for the encoding rules. func EncodeToBytes(val interface{}) ([]byte, error) { eb := encbufPool.Get().(*encbuf) @@ -104,7 +104,7 @@ func EncodeToBytes(val interface{}) ([]byte, error) { return eb.toBytes(), nil } -// EncodeReader returns a reader from which the RLP encoding of val +// EncodeToReader returns a reader from which the RLP encoding of val // can be read. The returned size is the total size of the encoded // data. // @@ -151,11 +151,10 @@ func puthead(buf []byte, smalltag, largetag byte, size uint64) int { if size < 56 { buf[0] = smalltag + byte(size) return 1 - } else { - sizesize := putint(buf[1:], size) - buf[0] = largetag + byte(sizesize) - return sizesize + 1 } + sizesize := putint(buf[1:], size) + buf[0] = largetag + byte(sizesize) + return sizesize + 1 } // encbufs are pooled. @@ -218,7 +217,7 @@ func (w *encbuf) list() *listhead { func (w *encbuf) listEnd(lh *listhead) { lh.size = w.size() - lh.offset - lh.size if lh.size < 56 { - w.lhsize += 1 // length encoded into kind tag + w.lhsize++ // length encoded into kind tag } else { w.lhsize += 1 + intsize(uint64(lh.size)) } @@ -322,10 +321,9 @@ func (r *encReader) next() []byte { p := r.buf.str[r.strpos:head.offset] r.strpos += sizebefore return p - } else { - r.lhpos++ - return head.encode(r.buf.sizebuf) } + r.lhpos++ + return head.encode(r.buf.sizebuf) case r.strpos < len(r.buf.str): // String data at the end, after all list headers. @@ -576,9 +574,8 @@ func makePtrWriter(typ reflect.Type) (writer, error) { writer := func(val reflect.Value, w *encbuf) error { if val.IsNil() { return nilfunc(w) - } else { - return etypeinfo.writer(val.Elem(), w) } + return etypeinfo.writer(val.Elem(), w) } return writer, err } diff --git a/rpc/http.go b/rpc/http.go index 0c2e060fb4..feaa7348c4 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -191,7 +191,7 @@ func (srv *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { defer codec.Close() w.Header().Set("content-type", contentType) - srv.ServeSingleRequest(codec, OptionMethodInvocation, ctx) + srv.ServeSingleRequest(ctx, codec, OptionMethodInvocation) } // validateRequest returns a non-zero response code and error message if the diff --git a/rpc/inproc.go b/rpc/inproc.go index 595a7ca651..cbe65d10e7 100644 --- a/rpc/inproc.go +++ b/rpc/inproc.go @@ -21,7 +21,7 @@ import ( "net" ) -// NewInProcClient attaches an in-process connection to the given RPC server. +// DialInProc attaches an in-process connection to the given RPC server. func DialInProc(handler *Server) *Client { initctx := context.Background() c, _ := newClient(initctx, func(context.Context) (net.Conn, error) { diff --git a/rpc/server.go b/rpc/server.go index 29eccad887..b8d42789ca 100644 --- a/rpc/server.go +++ b/rpc/server.go @@ -125,7 +125,7 @@ func (s *Server) RegisterName(name string, rcvr interface{}) error { // If singleShot is true it will process a single request, otherwise it will handle // requests until the codec returns an error when reading a request (in most cases // an EOF). It executes requests in parallel when singleShot is false. -func (s *Server) serveRequest(codec ServerCodec, singleShot bool, options CodecOption, ctx context.Context) error { +func (s *Server) serveRequest(ctx context.Context, codec ServerCodec, singleShot bool, options CodecOption) error { var pend sync.WaitGroup defer func() { @@ -216,14 +216,14 @@ func (s *Server) serveRequest(codec ServerCodec, singleShot bool, options CodecO // stopped. In either case the codec is closed. func (s *Server) ServeCodec(codec ServerCodec, options CodecOption) { defer codec.Close() - s.serveRequest(codec, false, options, context.Background()) + s.serveRequest(context.Background(), codec, false, options) } // ServeSingleRequest reads and processes a single RPC request from the given codec. It will not // close the codec unless a non-recoverable error has occurred. Note, this method will return after // a single request has been processed! -func (s *Server) ServeSingleRequest(codec ServerCodec, options CodecOption, ctx context.Context) { - s.serveRequest(codec, true, options, ctx) +func (s *Server) ServeSingleRequest(ctx context.Context, codec ServerCodec, options CodecOption) { + s.serveRequest(ctx, codec, true, options) } // Stop will stop reading new requests, wait for stopPendingRequestTimeout to allow pending requests to finish, diff --git a/signer/core/abihelper.go b/signer/core/abihelper.go index 67a088fc35..54ccfe7750 100644 --- a/signer/core/abihelper.go +++ b/signer/core/abihelper.go @@ -94,13 +94,12 @@ func parseCallData(calldata []byte, abidata string) (*decodedCallData, error) { for n, argument := range method.Inputs { if err != nil { return nil, fmt.Errorf("Failed to decode argument %d (signature %v): %v", n, method.Sig(), err) - } else { - decodedArg := decodedArgument{ - soltype: argument, - value: v[n], - } - decoded.inputs = append(decoded.inputs, decodedArg) } + decodedArg := decodedArgument{ + soltype: argument, + value: v[n], + } + decoded.inputs = append(decoded.inputs, decodedArg) } // We're finished decoding the data. At this point, we encode the decoded data to see if it matches with the @@ -240,7 +239,7 @@ func (db *AbiDb) saveCustomAbi(selector, signature string) error { return err } -// Adds a signature to the database, if custom database saving is enabled. +// AddSignature to the database, if custom database saving is enabled. // OBS: This method does _not_ validate the correctness of the data, // it is assumed that the caller has already done so func (db *AbiDb) AddSignature(selector string, data []byte) error { diff --git a/signer/core/api.go b/signer/core/api.go index 3b765a6ebf..71c38abd5e 100644 --- a/signer/core/api.go +++ b/signer/core/api.go @@ -474,7 +474,7 @@ func (api *SignerAPI) Export(ctx context.Context, addr common.Address) (json.Raw return ioutil.ReadFile(wallet.URL().Path) } -// Imports tries to import the given keyJSON in the local keystore. The keyJSON data is expected to be +// Import tries to import the given keyJSON in the local keystore. The keyJSON data is expected to be // in web3 keystore format. It will decrypt the keyJSON with the given passphrase and on successful // decryption it will encrypt the key with the given newPassphrase and store it in the keystore. func (api *SignerAPI) Import(ctx context.Context, keyJSON json.RawMessage) (Account, error) { diff --git a/signer/core/cliui.go b/signer/core/cliui.go index 3d58934f46..85781f81ec 100644 --- a/signer/core/cliui.go +++ b/signer/core/cliui.go @@ -13,6 +13,7 @@ // // You should have received a copy of the GNU General Public License // along with go-ethereum. If not, see . + package core import ( diff --git a/signer/core/types.go b/signer/core/types.go index 5f26490373..4b224ffa69 100644 --- a/signer/core/types.go +++ b/signer/core/types.go @@ -73,8 +73,8 @@ type SendTxArgs struct { Input *hexutil.Bytes `json:"input"` } -func (t SendTxArgs) String() string { - s, err := json.Marshal(t) +func (args SendTxArgs) String() string { + s, err := json.Marshal(args) if err == nil { return string(s) } diff --git a/signer/core/validation.go b/signer/core/validation.go index daf132431d..8e0c737eb0 100644 --- a/signer/core/validation.go +++ b/signer/core/validation.go @@ -128,7 +128,7 @@ func (v *Validator) validate(msgs *ValidationMessages, txargs *SendTxArgs, metho if len(data) == 0 { if txargs.Value.ToInt().Cmp(big.NewInt(0)) > 0 { // Sending ether into black hole - return errors.New(`Tx will create contract with value but empty code!`) + return errors.New("Tx will create contract with value but empty code!") } // No value submitted at least msgs.crit("Tx will create contract with empty code!") diff --git a/signer/rules/rules.go b/signer/rules/rules.go index bb000d4371..862618d4af 100644 --- a/signer/rules/rules.go +++ b/signer/rules/rules.go @@ -46,17 +46,17 @@ func consoleOutput(call otto.FunctionCall) otto.Value { return otto.Value{} } -// rulesetUi provides an implementation of SignerUI that evaluates a javascript +// rulesetUI provides an implementation of SignerUI that evaluates a javascript // file for each defined UI-method -type rulesetUi struct { +type rulesetUI struct { next core.SignerUI // The next handler, for manual processing storage storage.Storage credentials storage.Storage jsRules string // The rules to use } -func NewRuleEvaluator(next core.SignerUI, jsbackend, credentialsBackend storage.Storage) (*rulesetUi, error) { - c := &rulesetUi{ +func NewRuleEvaluator(next core.SignerUI, jsbackend, credentialsBackend storage.Storage) (*rulesetUI, error) { + c := &rulesetUI{ next: next, storage: jsbackend, credentials: credentialsBackend, @@ -66,11 +66,11 @@ func NewRuleEvaluator(next core.SignerUI, jsbackend, credentialsBackend storage. return c, nil } -func (r *rulesetUi) Init(javascriptRules string) error { +func (r *rulesetUI) Init(javascriptRules string) error { r.jsRules = javascriptRules return nil } -func (r *rulesetUi) execute(jsfunc string, jsarg interface{}) (otto.Value, error) { +func (r *rulesetUI) execute(jsfunc string, jsarg interface{}) (otto.Value, error) { // Instantiate a fresh vm engine every time vm := otto.New() @@ -115,7 +115,7 @@ func (r *rulesetUi) execute(jsfunc string, jsarg interface{}) (otto.Value, error return vm.Run(call) } -func (r *rulesetUi) checkApproval(jsfunc string, jsarg []byte, err error) (bool, error) { +func (r *rulesetUI) checkApproval(jsfunc string, jsarg []byte, err error) (bool, error) { if err != nil { return false, err } @@ -139,7 +139,7 @@ func (r *rulesetUi) checkApproval(jsfunc string, jsarg []byte, err error) (bool, return false, fmt.Errorf("Unknown response") } -func (r *rulesetUi) ApproveTx(request *core.SignTxRequest) (core.SignTxResponse, error) { +func (r *rulesetUI) ApproveTx(request *core.SignTxRequest) (core.SignTxResponse, error) { jsonreq, err := json.Marshal(request) approved, err := r.checkApproval("ApproveTx", jsonreq, err) if err != nil { @@ -158,11 +158,11 @@ func (r *rulesetUi) ApproveTx(request *core.SignTxRequest) (core.SignTxResponse, return core.SignTxResponse{Approved: false}, err } -func (r *rulesetUi) lookupPassword(address common.Address) string { +func (r *rulesetUI) lookupPassword(address common.Address) string { return r.credentials.Get(strings.ToLower(address.String())) } -func (r *rulesetUi) ApproveSignData(request *core.SignDataRequest) (core.SignDataResponse, error) { +func (r *rulesetUI) ApproveSignData(request *core.SignDataRequest) (core.SignDataResponse, error) { jsonreq, err := json.Marshal(request) approved, err := r.checkApproval("ApproveSignData", jsonreq, err) if err != nil { @@ -175,7 +175,7 @@ func (r *rulesetUi) ApproveSignData(request *core.SignDataRequest) (core.SignDat return core.SignDataResponse{Approved: false, Password: ""}, err } -func (r *rulesetUi) ApproveExport(request *core.ExportRequest) (core.ExportResponse, error) { +func (r *rulesetUI) ApproveExport(request *core.ExportRequest) (core.ExportResponse, error) { jsonreq, err := json.Marshal(request) approved, err := r.checkApproval("ApproveExport", jsonreq, err) if err != nil { @@ -188,13 +188,13 @@ func (r *rulesetUi) ApproveExport(request *core.ExportRequest) (core.ExportRespo return core.ExportResponse{Approved: false}, err } -func (r *rulesetUi) ApproveImport(request *core.ImportRequest) (core.ImportResponse, error) { +func (r *rulesetUI) ApproveImport(request *core.ImportRequest) (core.ImportResponse, error) { // This cannot be handled by rules, requires setting a password // dispatch to next return r.next.ApproveImport(request) } -func (r *rulesetUi) ApproveListing(request *core.ListRequest) (core.ListResponse, error) { +func (r *rulesetUI) ApproveListing(request *core.ListRequest) (core.ListResponse, error) { jsonreq, err := json.Marshal(request) approved, err := r.checkApproval("ApproveListing", jsonreq, err) if err != nil { @@ -207,22 +207,22 @@ func (r *rulesetUi) ApproveListing(request *core.ListRequest) (core.ListResponse return core.ListResponse{}, err } -func (r *rulesetUi) ApproveNewAccount(request *core.NewAccountRequest) (core.NewAccountResponse, error) { +func (r *rulesetUI) ApproveNewAccount(request *core.NewAccountRequest) (core.NewAccountResponse, error) { // This cannot be handled by rules, requires setting a password // dispatch to next return r.next.ApproveNewAccount(request) } -func (r *rulesetUi) ShowError(message string) { +func (r *rulesetUI) ShowError(message string) { log.Error(message) r.next.ShowError(message) } -func (r *rulesetUi) ShowInfo(message string) { +func (r *rulesetUI) ShowInfo(message string) { log.Info(message) r.next.ShowInfo(message) } -func (r *rulesetUi) OnSignerStartup(info core.StartupInfo) { +func (r *rulesetUI) OnSignerStartup(info core.StartupInfo) { jsonInfo, err := json.Marshal(info) if err != nil { log.Warn("failed marshalling data", "data", info) @@ -235,7 +235,7 @@ func (r *rulesetUi) OnSignerStartup(info core.StartupInfo) { } } -func (r *rulesetUi) OnApprovedTx(tx ethapi.SignTransactionResult) { +func (r *rulesetUI) OnApprovedTx(tx ethapi.SignTransactionResult) { jsonTx, err := json.Marshal(tx) if err != nil { log.Warn("failed marshalling transaction", "tx", tx) diff --git a/signer/rules/rules_test.go b/signer/rules/rules_test.go index 684ba4107b..8394431163 100644 --- a/signer/rules/rules_test.go +++ b/signer/rules/rules_test.go @@ -33,18 +33,18 @@ import ( const JS = ` /** -This is an example implementation of a Javascript rule file. +This is an example implementation of a Javascript rule file. -When the signer receives a request over the external API, the corresponding method is evaluated. -Three things can happen: +When the signer receives a request over the external API, the corresponding method is evaluated. +Three things can happen: -1. The method returns "Approve". This means the operation is permitted. -2. The method returns "Reject". This means the operation is rejected. +1. The method returns "Approve". This means the operation is permitted. +2. The method returns "Reject". This means the operation is rejected. 3. Anything else; other return values [*], method not implemented or exception occurred during processing. This means -that the operation will continue to manual processing, via the regular UI method chosen by the user. +that the operation will continue to manual processing, via the regular UI method chosen by the user. -[*] Note: Future version of the ruleset may use more complex json-based returnvalues, making it possible to not -only respond Approve/Reject/Manual, but also modify responses. For example, choose to list only one, but not all +[*] Note: Future version of the ruleset may use more complex json-based returnvalues, making it possible to not +only respond Approve/Reject/Manual, but also modify responses. For example, choose to list only one, but not all accounts in a list-request. The points above will continue to hold for non-json based responses ("Approve"/"Reject"). **/ @@ -72,49 +72,49 @@ func mixAddr(a string) (*common.MixedcaseAddress, error) { return common.NewMixedcaseAddressFromString(a) } -type alwaysDenyUi struct{} +type alwaysDenyUI struct{} -func (alwaysDenyUi) OnSignerStartup(info core.StartupInfo) { +func (alwaysDenyUI) OnSignerStartup(info core.StartupInfo) { } -func (alwaysDenyUi) ApproveTx(request *core.SignTxRequest) (core.SignTxResponse, error) { +func (alwaysDenyUI) ApproveTx(request *core.SignTxRequest) (core.SignTxResponse, error) { return core.SignTxResponse{Transaction: request.Transaction, Approved: false, Password: ""}, nil } -func (alwaysDenyUi) ApproveSignData(request *core.SignDataRequest) (core.SignDataResponse, error) { +func (alwaysDenyUI) ApproveSignData(request *core.SignDataRequest) (core.SignDataResponse, error) { return core.SignDataResponse{Approved: false, Password: ""}, nil } -func (alwaysDenyUi) ApproveExport(request *core.ExportRequest) (core.ExportResponse, error) { +func (alwaysDenyUI) ApproveExport(request *core.ExportRequest) (core.ExportResponse, error) { return core.ExportResponse{Approved: false}, nil } -func (alwaysDenyUi) ApproveImport(request *core.ImportRequest) (core.ImportResponse, error) { +func (alwaysDenyUI) ApproveImport(request *core.ImportRequest) (core.ImportResponse, error) { return core.ImportResponse{Approved: false, OldPassword: "", NewPassword: ""}, nil } -func (alwaysDenyUi) ApproveListing(request *core.ListRequest) (core.ListResponse, error) { +func (alwaysDenyUI) ApproveListing(request *core.ListRequest) (core.ListResponse, error) { return core.ListResponse{Accounts: nil}, nil } -func (alwaysDenyUi) ApproveNewAccount(request *core.NewAccountRequest) (core.NewAccountResponse, error) { +func (alwaysDenyUI) ApproveNewAccount(request *core.NewAccountRequest) (core.NewAccountResponse, error) { return core.NewAccountResponse{Approved: false, Password: ""}, nil } -func (alwaysDenyUi) ShowError(message string) { +func (alwaysDenyUI) ShowError(message string) { panic("implement me") } -func (alwaysDenyUi) ShowInfo(message string) { +func (alwaysDenyUI) ShowInfo(message string) { panic("implement me") } -func (alwaysDenyUi) OnApprovedTx(tx ethapi.SignTransactionResult) { +func (alwaysDenyUI) OnApprovedTx(tx ethapi.SignTransactionResult) { panic("implement me") } -func initRuleEngine(js string) (*rulesetUi, error) { - r, err := NewRuleEvaluator(&alwaysDenyUi{}, storage.NewEphemeralStorage(), storage.NewEphemeralStorage()) +func initRuleEngine(js string) (*rulesetUI, error) { + r, err := NewRuleEvaluator(&alwaysDenyUI{}, storage.NewEphemeralStorage(), storage.NewEphemeralStorage()) if err != nil { return nil, fmt.Errorf("failed to create js engine: %v", err) } @@ -196,59 +196,59 @@ func TestSignTxRequest(t *testing.T) { } } -type dummyUi struct { +type dummyUI struct { calls []string } -func (d *dummyUi) ApproveTx(request *core.SignTxRequest) (core.SignTxResponse, error) { +func (d *dummyUI) ApproveTx(request *core.SignTxRequest) (core.SignTxResponse, error) { d.calls = append(d.calls, "ApproveTx") return core.SignTxResponse{}, core.ErrRequestDenied } -func (d *dummyUi) ApproveSignData(request *core.SignDataRequest) (core.SignDataResponse, error) { +func (d *dummyUI) ApproveSignData(request *core.SignDataRequest) (core.SignDataResponse, error) { d.calls = append(d.calls, "ApproveSignData") return core.SignDataResponse{}, core.ErrRequestDenied } -func (d *dummyUi) ApproveExport(request *core.ExportRequest) (core.ExportResponse, error) { +func (d *dummyUI) ApproveExport(request *core.ExportRequest) (core.ExportResponse, error) { d.calls = append(d.calls, "ApproveExport") return core.ExportResponse{}, core.ErrRequestDenied } -func (d *dummyUi) ApproveImport(request *core.ImportRequest) (core.ImportResponse, error) { +func (d *dummyUI) ApproveImport(request *core.ImportRequest) (core.ImportResponse, error) { d.calls = append(d.calls, "ApproveImport") return core.ImportResponse{}, core.ErrRequestDenied } -func (d *dummyUi) ApproveListing(request *core.ListRequest) (core.ListResponse, error) { +func (d *dummyUI) ApproveListing(request *core.ListRequest) (core.ListResponse, error) { d.calls = append(d.calls, "ApproveListing") return core.ListResponse{}, core.ErrRequestDenied } -func (d *dummyUi) ApproveNewAccount(request *core.NewAccountRequest) (core.NewAccountResponse, error) { +func (d *dummyUI) ApproveNewAccount(request *core.NewAccountRequest) (core.NewAccountResponse, error) { d.calls = append(d.calls, "ApproveNewAccount") return core.NewAccountResponse{}, core.ErrRequestDenied } -func (d *dummyUi) ShowError(message string) { +func (d *dummyUI) ShowError(message string) { d.calls = append(d.calls, "ShowError") } -func (d *dummyUi) ShowInfo(message string) { +func (d *dummyUI) ShowInfo(message string) { d.calls = append(d.calls, "ShowInfo") } -func (d *dummyUi) OnApprovedTx(tx ethapi.SignTransactionResult) { +func (d *dummyUI) OnApprovedTx(tx ethapi.SignTransactionResult) { d.calls = append(d.calls, "OnApprovedTx") } -func (d *dummyUi) OnSignerStartup(info core.StartupInfo) { +func (d *dummyUI) OnSignerStartup(info core.StartupInfo) { } //TestForwarding tests that the rule-engine correctly dispatches requests to the next caller func TestForwarding(t *testing.T) { js := "" - ui := &dummyUi{make([]string, 0)} + ui := &dummyUI{make([]string, 0)} jsBackend := storage.NewEphemeralStorage() credBackend := storage.NewEphemeralStorage() r, err := NewRuleEvaluator(ui, jsBackend, credBackend) @@ -308,22 +308,22 @@ func TestStorage(t *testing.T) { function testStorage(){ storage.Put("mykey", "myvalue") a = storage.Get("mykey") - + storage.Put("mykey", ["a", "list"]) // Should result in "a,list" a += storage.Get("mykey") - + storage.Put("mykey", {"an": "object"}) // Should result in "[object Object]" a += storage.Get("mykey") - + storage.Put("mykey", JSON.stringify({"an": "object"})) // Should result in '{"an":"object"}' a += storage.Get("mykey") a += storage.Get("missingkey") //Missing keys should result in empty string storage.Put("","missing key==noop") // Can't store with 0-length key a += storage.Get("") // Should result in '' - + var b = new BigNumber(2) var c = new BigNumber(16)//"0xf0",16) var d = b.plus(c) @@ -361,7 +361,7 @@ const ExampleTxWindow = ` if(str.slice(0,2) == "0x"){ return new BigNumber(str.slice(2),16)} return new BigNumber(str) } - + // Time window: 1 week var window = 1000* 3600*24*7; @@ -370,7 +370,7 @@ const ExampleTxWindow = ` function isLimitOk(transaction){ var value = big(transaction.value) - // Start of our window function + // Start of our window function var windowstart = new Date().getTime() - window; var txs = []; @@ -382,17 +382,17 @@ const ExampleTxWindow = ` // First, remove all that have passed out of the time-window var newtxs = txs.filter(function(tx){return tx.tstamp > windowstart}); console.log(txs, newtxs.length); - + // Secondly, aggregate the current sum sum = new BigNumber(0) sum = newtxs.reduce(function(agg, tx){ return big(tx.value).plus(agg)}, sum); console.log("ApproveTx > Sum so far", sum); console.log("ApproveTx > Requested", value.toNumber()); - + // Would we exceed weekly limit ? return sum.plus(value).lt(limit) - + } function ApproveTx(r){ console.log(r) @@ -405,14 +405,14 @@ const ExampleTxWindow = ` /** * OnApprovedTx(str) is called when a transaction has been approved and signed. The parameter - * 'response_str' contains the return value that will be sent to the external caller. - * The return value from this method is ignore - the reason for having this callback is to allow the - * ruleset to keep track of approved transactions. + * 'response_str' contains the return value that will be sent to the external caller. + * The return value from this method is ignore - the reason for having this callback is to allow the + * ruleset to keep track of approved transactions. * - * When implementing rate-limited rules, this callback should be used. + * When implementing rate-limited rules, this callback should be used. * If a rule responds with neither 'Approve' nor 'Reject' - the tx goes to manual processing. If the user * then accepts the transaction, this method will be called. - * + * * TLDR; Use this method to keep track of signed transactions, instead of using the data in ApproveTx. */ function OnApprovedTx(resp){ diff --git a/signer/storage/aes_gcm_storage.go b/signer/storage/aes_gcm_storage.go index c01fb6d1e7..413bbfe55b 100644 --- a/signer/storage/aes_gcm_storage.go +++ b/signer/storage/aes_gcm_storage.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU General Public License // along with go-ethereum. If not, see . // + package storage import ( @@ -106,10 +107,8 @@ func (s *AESEncryptedStorage) readEncryptedStorage() (map[string]storedCredentia if os.IsNotExist(err) { // Doesn't exist yet return creds, nil - - } else { - log.Warn("Failed to read encrypted storage", "err", err, "file", s.filename) } + log.Warn("Failed to read encrypted storage", "err", err, "file", s.filename) } if err = json.Unmarshal(raw, &creds); err != nil { log.Warn("Failed to unmarshal encrypted storage", "err", err, "file", s.filename) diff --git a/tests/block_test_util.go b/tests/block_test_util.go index 8723e12d66..387f3b9949 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -98,13 +98,13 @@ func (t *BlockTest) Run() error { } // import pre accounts & construct test genesis block & state root - db, _ := ethdb.NewMemDatabase() + db := ethdb.NewMemDatabase() gblock, err := t.genesis(config).Commit(db) if err != nil { return err } if gblock.Hash() != t.json.Genesis.Hash { - return fmt.Errorf("genesis block hash doesn't match test: computed=%x, test=%x\n", gblock.Hash().Bytes()[:6], t.json.Genesis.Hash[:6]) + return fmt.Errorf("genesis block hash doesn't match test: computed=%x, test=%x", gblock.Hash().Bytes()[:6], t.json.Genesis.Hash[:6]) } if gblock.Root() != t.json.Genesis.StateRoot { return fmt.Errorf("genesis block state root does not match test: computed=%x, test=%x", gblock.Root().Bytes()[:6], t.json.Genesis.StateRoot[:6]) diff --git a/tests/init.go b/tests/init.go index 33655e044d..13ed3081b3 100644 --- a/tests/init.go +++ b/tests/init.go @@ -23,7 +23,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/params" ) -// This table defines supported forks and their chain config. +// Forks table defines supported forks and their chain config. var Forks = map[string]*params.ChainConfig{ "Frontier": { ChainId: big.NewInt(1), diff --git a/tests/init_test.go b/tests/init_test.go index 3620f186f2..340a9f0286 100644 --- a/tests/init_test.go +++ b/tests/init_test.go @@ -42,7 +42,7 @@ var ( difficultyTestDir = filepath.Join(baseDir, "BasicTests") ) -func readJson(reader io.Reader, value interface{}) error { +func readJSON(reader io.Reader, value interface{}) error { data, err := ioutil.ReadAll(reader) if err != nil { return fmt.Errorf("error reading JSON file: %v", err) @@ -57,14 +57,14 @@ func readJson(reader io.Reader, value interface{}) error { return nil } -func readJsonFile(fn string, value interface{}) error { +func readJSONFile(fn string, value interface{}) error { file, err := os.Open(fn) if err != nil { return err } defer file.Close() - err = readJson(file, value) + err = readJSON(file, value) if err != nil { return fmt.Errorf("%s in file %s", err.Error(), fn) } @@ -169,9 +169,8 @@ func (tm *testMatcher) checkFailure(t *testing.T, name string, err error) error if err != nil { t.Logf("error: %v", err) return nil - } else { - return fmt.Errorf("test succeeded unexpectedly") } + return fmt.Errorf("test succeeded unexpectedly") } return err } @@ -213,7 +212,7 @@ func (tm *testMatcher) runTestFile(t *testing.T, path, name string, runTest inte // Load the file as map[string]. m := makeMapFromTestFunc(runTest) - if err := readJsonFile(path, m.Addr().Interface()); err != nil { + if err := readJSONFile(path, m.Addr().Interface()); err != nil { t.Fatal(err) } diff --git a/tests/state_test_util.go b/tests/state_test_util.go index c85dc72f5e..33f1905a6f 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -126,8 +126,7 @@ func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config) (*state.StateD return nil, UnsupportedForkError{subtest.Fork} } block := t.genesis(config).ToBlock(nil) - db, _ := ethdb.NewMemDatabase() - statedb := MakePreState(db, t.json.Pre) + statedb := MakePreState(ethdb.NewMemDatabase(), t.json.Pre) post := t.json.Post[subtest.Fork][subtest.Index] msg, err := t.json.Tx.toMessage(post) diff --git a/tests/transaction_test_util.go b/tests/transaction_test_util.go index 82ebfea69f..882deed8dc 100644 --- a/tests/transaction_test_util.go +++ b/tests/transaction_test_util.go @@ -72,9 +72,8 @@ func (tt *TransactionTest) Run(config *params.ChainConfig) error { if err := rlp.DecodeBytes(tt.json.RLP, tx); err != nil { if tt.json.Transaction == nil { return nil - } else { - return fmt.Errorf("RLP decoding failed: %v", err) } + return fmt.Errorf("RLP decoding failed: %v", err) } // Check sender derivation. signer := types.MakeSigner(config, new(big.Int).SetUint64(uint64(tt.json.BlockNumber))) diff --git a/tests/vm_test_util.go b/tests/vm_test_util.go index cb88ad44c2..373a34a515 100644 --- a/tests/vm_test_util.go +++ b/tests/vm_test_util.go @@ -79,8 +79,7 @@ type vmExecMarshaling struct { } func (t *VMTest) Run(vmconfig vm.Config) error { - db, _ := ethdb.NewMemDatabase() - statedb := MakePreState(db, t.json.Pre) + statedb := MakePreState(ethdb.NewMemDatabase(), t.json.Pre) ret, gasRemaining, err := t.exec(statedb, vmconfig) if t.json.GasRemaining == nil { diff --git a/trie/hasher.go b/trie/hasher.go index b6a15ff4df..dcd3484173 100644 --- a/trie/hasher.go +++ b/trie/hasher.go @@ -196,12 +196,12 @@ func (h *hasher) store(n node, db *Database, force bool) (node, error) { if h.onleaf != nil { switch n := n.(type) { case *shortNode: - if child, ok := n.Val.(valueNode); ok { + if child, ok := n.Val.(valueNode); ok && child != nil { h.onleaf(child, hash) } case *fullNode: for i := 0; i < 16; i++ { - if child, ok := n.Children[i].(valueNode); ok { + if child, ok := n.Children[i].(valueNode); ok && child != nil { h.onleaf(child, hash) } } diff --git a/trie/iterator.go b/trie/iterator.go index 7ad47b7208..e30ef22cc0 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -22,6 +22,7 @@ import ( "errors" "github.com/EthereumCommonwealth/go-callisto/common" + "github.com/EthereumCommonwealth/go-callisto/rlp" ) // Iterator is a key-value trie iterator that traverses a Trie. @@ -55,31 +56,50 @@ func (it *Iterator) Next() bool { return false } +// Prove generates the Merkle proof for the leaf node the iterator is currently +// positioned on. +func (it *Iterator) Prove() [][]byte { + return it.nodeIt.LeafProof() +} + // NodeIterator is an iterator to traverse the trie pre-order. type NodeIterator interface { // Next moves the iterator to the next node. If the parameter is false, any child // nodes will be skipped. Next(bool) bool + // Error returns the error status of the iterator. Error() error // Hash returns the hash of the current node. Hash() common.Hash + // Parent returns the hash of the parent of the current node. The hash may be the one // grandparent if the immediate parent is an internal node with no hash. Parent() common.Hash + // Path returns the hex-encoded path to the current node. // Callers must not retain references to the return value after calling Next. // For leaf nodes, the last element of the path is the 'terminator symbol' 0x10. Path() []byte // Leaf returns true iff the current node is a leaf node. - // LeafBlob, LeafKey return the contents and key of the leaf node. These - // method panic if the iterator is not positioned at a leaf. - // Callers must not retain references to their return value after calling Next Leaf() bool - LeafBlob() []byte + + // LeafKey returns the key of the leaf. The method panics if the iterator is not + // positioned at a leaf. Callers must not retain references to the value after + // calling Next. LeafKey() []byte + + // LeafBlob returns the content of the leaf. The method panics if the iterator + // is not positioned at a leaf. Callers must not retain references to the value + // after calling Next. + LeafBlob() []byte + + // LeafProof returns the Merkle proof of the leaf. The method panics if the + // iterator is not positioned at a leaf. Callers must not retain references + // to the value after calling Next. + LeafProof() [][]byte } // nodeIteratorState represents the iteration state at one particular node of the @@ -99,8 +119,8 @@ type nodeIterator struct { err error // Failure set in case of an internal error in the iterator } -// iteratorEnd is stored in nodeIterator.err when iteration is done. -var iteratorEnd = errors.New("end of iteration") +// errIteratorEnd is stored in nodeIterator.err when iteration is done. +var errIteratorEnd = errors.New("end of iteration") // seekError is stored in nodeIterator.err if the initial seek has failed. type seekError struct { @@ -139,6 +159,15 @@ func (it *nodeIterator) Leaf() bool { return hasTerm(it.path) } +func (it *nodeIterator) LeafKey() []byte { + if len(it.stack) > 0 { + if _, ok := it.stack[len(it.stack)-1].node.(valueNode); ok { + return hexToKeybytes(it.path) + } + } + panic("not at leaf") +} + func (it *nodeIterator) LeafBlob() []byte { if len(it.stack) > 0 { if node, ok := it.stack[len(it.stack)-1].node.(valueNode); ok { @@ -148,10 +177,22 @@ func (it *nodeIterator) LeafBlob() []byte { panic("not at leaf") } -func (it *nodeIterator) LeafKey() []byte { +func (it *nodeIterator) LeafProof() [][]byte { if len(it.stack) > 0 { if _, ok := it.stack[len(it.stack)-1].node.(valueNode); ok { - return hexToKeybytes(it.path) + hasher := newHasher(0, 0, nil) + proofs := make([][]byte, 0, len(it.stack)) + + for i, item := range it.stack[:len(it.stack)-1] { + // Gather nodes that end up as hash nodes (or the root) + node, _, _ := hasher.hashChildren(item.node, nil) + hashed, _ := hasher.store(node, nil, false) + if _, ok := hashed.(hashNode); ok || i == 0 { + enc, _ := rlp.EncodeToBytes(node) + proofs = append(proofs, enc) + } + } + return proofs } } panic("not at leaf") @@ -162,7 +203,7 @@ func (it *nodeIterator) Path() []byte { } func (it *nodeIterator) Error() error { - if it.err == iteratorEnd { + if it.err == errIteratorEnd { return nil } if seek, ok := it.err.(seekError); ok { @@ -176,7 +217,7 @@ func (it *nodeIterator) Error() error { // sets the Error field to the encountered failure. If `descend` is false, // skips iterating over any subnodes of the current node. func (it *nodeIterator) Next(descend bool) bool { - if it.err == iteratorEnd { + if it.err == errIteratorEnd { return false } if seek, ok := it.err.(seekError); ok { @@ -201,8 +242,8 @@ func (it *nodeIterator) seek(prefix []byte) error { // Move forward until we're just before the closest match to key. for { state, parentIndex, path, err := it.peek(bytes.HasPrefix(key, it.path)) - if err == iteratorEnd { - return iteratorEnd + if err == errIteratorEnd { + return errIteratorEnd } else if err != nil { return seekError{prefix, err} } else if bytes.Compare(path, key) >= 0 { @@ -246,7 +287,7 @@ func (it *nodeIterator) peek(descend bool) (*nodeIteratorState, *int, []byte, er // No more child nodes, move back up. it.pop() } - return nil, nil, nil, iteratorEnd + return nil, nil, nil, errIteratorEnd } func (st *nodeIteratorState) resolve(tr *Trie, path []byte) error { @@ -303,7 +344,7 @@ func (it *nodeIterator) push(state *nodeIteratorState, parentIndex *int, path [] it.path = path it.stack = append(it.stack, state) if parentIndex != nil { - *parentIndex += 1 + *parentIndex++ } } @@ -361,12 +402,16 @@ func (it *differenceIterator) Leaf() bool { return it.b.Leaf() } +func (it *differenceIterator) LeafKey() []byte { + return it.b.LeafKey() +} + func (it *differenceIterator) LeafBlob() []byte { return it.b.LeafBlob() } -func (it *differenceIterator) LeafKey() []byte { - return it.b.LeafKey() +func (it *differenceIterator) LeafProof() [][]byte { + return it.b.LeafProof() } func (it *differenceIterator) Path() []byte { @@ -380,7 +425,7 @@ func (it *differenceIterator) Next(bool) bool { if !it.b.Next(true) { return false } - it.count += 1 + it.count++ if it.eof { // a has reached eof, so we just return all elements from b @@ -395,7 +440,7 @@ func (it *differenceIterator) Next(bool) bool { it.eof = true return true } - it.count += 1 + it.count++ case 1: // b is before a return true @@ -405,12 +450,12 @@ func (it *differenceIterator) Next(bool) bool { if !it.b.Next(hasHash) { return false } - it.count += 1 + it.count++ if !it.a.Next(hasHash) { it.eof = true return true } - it.count += 1 + it.count++ } } } @@ -464,12 +509,16 @@ func (it *unionIterator) Leaf() bool { return (*it.items)[0].Leaf() } +func (it *unionIterator) LeafKey() []byte { + return (*it.items)[0].LeafKey() +} + func (it *unionIterator) LeafBlob() []byte { return (*it.items)[0].LeafBlob() } -func (it *unionIterator) LeafKey() []byte { - return (*it.items)[0].LeafKey() +func (it *unionIterator) LeafProof() [][]byte { + return (*it.items)[0].LeafProof() } func (it *unionIterator) Path() []byte { @@ -504,17 +553,15 @@ func (it *unionIterator) Next(descend bool) bool { skipped := heap.Pop(it.items).(NodeIterator) // Skip the whole subtree if the nodes have hashes; otherwise just skip this node if skipped.Next(skipped.Hash() == common.Hash{}) { - it.count += 1 + it.count++ // If there are more elements, push the iterator back on the heap heap.Push(it.items, skipped) } } - if least.Next(descend) { - it.count += 1 + it.count++ heap.Push(it.items, least) } - return len(*it.items) > 0 } diff --git a/trie/iterator_test.go b/trie/iterator_test.go index 69e471947c..20c5c75684 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -289,7 +289,7 @@ func TestIteratorContinueAfterErrorDisk(t *testing.T) { testIteratorContinueA func TestIteratorContinueAfterErrorMemonly(t *testing.T) { testIteratorContinueAfterError(t, true) } func testIteratorContinueAfterError(t *testing.T, memonly bool) { - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) tr, _ := New(common.Hash{}, triedb) @@ -376,7 +376,7 @@ func TestIteratorContinueAfterSeekErrorMemonly(t *testing.T) { func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) { // Commit test trie to db, then remove the node containing "bars". - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) ctr, _ := New(common.Hash{}, triedb) diff --git a/trie/node.go b/trie/node.go index 6ce11d8c19..b5effdfa44 100644 --- a/trie/node.go +++ b/trie/node.go @@ -123,17 +123,17 @@ func decodeNode(hash, buf []byte, cachegen uint16) (node, error) { } switch c, _ := rlp.CountValues(elems); c { case 2: - n, err := decodeShort(hash, buf, elems, cachegen) + n, err := decodeShort(hash, elems, cachegen) return n, wrapError(err, "short") case 17: - n, err := decodeFull(hash, buf, elems, cachegen) + n, err := decodeFull(hash, elems, cachegen) return n, wrapError(err, "full") default: return nil, fmt.Errorf("invalid number of list elements: %v", c) } } -func decodeShort(hash, buf, elems []byte, cachegen uint16) (node, error) { +func decodeShort(hash, elems []byte, cachegen uint16) (node, error) { kbuf, rest, err := rlp.SplitString(elems) if err != nil { return nil, err @@ -155,7 +155,7 @@ func decodeShort(hash, buf, elems []byte, cachegen uint16) (node, error) { return &shortNode{key, r, flag}, nil } -func decodeFull(hash, buf, elems []byte, cachegen uint16) (*fullNode, error) { +func decodeFull(hash, elems []byte, cachegen uint16) (*fullNode, error) { n := &fullNode{flags: nodeFlag{hash: hash, gen: cachegen}} for i := 0; i < 16; i++ { cld, rest, err := decodeRef(elems, cachegen) diff --git a/trie/proof.go b/trie/proof.go index c8c0703c66..65ab3bd50a 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -102,28 +102,28 @@ func (t *SecureTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) err // VerifyProof checks merkle proofs. The given proof must contain the value for // key in a trie with the given root hash. VerifyProof returns an error if the // proof contains invalid trie nodes or the wrong value. -func VerifyProof(rootHash common.Hash, key []byte, proofDb DatabaseReader) (value []byte, err error, nodes int) { +func VerifyProof(rootHash common.Hash, key []byte, proofDb DatabaseReader) (value []byte, nodes int, err error) { key = keybytesToHex(key) wantHash := rootHash for i := 0; ; i++ { buf, _ := proofDb.Get(wantHash[:]) if buf == nil { - return nil, fmt.Errorf("proof node %d (hash %064x) missing", i, wantHash), i + return nil, i, fmt.Errorf("proof node %d (hash %064x) missing", i, wantHash) } n, err := decodeNode(wantHash[:], buf, 0) if err != nil { - return nil, fmt.Errorf("bad proof node %d: %v", i, err), i + return nil, i, fmt.Errorf("bad proof node %d: %v", i, err) } keyrest, cld := get(n, key) switch cld := cld.(type) { case nil: // The trie doesn't contain the key. - return nil, nil, i + return nil, i, nil case hashNode: key = keyrest copy(wantHash[:], cld) case valueNode: - return cld, nil, i + 1 + return cld, i + 1, nil } } } diff --git a/trie/proof_test.go b/trie/proof_test.go index 640847137f..0d519b6c0d 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -32,20 +32,46 @@ func init() { mrand.Seed(time.Now().Unix()) } +// makeProvers creates Merkle trie provers based on different implementations to +// test all variations. +func makeProvers(trie *Trie) []func(key []byte) *ethdb.MemDatabase { + var provers []func(key []byte) *ethdb.MemDatabase + + // Create a direct trie based Merkle prover + provers = append(provers, func(key []byte) *ethdb.MemDatabase { + proof := ethdb.NewMemDatabase() + trie.Prove(key, 0, proof) + return proof + }) + // Create a leaf iterator based Merkle prover + provers = append(provers, func(key []byte) *ethdb.MemDatabase { + proof := ethdb.NewMemDatabase() + if it := NewIterator(trie.NodeIterator(key)); it.Next() && bytes.Equal(key, it.Key) { + for _, p := range it.Prove() { + proof.Put(crypto.Keccak256(p), p) + } + } + return proof + }) + return provers +} + func TestProof(t *testing.T) { trie, vals := randomTrie(500) root := trie.Hash() - for _, kv := range vals { - proofs, _ := ethdb.NewMemDatabase() - if trie.Prove(kv.k, 0, proofs) != nil { - t.Fatalf("missing key %x while constructing proof", kv.k) - } - val, err, _ := VerifyProof(root, kv.k, proofs) - if err != nil { - t.Fatalf("VerifyProof error for key %x: %v\nraw proof: %v", kv.k, err, proofs) - } - if !bytes.Equal(val, kv.v) { - t.Fatalf("VerifyProof returned wrong value for key %x: got %x, want %x", kv.k, val, kv.v) + for i, prover := range makeProvers(trie) { + for _, kv := range vals { + proof := prover(kv.k) + if proof == nil { + t.Fatalf("prover %d: missing key %x while constructing proof", i, kv.k) + } + val, _, err := VerifyProof(root, kv.k, proof) + if err != nil { + t.Fatalf("prover %d: failed to verify proof for key %x: %v\nraw proof: %x", i, kv.k, err, proof) + } + if !bytes.Equal(val, kv.v) { + t.Fatalf("prover %d: verified value mismatch for key %x: have %x, want %x", i, kv.k, val, kv.v) + } } } } @@ -53,37 +79,66 @@ func TestProof(t *testing.T) { func TestOneElementProof(t *testing.T) { trie := new(Trie) updateString(trie, "k", "v") - proofs, _ := ethdb.NewMemDatabase() - trie.Prove([]byte("k"), 0, proofs) - if len(proofs.Keys()) != 1 { - t.Error("proof should have one element") - } - val, err, _ := VerifyProof(trie.Hash(), []byte("k"), proofs) - if err != nil { - t.Fatalf("VerifyProof error: %v\nproof hashes: %v", err, proofs.Keys()) - } - if !bytes.Equal(val, []byte("v")) { - t.Fatalf("VerifyProof returned wrong value: got %x, want 'k'", val) + for i, prover := range makeProvers(trie) { + proof := prover([]byte("k")) + if proof == nil { + t.Fatalf("prover %d: nil proof", i) + } + if proof.Len() != 1 { + t.Errorf("prover %d: proof should have one element", i) + } + val, _, err := VerifyProof(trie.Hash(), []byte("k"), proof) + if err != nil { + t.Fatalf("prover %d: failed to verify proof: %v\nraw proof: %x", i, err, proof) + } + if !bytes.Equal(val, []byte("v")) { + t.Fatalf("prover %d: verified value mismatch: have %x, want 'k'", i, val) + } } } -func TestVerifyBadProof(t *testing.T) { +func TestBadProof(t *testing.T) { trie, vals := randomTrie(800) root := trie.Hash() - for _, kv := range vals { - proofs, _ := ethdb.NewMemDatabase() - trie.Prove(kv.k, 0, proofs) - if len(proofs.Keys()) == 0 { - t.Fatal("zero length proof") + for i, prover := range makeProvers(trie) { + for _, kv := range vals { + proof := prover(kv.k) + if proof == nil { + t.Fatalf("prover %d: nil proof", i) + } + key := proof.Keys()[mrand.Intn(proof.Len())] + val, _ := proof.Get(key) + proof.Delete(key) + + mutateByte(val) + proof.Put(crypto.Keccak256(val), val) + + if _, _, err := VerifyProof(root, kv.k, proof); err == nil { + t.Fatalf("prover %d: expected proof to fail for key %x", i, kv.k) + } } - keys := proofs.Keys() - key := keys[mrand.Intn(len(keys))] - node, _ := proofs.Get(key) - proofs.Delete(key) - mutateByte(node) - proofs.Put(crypto.Keccak256(node), node) - if _, err, _ := VerifyProof(root, kv.k, proofs); err == nil { - t.Fatalf("expected proof to fail for key %x", kv.k) + } +} + +// Tests that missing keys can also be proven. The test explicitly uses a single +// entry trie and checks for missing keys both before and after the single entry. +func TestMissingKeyProof(t *testing.T) { + trie := new(Trie) + updateString(trie, "k", "v") + + for i, key := range []string{"a", "j", "l", "z"} { + proof := ethdb.NewMemDatabase() + trie.Prove([]byte(key), 0, proof) + + if proof.Len() != 1 { + t.Errorf("test %d: proof should have one element", i) + } + val, _, err := VerifyProof(trie.Hash(), []byte(key), proof) + if err != nil { + t.Fatalf("test %d: failed to verify proof: %v\nraw proof: %x", i, err, proof) + } + if val != nil { + t.Fatalf("test %d: verified value mismatch: have %x, want nil", i, val) } } } @@ -109,7 +164,7 @@ func BenchmarkProve(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { kv := vals[keys[i%len(keys)]] - proofs, _ := ethdb.NewMemDatabase() + proofs := ethdb.NewMemDatabase() if trie.Prove(kv.k, 0, proofs); len(proofs.Keys()) == 0 { b.Fatalf("zero length proof for %x", kv.k) } @@ -123,7 +178,7 @@ func BenchmarkVerifyProof(b *testing.B) { var proofs []*ethdb.MemDatabase for k := range vals { keys = append(keys, k) - proof, _ := ethdb.NewMemDatabase() + proof := ethdb.NewMemDatabase() trie.Prove([]byte(k), 0, proof) proofs = append(proofs, proof) } @@ -131,7 +186,7 @@ func BenchmarkVerifyProof(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { im := i % len(keys) - if _, err, _ := VerifyProof(root, []byte(keys[im]), proofs[im]); err != nil { + if _, _, err := VerifyProof(root, []byte(keys[im]), proofs[im]); err != nil { b.Fatalf("key %x: %v", keys[im], err) } } diff --git a/trie/secure_trie.go b/trie/secure_trie.go index c1cc68482c..db94ffa04a 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -155,14 +155,19 @@ func (t *SecureTrie) Commit(onleaf LeafCallback) (root common.Hash, err error) { return t.trie.Commit(onleaf) } +// Hash returns the root hash of SecureTrie. It does not write to the +// database and can be used even if the trie doesn't have one. func (t *SecureTrie) Hash() common.Hash { return t.trie.Hash() } +// Root returns the root hash of SecureTrie. +// Deprecated: use Hash instead. func (t *SecureTrie) Root() []byte { return t.trie.Root() } +// Copy returns a copy of SecureTrie. func (t *SecureTrie) Copy() *SecureTrie { cpy := *t return &cpy diff --git a/trie/secure_trie_test.go b/trie/secure_trie_test.go index 0e4a8aa326..0ed8f18fbd 100644 --- a/trie/secure_trie_test.go +++ b/trie/secure_trie_test.go @@ -28,18 +28,14 @@ import ( ) func newEmptySecure() *SecureTrie { - diskdb, _ := ethdb.NewMemDatabase() - triedb := NewDatabase(diskdb) - - trie, _ := NewSecure(common.Hash{}, triedb, 0) + trie, _ := NewSecure(common.Hash{}, NewDatabase(ethdb.NewMemDatabase()), 0) return trie } // makeTestSecureTrie creates a large enough secure trie for testing. func makeTestSecureTrie() (*Database, *SecureTrie, map[string][]byte) { // Create an empty trie - diskdb, _ := ethdb.NewMemDatabase() - triedb := NewDatabase(diskdb) + triedb := NewDatabase(ethdb.NewMemDatabase()) trie, _ := NewSecure(common.Hash{}, triedb, 0) diff --git a/trie/sync_test.go b/trie/sync_test.go index adc6572880..55f4fb28f2 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -27,8 +27,7 @@ import ( // makeTestTrie create a sample test trie to test node-wise reconstruction. func makeTestTrie() (*Database, *Trie, map[string][]byte) { // Create an empty trie - diskdb, _ := ethdb.NewMemDatabase() - triedb := NewDatabase(diskdb) + triedb := NewDatabase(ethdb.NewMemDatabase()) trie, _ := New(common.Hash{}, triedb) // Fill it with some arbitrary data @@ -89,18 +88,13 @@ func checkTrieConsistency(db *Database, root common.Hash) error { // Tests that an empty trie is not scheduled for syncing. func TestEmptyTrieSync(t *testing.T) { - diskdbA, _ := ethdb.NewMemDatabase() - triedbA := NewDatabase(diskdbA) - - diskdbB, _ := ethdb.NewMemDatabase() - triedbB := NewDatabase(diskdbB) - - emptyA, _ := New(common.Hash{}, triedbA) - emptyB, _ := New(emptyRoot, triedbB) + dbA := NewDatabase(ethdb.NewMemDatabase()) + dbB := NewDatabase(ethdb.NewMemDatabase()) + emptyA, _ := New(common.Hash{}, dbA) + emptyB, _ := New(emptyRoot, dbB) for i, trie := range []*Trie{emptyA, emptyB} { - diskdb, _ := ethdb.NewMemDatabase() - if req := NewTrieSync(trie.Hash(), diskdb, nil).Missing(1); len(req) != 0 { + if req := NewTrieSync(trie.Hash(), ethdb.NewMemDatabase(), nil).Missing(1); len(req) != 0 { t.Errorf("test %d: content requested for empty trie: %v", i, req) } } @@ -116,7 +110,7 @@ func testIterativeTrieSync(t *testing.T, batch int) { srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) sched := NewTrieSync(srcTrie.Hash(), diskdb, nil) @@ -149,7 +143,7 @@ func TestIterativeDelayedTrieSync(t *testing.T) { srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) sched := NewTrieSync(srcTrie.Hash(), diskdb, nil) @@ -187,7 +181,7 @@ func testIterativeRandomTrieSync(t *testing.T, batch int) { srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) sched := NewTrieSync(srcTrie.Hash(), diskdb, nil) @@ -228,7 +222,7 @@ func TestIterativeRandomDelayedTrieSync(t *testing.T) { srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) sched := NewTrieSync(srcTrie.Hash(), diskdb, nil) @@ -275,7 +269,7 @@ func TestDuplicateAvoidanceTrieSync(t *testing.T) { srcDb, srcTrie, srcData := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) sched := NewTrieSync(srcTrie.Hash(), diskdb, nil) @@ -315,7 +309,7 @@ func TestIncompleteTrieSync(t *testing.T) { srcDb, srcTrie, _ := makeTestTrie() // Create a destination trie and sync with the scheduler - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) sched := NewTrieSync(srcTrie.Hash(), diskdb, nil) diff --git a/trie/trie.go b/trie/trie.go index c52475af80..7678935b4a 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -101,7 +101,7 @@ func New(root common.Hash, db *Database) (*Trie, error) { db: db, originalRoot: root, } - if (root != common.Hash{}) && root != emptyRoot { + if root != (common.Hash{}) && root != emptyRoot { rootnode, err := trie.resolveHash(root[:], nil) if err != nil { return nil, err diff --git a/trie/trie_test.go b/trie/trie_test.go index 9103a4c69e..b923a8be6d 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -43,8 +43,7 @@ func init() { // Used for testing func newEmpty() *Trie { - diskdb, _ := ethdb.NewMemDatabase() - trie, _ := New(common.Hash{}, NewDatabase(diskdb)) + trie, _ := New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase())) return trie } @@ -68,8 +67,7 @@ func TestNull(t *testing.T) { } func TestMissingRoot(t *testing.T) { - diskdb, _ := ethdb.NewMemDatabase() - trie, err := New(common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), NewDatabase(diskdb)) + trie, err := New(common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), NewDatabase(ethdb.NewMemDatabase())) if trie != nil { t.Error("New returned non-nil trie for invalid root") } @@ -82,7 +80,7 @@ func TestMissingNodeDisk(t *testing.T) { testMissingNode(t, false) } func TestMissingNodeMemonly(t *testing.T) { testMissingNode(t, true) } func testMissingNode(t *testing.T, memonly bool) { - diskdb, _ := ethdb.NewMemDatabase() + diskdb := ethdb.NewMemDatabase() triedb := NewDatabase(diskdb) trie, _ := New(common.Hash{}, triedb) @@ -413,8 +411,7 @@ func (randTest) Generate(r *rand.Rand, size int) reflect.Value { } func runRandTest(rt randTest) bool { - diskdb, _ := ethdb.NewMemDatabase() - triedb := NewDatabase(diskdb) + triedb := NewDatabase(ethdb.NewMemDatabase()) tr, _ := New(common.Hash{}, triedb) values := make(map[string]string) // tracks content of the trie diff --git a/vendor/github.com/fjl/memsize/LICENSE b/vendor/github.com/fjl/memsize/LICENSE new file mode 100644 index 0000000000..8b80456419 --- /dev/null +++ b/vendor/github.com/fjl/memsize/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Felix Lange + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/fjl/memsize/bitmap.go b/vendor/github.com/fjl/memsize/bitmap.go new file mode 100644 index 0000000000..47799ea8d3 --- /dev/null +++ b/vendor/github.com/fjl/memsize/bitmap.go @@ -0,0 +1,119 @@ +package memsize + +import ( + "math/bits" +) + +const ( + uintptrBits = 32 << (uint64(^uintptr(0)) >> 63) + uintptrBytes = uintptrBits / 8 + bmBlockRange = 1 * 1024 * 1024 // bytes covered by bmBlock + bmBlockWords = bmBlockRange / uintptrBits +) + +// bitmap is a sparse bitmap. +type bitmap struct { + blocks map[uintptr]*bmBlock +} + +func newBitmap() *bitmap { + return &bitmap{make(map[uintptr]*bmBlock)} +} + +// markRange sets n consecutive bits starting at addr. +func (b *bitmap) markRange(addr, n uintptr) { + for end := addr + n; addr < end; { + block, baddr := b.block(addr) + for i := baddr; i < bmBlockRange && addr < end; i++ { + block.mark(i) + addr++ + } + } +} + +// isMarked returns the value of the bit at the given address. +func (b *bitmap) isMarked(addr uintptr) bool { + block, baddr := b.block(addr) + return block.isMarked(baddr) +} + +// countRange returns the number of set bits in the range (addr,addr+n). +func (b *bitmap) countRange(addr, n uintptr) uintptr { + c := uintptr(0) + for end := addr + n; addr < end; { + block, baddr := b.block(addr) + bend := uintptr(bmBlockRange - 1) + if baddr+(end-addr) < bmBlockRange { + bend = baddr + (end - addr) + } + c += uintptr(block.count(baddr, bend)) + // Move addr to next block. + addr += bmBlockRange - baddr + } + return c +} + +// block finds the block corresponding to the given memory address. +// It also returns the block's starting address. +func (b *bitmap) block(addr uintptr) (*bmBlock, uintptr) { + index := addr / bmBlockRange + block := b.blocks[index] + if block == nil { + block = new(bmBlock) + b.blocks[index] = block + } + return block, addr % bmBlockRange +} + +// size returns the sum of the byte sizes of all blocks. +func (b *bitmap) size() uintptr { + return uintptr(len(b.blocks)) * bmBlockWords * uintptrBytes +} + +// utilization returns the mean percentage of one bits across all blocks. +func (b *bitmap) utilization() float32 { + var avg float32 + for _, block := range b.blocks { + avg += float32(block.count(0, bmBlockRange-1)) / float32(bmBlockRange) + } + return avg / float32(len(b.blocks)) +} + +// bmBlock is a bitmap block. +type bmBlock [bmBlockWords]uintptr + +// mark sets the i'th bit to one. +func (b *bmBlock) mark(i uintptr) { + b[i/uintptrBits] |= 1 << (i % uintptrBits) +} + +// isMarked returns the value of the i'th bit. +func (b *bmBlock) isMarked(i uintptr) bool { + return (b[i/uintptrBits] & (1 << (i % uintptrBits))) != 0 +} + +// count returns the number of set bits in the range (start,end). +func (b *bmBlock) count(start, end uintptr) (count int) { + br := b[start/uintptrBits : end/uintptrBits+1] + for i, w := range br { + if i == 0 { + w &= blockmask(start) + } + if i == len(br)-1 { + w &^= blockmask(end) + } + count += onesCountPtr(w) + } + return count +} + +func blockmask(x uintptr) uintptr { + return ^uintptr(0) << (x % uintptrBits) +} + +func onesCountPtr(x uintptr) int { + if uintptrBits == 64 { + return bits.OnesCount64(uint64(x)) + } + return bits.OnesCount32(uint32(x)) +} diff --git a/vendor/github.com/fjl/memsize/doc.go b/vendor/github.com/fjl/memsize/doc.go new file mode 100644 index 0000000000..640cfba5eb --- /dev/null +++ b/vendor/github.com/fjl/memsize/doc.go @@ -0,0 +1,16 @@ +/* +Package memsize computes the size of your object graph. + +So you made a spiffy algorithm and it works really well, but geez it's using +way too much memory. Where did it all go? memsize to the rescue! + +To get started, find a value that references all your objects and scan it. +This traverses the graph, counting sizes per type. + + sizes := memsize.Scan(myValue) + fmt.Println(sizes.Total) + +memsize can handle cycles just fine and tracks both private and public struct fields. +Unfortunately function closures cannot be inspected in any way. +*/ +package memsize diff --git a/vendor/github.com/fjl/memsize/memsize.go b/vendor/github.com/fjl/memsize/memsize.go new file mode 100644 index 0000000000..2664e87c46 --- /dev/null +++ b/vendor/github.com/fjl/memsize/memsize.go @@ -0,0 +1,243 @@ +package memsize + +import ( + "bytes" + "fmt" + "reflect" + "sort" + "strings" + "text/tabwriter" + "unsafe" +) + +// Scan traverses all objects reachable from v and counts how much memory +// is used per type. The value must be a non-nil pointer to any value. +func Scan(v interface{}) Sizes { + rv := reflect.ValueOf(v) + if rv.Kind() != reflect.Ptr || rv.IsNil() { + panic("value to scan must be non-nil pointer") + } + + stopTheWorld("memsize scan") + defer startTheWorld() + + ctx := newContext() + ctx.scan(invalidAddr, rv, false) + ctx.s.BitmapSize = ctx.seen.size() + ctx.s.BitmapUtilization = ctx.seen.utilization() + return *ctx.s +} + +// Sizes is the result of a scan. +type Sizes struct { + Total uintptr + ByType map[reflect.Type]*TypeSize + // Internal stats (for debugging) + BitmapSize uintptr + BitmapUtilization float32 +} + +type TypeSize struct { + Total uintptr + Count uintptr +} + +func newSizes() *Sizes { + return &Sizes{ByType: make(map[reflect.Type]*TypeSize)} +} + +// Report returns a human-readable report. +func (s Sizes) Report() string { + type typLine struct { + name string + count uintptr + total uintptr + } + tab := []typLine{{"ALL", 0, s.Total}} + for _, typ := range s.ByType { + tab[0].count += typ.Count + } + maxname := 0 + for typ, s := range s.ByType { + line := typLine{typ.String(), s.Count, s.Total} + tab = append(tab, line) + if len(line.name) > maxname { + maxname = len(line.name) + } + } + sort.Slice(tab, func(i, j int) bool { return tab[i].total > tab[j].total }) + + buf := new(bytes.Buffer) + w := tabwriter.NewWriter(buf, 0, 0, 0, ' ', tabwriter.AlignRight) + for _, line := range tab { + namespace := strings.Repeat(" ", maxname-len(line.name)) + fmt.Fprintf(w, "%s%s\t %v\t %s\t\n", line.name, namespace, line.count, HumanSize(line.total)) + } + w.Flush() + return buf.String() +} + +// addValue is called during scan and adds the memory of given object. +func (s *Sizes) addValue(v reflect.Value, size uintptr) { + s.Total += size + rs := s.ByType[v.Type()] + if rs == nil { + rs = new(TypeSize) + s.ByType[v.Type()] = rs + } + rs.Total += size + rs.Count++ +} + +type context struct { + // We track previously scanned objects to prevent infinite loops + // when scanning cycles and to prevent counting objects more than once. + seen *bitmap + tc typCache + s *Sizes +} + +func newContext() *context { + return &context{seen: newBitmap(), tc: make(typCache), s: newSizes()} +} + +// scan walks all objects below v, determining their size. All scan* functions return the +// amount of 'extra' memory (e.g. slice data) that is referenced by the object. +func (c *context) scan(addr address, v reflect.Value, add bool) (extraSize uintptr) { + size := v.Type().Size() + var marked uintptr + if addr.valid() { + marked = c.seen.countRange(uintptr(addr), size) + if marked == size { + return 0 // Skip if we have already seen the whole object. + } + c.seen.markRange(uintptr(addr), size) + } + // fmt.Printf("%v: %v ⮑ (marked %d)\n", addr, v.Type(), marked) + if c.tc.needScan(v.Type()) { + extraSize = c.scanContent(addr, v) + } + // fmt.Printf("%v: %v %d (add %v, size %d, marked %d, extra %d)\n", addr, v.Type(), size+extraSize, add, v.Type().Size(), marked, extraSize) + if add { + size -= marked + size += extraSize + c.s.addValue(v, size) + } + return extraSize +} + +func (c *context) scanContent(addr address, v reflect.Value) uintptr { + switch v.Kind() { + case reflect.Array: + return c.scanArray(addr, v) + case reflect.Chan: + return c.scanChan(v) + case reflect.Func: + // can't do anything here + return 0 + case reflect.Interface: + return c.scanInterface(v) + case reflect.Map: + return c.scanMap(v) + case reflect.Ptr: + if !v.IsNil() { + c.scan(address(v.Pointer()), v.Elem(), true) + } + return 0 + case reflect.Slice: + return c.scanSlice(v) + case reflect.String: + return uintptr(v.Len()) + case reflect.Struct: + return c.scanStruct(addr, v) + default: + unhandledKind(v.Kind()) + return 0 + } +} + +func (c *context) scanChan(v reflect.Value) uintptr { + etyp := v.Type().Elem() + extra := uintptr(0) + if c.tc.needScan(etyp) { + // Scan the channel buffer. This is unsafe but doesn't race because + // the world is stopped during scan. + hchan := unsafe.Pointer(v.Pointer()) + for i := uint(0); i < uint(v.Cap()); i++ { + addr := chanbuf(hchan, i) + elem := reflect.NewAt(etyp, addr).Elem() + extra += c.scanContent(address(addr), elem) + } + } + return uintptr(v.Cap())*etyp.Size() + extra +} + +func (c *context) scanStruct(base address, v reflect.Value) uintptr { + extra := uintptr(0) + for i := 0; i < v.NumField(); i++ { + f := v.Type().Field(i) + if c.tc.needScan(f.Type) { + addr := base.addOffset(f.Offset) + extra += c.scanContent(addr, v.Field(i)) + } + } + return extra +} + +func (c *context) scanArray(addr address, v reflect.Value) uintptr { + esize := v.Type().Elem().Size() + extra := uintptr(0) + for i := 0; i < v.Len(); i++ { + extra += c.scanContent(addr, v.Index(i)) + addr = addr.addOffset(esize) + } + return extra +} + +func (c *context) scanSlice(v reflect.Value) uintptr { + slice := v.Slice(0, v.Cap()) + esize := slice.Type().Elem().Size() + base := slice.Pointer() + // Add size of the unscanned portion of the backing array to extra. + blen := uintptr(slice.Len()) * esize + marked := c.seen.countRange(base, blen) + extra := blen - marked + c.seen.markRange(uintptr(base), blen) + if c.tc.needScan(slice.Type().Elem()) { + // Elements may contain pointers, scan them individually. + addr := address(base) + for i := 0; i < slice.Len(); i++ { + extra += c.scanContent(addr, slice.Index(i)) + addr = addr.addOffset(esize) + } + } + return extra +} + +func (c *context) scanMap(v reflect.Value) uintptr { + var ( + typ = v.Type() + len = uintptr(v.Len()) + extra = uintptr(0) + ) + if c.tc.needScan(typ.Key()) || c.tc.needScan(typ.Elem()) { + for _, k := range v.MapKeys() { + extra += c.scan(invalidAddr, k, false) + extra += c.scan(invalidAddr, v.MapIndex(k), false) + } + } + return len*typ.Key().Size() + len*typ.Elem().Size() + extra +} + +func (c *context) scanInterface(v reflect.Value) uintptr { + elem := v.Elem() + if !elem.IsValid() { + return 0 // nil interface + } + c.scan(invalidAddr, elem, false) + if !c.tc.isPointer(elem.Type()) { + // Account for non-pointer size of the value. + return elem.Type().Size() + } + return 0 +} diff --git a/vendor/github.com/fjl/memsize/memsizeui/template.go b/vendor/github.com/fjl/memsize/memsizeui/template.go new file mode 100644 index 0000000000..b60fe6ba54 --- /dev/null +++ b/vendor/github.com/fjl/memsize/memsizeui/template.go @@ -0,0 +1,106 @@ +package memsizeui + +import ( + "html/template" + "strconv" + "sync" + + "github.com/fjl/memsize" +) + +var ( + base *template.Template // the "base" template + baseInitOnce sync.Once +) + +func baseInit() { + base = template.Must(template.New("base").Parse(` + + + + memsize + + + + {{template "content" .}} + +`)) + + base.Funcs(template.FuncMap{ + "quote": strconv.Quote, + "humansize": memsize.HumanSize, + }) + + template.Must(base.New("rootbuttons").Parse(` +Overview +{{- range $root := .Roots -}} +
+ +
+{{- end -}}`)) +} + +func contentTemplate(source string) *template.Template { + baseInitOnce.Do(baseInit) + t := template.Must(base.Clone()) + template.Must(t.New("content").Parse(source)) + return t +} + +var rootTemplate = contentTemplate(` +

Memsize

+{{template "rootbuttons" .}} +
+

Reports

+ +`) + +var notFoundTemplate = contentTemplate(` +

{{.Data}}

+{{template "rootbuttons" .}} +`) + +var reportTemplate = contentTemplate(` +{{- $report := .Data -}} +

Memsize Report {{$report.ID}}

+
+ Overview + +
+
+Root: {{quote $report.RootName}}
+Date: {{$report.Date}}
+Duration: {{$report.Duration}}
+Bitmap Size: {{$report.Sizes.BitmapSize | humansize}}
+Bitmap Utilization: {{$report.Sizes.BitmapUtilization}}
+
+
+
+{{$report.Sizes.Report}}
+
+`) diff --git a/vendor/github.com/fjl/memsize/memsizeui/ui.go b/vendor/github.com/fjl/memsize/memsizeui/ui.go new file mode 100644 index 0000000000..c48fc53f7f --- /dev/null +++ b/vendor/github.com/fjl/memsize/memsizeui/ui.go @@ -0,0 +1,153 @@ +package memsizeui + +import ( + "bytes" + "fmt" + "html/template" + "net/http" + "reflect" + "sort" + "strings" + "sync" + "time" + + "github.com/fjl/memsize" +) + +type Handler struct { + init sync.Once + mux http.ServeMux + mu sync.Mutex + reports map[int]Report + roots map[string]interface{} + reportID int +} + +type Report struct { + ID int + Date time.Time + Duration time.Duration + RootName string + Sizes memsize.Sizes +} + +type templateInfo struct { + Roots []string + Reports map[int]Report + PathDepth int + Data interface{} +} + +func (ti *templateInfo) Link(path ...string) string { + prefix := strings.Repeat("../", ti.PathDepth) + return prefix + strings.Join(path, "") +} + +func (h *Handler) Add(name string, v interface{}) { + rv := reflect.ValueOf(v) + if rv.Kind() != reflect.Ptr || rv.IsNil() { + panic("root must be non-nil pointer") + } + h.mu.Lock() + if h.roots == nil { + h.roots = make(map[string]interface{}) + } + h.roots[name] = v + h.mu.Unlock() +} + +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h.init.Do(func() { + h.reports = make(map[int]Report) + h.mux.HandleFunc("/", h.handleRoot) + h.mux.HandleFunc("/scan", h.handleScan) + h.mux.HandleFunc("/report/", h.handleReport) + }) + h.mux.ServeHTTP(w, r) +} + +func (h *Handler) templateInfo(r *http.Request, data interface{}) *templateInfo { + h.mu.Lock() + roots := make([]string, 0, len(h.roots)) + for name := range h.roots { + roots = append(roots, name) + } + h.mu.Unlock() + sort.Strings(roots) + + return &templateInfo{ + Roots: roots, + Reports: h.reports, + PathDepth: strings.Count(r.URL.Path, "/") - 1, + Data: data, + } +} + +func (h *Handler) handleRoot(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + http.NotFound(w, r) + return + } + serveHTML(w, rootTemplate, http.StatusOK, h.templateInfo(r, nil)) +} + +func (h *Handler) handleScan(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "invalid HTTP method, want POST", http.StatusMethodNotAllowed) + return + } + ti := h.templateInfo(r, "Unknown root") + id, ok := h.scan(r.URL.Query().Get("root")) + if !ok { + serveHTML(w, notFoundTemplate, http.StatusNotFound, ti) + return + } + w.Header().Add("Location", ti.Link(fmt.Sprintf("report/%d", id))) + w.WriteHeader(http.StatusSeeOther) +} + +func (h *Handler) handleReport(w http.ResponseWriter, r *http.Request) { + var id int + fmt.Sscan(strings.TrimPrefix(r.URL.Path, "/report/"), &id) + h.mu.Lock() + report, ok := h.reports[id] + h.mu.Unlock() + + if !ok { + serveHTML(w, notFoundTemplate, http.StatusNotFound, h.templateInfo(r, "Report not found")) + } else { + serveHTML(w, reportTemplate, http.StatusOK, h.templateInfo(r, report)) + } +} + +func (h *Handler) scan(root string) (int, bool) { + h.mu.Lock() + defer h.mu.Unlock() + + val, ok := h.roots[root] + if !ok { + return 0, false + } + id := h.reportID + start := time.Now() + sizes := memsize.Scan(val) + h.reports[id] = Report{ + ID: id, + RootName: root, + Date: start.Truncate(1 * time.Second), + Duration: time.Since(start), + Sizes: sizes, + } + h.reportID++ + return id, true +} + +func serveHTML(w http.ResponseWriter, tpl *template.Template, status int, ti *templateInfo) { + w.Header().Set("content-type", "text/html") + var buf bytes.Buffer + if err := tpl.Execute(&buf, ti); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + buf.WriteTo(w) +} diff --git a/vendor/github.com/fjl/memsize/runtimefunc.go b/vendor/github.com/fjl/memsize/runtimefunc.go new file mode 100644 index 0000000000..912a3e768d --- /dev/null +++ b/vendor/github.com/fjl/memsize/runtimefunc.go @@ -0,0 +1,14 @@ +package memsize + +import "unsafe" + +var _ = unsafe.Pointer(nil) + +//go:linkname stopTheWorld runtime.stopTheWorld +func stopTheWorld(reason string) + +//go:linkname startTheWorld runtime.startTheWorld +func startTheWorld() + +//go:linkname chanbuf runtime.chanbuf +func chanbuf(ch unsafe.Pointer, i uint) unsafe.Pointer diff --git a/vendor/github.com/fjl/memsize/runtimefunc.s b/vendor/github.com/fjl/memsize/runtimefunc.s new file mode 100644 index 0000000000..a091e2fa72 --- /dev/null +++ b/vendor/github.com/fjl/memsize/runtimefunc.s @@ -0,0 +1 @@ +// This file is required to make stub function declarations work. diff --git a/vendor/github.com/fjl/memsize/type.go b/vendor/github.com/fjl/memsize/type.go new file mode 100644 index 0000000000..5d6f59e9ff --- /dev/null +++ b/vendor/github.com/fjl/memsize/type.go @@ -0,0 +1,119 @@ +package memsize + +import ( + "fmt" + "reflect" +) + +// address is a memory location. +// +// Code dealing with uintptr is oblivious to the zero address. +// Code dealing with address is not: it treats the zero address +// as invalid. Offsetting an invalid address doesn't do anything. +// +// This distinction is useful because there are objects that we can't +// get the pointer to. +type address uintptr + +const invalidAddr = address(0) + +func (a address) valid() bool { + return a != 0 +} + +func (a address) addOffset(off uintptr) address { + if !a.valid() { + return invalidAddr + } + return a + address(off) +} + +func (a address) String() string { + if uintptrBits == 32 { + return fmt.Sprintf("%#0.8x", uintptr(a)) + } + return fmt.Sprintf("%#0.16x", uintptr(a)) +} + +type typCache map[reflect.Type]typInfo + +type typInfo struct { + isPointer bool + needScan bool +} + +// isPointer returns true for pointer-ish values. The notion of +// pointer includes everything but plain values, i.e. slices, maps +// channels, interfaces are 'pointer', too. +func (tc *typCache) isPointer(typ reflect.Type) bool { + return tc.info(typ).isPointer +} + +// needScan reports whether a value of the type needs to be scanned +// recursively because it may contain pointers. +func (tc *typCache) needScan(typ reflect.Type) bool { + return tc.info(typ).needScan +} + +func (tc *typCache) info(typ reflect.Type) typInfo { + info, found := (*tc)[typ] + switch { + case found: + return info + case isPointer(typ): + info = typInfo{true, true} + default: + info = typInfo{false, tc.checkNeedScan(typ)} + } + (*tc)[typ] = info + return info +} + +func (tc *typCache) checkNeedScan(typ reflect.Type) bool { + switch k := typ.Kind(); k { + case reflect.Struct: + // Structs don't need scan if none of their fields need it. + for i := 0; i < typ.NumField(); i++ { + if tc.needScan(typ.Field(i).Type) { + return true + } + } + case reflect.Array: + // Arrays don't need scan if their element type doesn't. + return tc.needScan(typ.Elem()) + } + return false +} + +func isPointer(typ reflect.Type) bool { + k := typ.Kind() + switch { + case k <= reflect.Complex128: + return false + case k == reflect.Array: + return false + case k >= reflect.Chan && k <= reflect.String: + return true + case k == reflect.Struct || k == reflect.UnsafePointer: + return false + default: + unhandledKind(k) + return false + } +} + +func unhandledKind(k reflect.Kind) { + panic("unhandled kind " + k.String()) +} + +// HumanSize formats the given number of bytes as a readable string. +func HumanSize(bytes uintptr) string { + switch { + case bytes < 1024: + return fmt.Sprintf("%d B", bytes) + case bytes < 1024*1024: + return fmt.Sprintf("%.3f KB", float64(bytes)/1024) + default: + return fmt.Sprintf("%.3f MB", float64(bytes)/1024/1024) + } +} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db.go b/vendor/github.com/syndtr/goleveldb/leveldb/db.go index 3655418ad3..e7ac065418 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/db.go @@ -35,6 +35,7 @@ type DB struct { // Stats. Need 64-bit alignment. cWriteDelay int64 // The cumulative duration of write delays cWriteDelayN int32 // The cumulative number of write delays + inWritePaused int32 // The indicator whether write operation is paused by compaction aliveSnaps, aliveIters int32 // Session. @@ -967,7 +968,8 @@ func (db *DB) GetProperty(name string) (value string, err error) { float64(db.s.stor.writes())/1048576.0) case p == "writedelay": writeDelayN, writeDelay := atomic.LoadInt32(&db.cWriteDelayN), time.Duration(atomic.LoadInt64(&db.cWriteDelay)) - value = fmt.Sprintf("DelayN:%d Delay:%s", writeDelayN, writeDelay) + paused := atomic.LoadInt32(&db.inWritePaused) == 1 + value = fmt.Sprintf("DelayN:%d Delay:%s Paused:%t", writeDelayN, writeDelay, paused) case p == "sstables": for level, tables := range v.levels { value += fmt.Sprintf("--- level %d ---\n", level) @@ -996,6 +998,75 @@ func (db *DB) GetProperty(name string) (value string, err error) { return } +// DBStats is database statistics. +type DBStats struct { + WriteDelayCount int32 + WriteDelayDuration time.Duration + WritePaused bool + + AliveSnapshots int32 + AliveIterators int32 + + IOWrite uint64 + IORead uint64 + + BlockCacheSize int + OpenedTablesCount int + + LevelSizes []int64 + LevelTablesCounts []int + LevelRead []int64 + LevelWrite []int64 + LevelDurations []time.Duration +} + +// Stats populates s with database statistics. +func (db *DB) Stats(s *DBStats) error { + err := db.ok() + if err != nil { + return err + } + + s.IORead = db.s.stor.reads() + s.IOWrite = db.s.stor.writes() + s.WriteDelayCount = atomic.LoadInt32(&db.cWriteDelayN) + s.WriteDelayDuration = time.Duration(atomic.LoadInt64(&db.cWriteDelay)) + s.WritePaused = atomic.LoadInt32(&db.inWritePaused) == 1 + + s.OpenedTablesCount = db.s.tops.cache.Size() + if db.s.tops.bcache != nil { + s.BlockCacheSize = db.s.tops.bcache.Size() + } else { + s.BlockCacheSize = 0 + } + + s.AliveIterators = atomic.LoadInt32(&db.aliveIters) + s.AliveSnapshots = atomic.LoadInt32(&db.aliveSnaps) + + s.LevelDurations = s.LevelDurations[:0] + s.LevelRead = s.LevelRead[:0] + s.LevelWrite = s.LevelWrite[:0] + s.LevelSizes = s.LevelSizes[:0] + s.LevelTablesCounts = s.LevelTablesCounts[:0] + + v := db.s.version() + defer v.release() + + for level, tables := range v.levels { + duration, read, write := db.compStats.getStat(level) + if len(tables) == 0 && duration == 0 { + continue + } + s.LevelDurations = append(s.LevelDurations, duration) + s.LevelRead = append(s.LevelRead, read) + s.LevelWrite = append(s.LevelWrite, write) + s.LevelSizes = append(s.LevelSizes, tables.size()) + s.LevelTablesCounts = append(s.LevelTablesCounts, len(tables)) + } + + return nil +} + // SizeOf calculates approximate sizes of the given key ranges. // The length of the returned sizes are equal with the length of the given // ranges. The returned sizes measure storage space usage, so if the user diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db_write.go b/vendor/github.com/syndtr/goleveldb/leveldb/db_write.go index 31f4bc5efc..db0c1bece1 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db_write.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/db_write.go @@ -89,7 +89,11 @@ func (db *DB) flush(n int) (mdb *memDB, mdbFree int, err error) { return false case tLen >= pauseTrigger: delayed = true + // Set the write paused flag explicitly. + atomic.StoreInt32(&db.inWritePaused, 1) err = db.compTriggerWait(db.tcompCmdC) + // Unset the write paused flag. + atomic.StoreInt32(&db.inWritePaused, 0) if err != nil { return false } diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/storage/mem_storage.go b/vendor/github.com/syndtr/goleveldb/leveldb/storage/mem_storage.go index 9b0421f035..838f1bee1b 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/storage/mem_storage.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/storage/mem_storage.go @@ -12,7 +12,11 @@ import ( "sync" ) -const typeShift = 3 +const typeShift = 4 + +// Verify at compile-time that typeShift is large enough to cover all FileType +// values by confirming that 0 == 0. +var _ [0]struct{} = [TypeAll >> typeShift]struct{}{} type memStorageLock struct { ms *memStorage @@ -143,7 +147,7 @@ func (ms *memStorage) Remove(fd FileDesc) error { } func (ms *memStorage) Rename(oldfd, newfd FileDesc) error { - if FileDescOk(oldfd) || FileDescOk(newfd) { + if !FileDescOk(oldfd) || !FileDescOk(newfd) { return ErrInvalidFile } if oldfd == newfd { diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/util.go b/vendor/github.com/syndtr/goleveldb/leveldb/util.go index e572a329e9..0e2b519e5c 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/util.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/util.go @@ -20,7 +20,7 @@ func shorten(str string) string { return str[:3] + ".." + str[len(str)-3:] } -var bunits = [...]string{"", "Ki", "Mi", "Gi"} +var bunits = [...]string{"", "Ki", "Mi", "Gi", "Ti"} func shortenb(bytes int) string { i := 0 diff --git a/vendor/vendor.json b/vendor/vendor.json index 20d6b7715b..5ee79d4d64 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -110,6 +110,18 @@ "revision": "5ec5d9d3c2cf82e9688b34e9bc27a94d616a7193", "revisionTime": "2017-02-09T08:00:14Z" }, + { + "checksumSHA1": "Jq1rrHSGPfh689nA2hL1QVb62zE=", + "path": "github.com/fjl/memsize", + "revision": "ca190fb6ffbc076ff49197b7168a760f30182d2e", + "revisionTime": "2018-04-18T12:24:29Z" + }, + { + "checksumSHA1": "Z13QAYTqeW4cTiglkc2F05gWLu4=", + "path": "github.com/fjl/memsize/memsizeui", + "revision": "ca190fb6ffbc076ff49197b7168a760f30182d2e", + "revisionTime": "2018-04-18T12:24:29Z" + }, { "checksumSHA1": "0orwvPL96wFckVJyPl39fz2QsgA=", "path": "github.com/gizak/termui", @@ -406,76 +418,76 @@ "revisionTime": "2017-07-05T02:17:15Z" }, { - "checksumSHA1": "3QsnhPTXGytTbW3uDvQLgSo9s9M=", + "checksumSHA1": "TJV50D0q8E3vtc90ibC+qOYdjrw=", "path": "github.com/syndtr/goleveldb/leveldb", - "revision": "169b1b37be738edb2813dab48c97a549bcf99bb5", - "revisionTime": "2018-03-07T11:33:52Z" + "revision": "59047f74db0d042c8d8dd8e30bb030bc774a7d7a", + "revisionTime": "2018-05-21T04:45:49Z" }, { "checksumSHA1": "EKIow7XkgNdWvR/982ffIZxKG8Y=", "path": "github.com/syndtr/goleveldb/leveldb/cache", - "revision": "169b1b37be738edb2813dab48c97a549bcf99bb5", - "revisionTime": "2018-03-07T11:33:52Z" + "revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a", + "revisionTime": "2018-05-02T07:23:49Z" }, { "checksumSHA1": "5KPgnvCPlR0ysDAqo6jApzRQ3tw=", "path": "github.com/syndtr/goleveldb/leveldb/comparer", - "revision": "169b1b37be738edb2813dab48c97a549bcf99bb5", - "revisionTime": "2018-03-07T11:33:52Z" + "revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a", + "revisionTime": "2018-05-02T07:23:49Z" }, { "checksumSHA1": "1DRAxdlWzS4U0xKN/yQ/fdNN7f0=", "path": "github.com/syndtr/goleveldb/leveldb/errors", - "revision": "169b1b37be738edb2813dab48c97a549bcf99bb5", - "revisionTime": "2018-03-07T11:33:52Z" + "revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a", + "revisionTime": "2018-05-02T07:23:49Z" }, { "checksumSHA1": "eqKeD6DS7eNCtxVYZEHHRKkyZrw=", "path": "github.com/syndtr/goleveldb/leveldb/filter", - "revision": "169b1b37be738edb2813dab48c97a549bcf99bb5", - "revisionTime": "2018-03-07T11:33:52Z" + "revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a", + "revisionTime": "2018-05-02T07:23:49Z" }, { "checksumSHA1": "weSsccMav4BCerDpSLzh3mMxAYo=", "path": "github.com/syndtr/goleveldb/leveldb/iterator", - "revision": "169b1b37be738edb2813dab48c97a549bcf99bb5", - "revisionTime": "2018-03-07T11:33:52Z" + "revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a", + "revisionTime": "2018-05-02T07:23:49Z" }, { "checksumSHA1": "gJY7bRpELtO0PJpZXgPQ2BYFJ88=", "path": "github.com/syndtr/goleveldb/leveldb/journal", - "revision": "169b1b37be738edb2813dab48c97a549bcf99bb5", - "revisionTime": "2018-03-07T11:33:52Z" + "revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a", + "revisionTime": "2018-05-02T07:23:49Z" }, { "checksumSHA1": "MtYY1b2234y/MlS+djL8tXVAcQs=", "path": "github.com/syndtr/goleveldb/leveldb/memdb", - "revision": "169b1b37be738edb2813dab48c97a549bcf99bb5", - "revisionTime": "2018-03-07T11:33:52Z" + "revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a", + "revisionTime": "2018-05-02T07:23:49Z" }, { "checksumSHA1": "UmQeotV+m8/FduKEfLOhjdp18rs=", "path": "github.com/syndtr/goleveldb/leveldb/opt", - "revision": "169b1b37be738edb2813dab48c97a549bcf99bb5", - "revisionTime": "2018-03-07T11:33:52Z" + "revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a", + "revisionTime": "2018-05-02T07:23:49Z" }, { - "checksumSHA1": "QCSae2ub87f8awH+PKMpd8ZYOtg=", + "checksumSHA1": "7H3fa12T7WoMAeXq1+qG5O7LD0w=", "path": "github.com/syndtr/goleveldb/leveldb/storage", - "revision": "169b1b37be738edb2813dab48c97a549bcf99bb5", - "revisionTime": "2018-03-07T11:33:52Z" + "revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a", + "revisionTime": "2018-05-02T07:23:49Z" }, { "checksumSHA1": "gWFPMz8OQeul0t54RM66yMTX49g=", "path": "github.com/syndtr/goleveldb/leveldb/table", - "revision": "169b1b37be738edb2813dab48c97a549bcf99bb5", - "revisionTime": "2018-03-07T11:33:52Z" + "revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a", + "revisionTime": "2018-05-02T07:23:49Z" }, { "checksumSHA1": "V/Dh7NV0/fy/5jX1KaAjmGcNbzI=", "path": "github.com/syndtr/goleveldb/leveldb/util", - "revision": "169b1b37be738edb2813dab48c97a549bcf99bb5", - "revisionTime": "2018-03-07T11:33:52Z" + "revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a", + "revisionTime": "2018-05-02T07:23:49Z" }, { "checksumSHA1": "TT1rac6kpQp2vz24m5yDGUNQ/QQ=", diff --git a/whisper/mailserver/mailserver.go b/whisper/mailserver/mailserver.go index 3bd7334536..0d3871b703 100644 --- a/whisper/mailserver/mailserver.go +++ b/whisper/mailserver/mailserver.go @@ -20,7 +20,6 @@ import ( "encoding/binary" "fmt" - "github.com/EthereumCommonwealth/go-callisto/cmd/utils" "github.com/EthereumCommonwealth/go-callisto/common" "github.com/EthereumCommonwealth/go-callisto/crypto" "github.com/EthereumCommonwealth/go-callisto/log" @@ -54,19 +53,19 @@ func NewDbKey(t uint32, h common.Hash) *DBKey { return &k } -func (s *WMailServer) Init(shh *whisper.Whisper, path string, password string, pow float64) { +func (s *WMailServer) Init(shh *whisper.Whisper, path string, password string, pow float64) error { var err error if len(path) == 0 { - utils.Fatalf("DB file is not specified") + return fmt.Errorf("DB file is not specified") } if len(password) == 0 { - utils.Fatalf("Password is not specified for MailServer") + return fmt.Errorf("password is not specified") } s.db, err = leveldb.OpenFile(path, nil) if err != nil { - utils.Fatalf("Failed to open DB file: %s", err) + return fmt.Errorf("open DB file: %s", err) } s.w = shh @@ -74,12 +73,13 @@ func (s *WMailServer) Init(shh *whisper.Whisper, path string, password string, p MailServerKeyID, err := s.w.AddSymKeyFromPassword(password) if err != nil { - utils.Fatalf("Failed to create symmetric key for MailServer: %s", err) + return fmt.Errorf("create symmetric key: %s", err) } s.key, err = s.w.GetSymKey(MailServerKeyID) if err != nil { - utils.Fatalf("Failed to save symmetric key for MailServer") + return fmt.Errorf("save symmetric key: %s", err) } + return nil } func (s *WMailServer) Close() { diff --git a/whisper/mailserver/server_test.go b/whisper/mailserver/server_test.go index 46047ae10b..cd260babbc 100644 --- a/whisper/mailserver/server_test.go +++ b/whisper/mailserver/server_test.go @@ -92,7 +92,10 @@ func TestMailServer(t *testing.T) { shh = whisper.New(&whisper.DefaultConfig) shh.RegisterServer(&server) - server.Init(shh, dir, password, powRequirement) + err = server.Init(shh, dir, password, powRequirement) + if err != nil { + t.Fatal(err) + } defer server.Close() keyID, err = shh.AddSymKeyFromPassword(password) diff --git a/whisper/shhclient/client.go b/whisper/shhclient/client.go index 6ea1fa7bac..a8a2b2edab 100644 --- a/whisper/shhclient/client.go +++ b/whisper/shhclient/client.go @@ -67,7 +67,6 @@ func (sc *Client) SetMaxMessageSize(ctx context.Context, size uint32) error { } // SetMinimumPoW (experimental) sets the minimal PoW required by this node. - // This experimental function was introduced for the future dynamic adjustment of // PoW requirement. If the node is overwhelmed with messages, it should raise the // PoW requirement and notify the peers. The new value should be set relative to @@ -77,7 +76,7 @@ func (sc *Client) SetMinimumPoW(ctx context.Context, pow float64) error { return sc.c.CallContext(ctx, &ignored, "shh_setMinPoW", pow) } -// Marks specific peer trusted, which will allow it to send historic (expired) messages. +// MarkTrustedPeer marks specific peer trusted, which will allow it to send historic (expired) messages. // Note This function is not adding new nodes, the node needs to exists as a peer. func (sc *Client) MarkTrustedPeer(ctx context.Context, enode string) error { var ignored bool @@ -136,9 +135,9 @@ func (sc *Client) AddSymmetricKey(ctx context.Context, key []byte) (string, erro } // GenerateSymmetricKeyFromPassword generates the key from password, stores it, and returns its identifier. -func (sc *Client) GenerateSymmetricKeyFromPassword(ctx context.Context, passwd []byte) (string, error) { +func (sc *Client) GenerateSymmetricKeyFromPassword(ctx context.Context, passwd string) (string, error) { var id string - return id, sc.c.CallContext(ctx, &id, "shh_generateSymKeyFromPassword", hexutil.Bytes(passwd)) + return id, sc.c.CallContext(ctx, &id, "shh_generateSymKeyFromPassword", passwd) } // HasSymmetricKey returns an indication if the key associated with the given id is stored in the node. diff --git a/whisper/whisperv5/api.go b/whisper/whisperv5/api.go index 32c0b469a6..48d9a77822 100644 --- a/whisper/whisperv5/api.go +++ b/whisper/whisperv5/api.go @@ -89,7 +89,7 @@ func (api *PublicWhisperAPI) SetMaxMessageSize(ctx context.Context, size uint32) return true, api.w.SetMaxMessageSize(size) } -// SetMinPow sets the minimum PoW for a message before it is accepted. +// SetMinPoW sets the minimum PoW for a message before it is accepted. func (api *PublicWhisperAPI) SetMinPoW(ctx context.Context, pow float64) (bool, error) { return true, api.w.SetMinimumPoW(pow) } @@ -142,7 +142,7 @@ func (api *PublicWhisperAPI) GetPublicKey(ctx context.Context, id string) (hexut return crypto.FromECDSAPub(&key.PublicKey), nil } -// GetPublicKey returns the private key associated with the given key. The key is the hex +// GetPrivateKey returns the private key associated with the given key. The key is the hex // encoded representation of a key in the form specified in section 4.3.6 of ANSI X9.62. func (api *PublicWhisperAPI) GetPrivateKey(ctx context.Context, id string) (hexutil.Bytes, error) { key, err := api.w.GetPrivateKey(id) diff --git a/whisper/whisperv5/doc.go b/whisper/whisperv5/doc.go index 7a57488bd7..8161db8ed6 100644 --- a/whisper/whisperv5/doc.go +++ b/whisper/whisperv5/doc.go @@ -15,7 +15,7 @@ // along with the go-ethereum library. If not, see . /* -Package whisper implements the Whisper protocol (version 5). +Package whisperv5 implements the Whisper protocol (version 5). Whisper combines aspects of both DHTs and datagram messaging systems (e.g. UDP). As such it may be likened and compared to both, not dissimilar to the diff --git a/whisper/whisperv5/message.go b/whisper/whisperv5/message.go index 4e740a79ca..ad28367251 100644 --- a/whisper/whisperv5/message.go +++ b/whisper/whisperv5/message.go @@ -33,7 +33,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/log" ) -// Options specifies the exact way a message should be wrapped into an Envelope. +// MessageParams specifies the exact way a message should be wrapped into an Envelope. type MessageParams struct { TTL uint32 Src *ecdsa.PrivateKey @@ -86,7 +86,7 @@ func (msg *ReceivedMessage) isAsymmetricEncryption() bool { return msg.Dst != nil } -// NewMessage creates and initializes a non-signed, non-encrypted Whisper message. +// NewSentMessage creates and initializes a non-signed, non-encrypted Whisper message. func NewSentMessage(params *MessageParams) (*sentMessage, error) { msg := sentMessage{} msg.Raw = make([]byte, 1, len(params.Payload)+len(params.Padding)+signatureLength+padSizeLimit) @@ -330,7 +330,7 @@ func (msg *ReceivedMessage) extractPadding(end int) (int, bool) { return paddingSize, true } -// Recover retrieves the public key of the message signer. +// SigToPubKey retrieves the public key of the message signer. func (msg *ReceivedMessage) SigToPubKey() *ecdsa.PublicKey { defer func() { recover() }() // in case of invalid signature diff --git a/whisper/whisperv5/peer.go b/whisper/whisperv5/peer.go index a34aa94363..5f76d1dc35 100644 --- a/whisper/whisperv5/peer.go +++ b/whisper/whisperv5/peer.go @@ -27,7 +27,7 @@ import ( set "gopkg.in/fatih/set.v0" ) -// peer represents a whisper protocol peer connection. +// Peer represents a whisper protocol peer connection. type Peer struct { host *Whisper peer *p2p.Peer @@ -53,51 +53,51 @@ func newPeer(host *Whisper, remote *p2p.Peer, rw p2p.MsgReadWriter) *Peer { // start initiates the peer updater, periodically broadcasting the whisper packets // into the network. -func (p *Peer) start() { - go p.update() - log.Trace("start", "peer", p.ID()) +func (peer *Peer) start() { + go peer.update() + log.Trace("start", "peer", peer.ID()) } // stop terminates the peer updater, stopping message forwarding to it. -func (p *Peer) stop() { - close(p.quit) - log.Trace("stop", "peer", p.ID()) +func (peer *Peer) stop() { + close(peer.quit) + log.Trace("stop", "peer", peer.ID()) } // handshake sends the protocol initiation status message to the remote peer and // verifies the remote status too. -func (p *Peer) handshake() error { +func (peer *Peer) handshake() error { // Send the handshake status message asynchronously errc := make(chan error, 1) go func() { - errc <- p2p.Send(p.ws, statusCode, ProtocolVersion) + errc <- p2p.Send(peer.ws, statusCode, ProtocolVersion) }() // Fetch the remote status packet and verify protocol match - packet, err := p.ws.ReadMsg() + packet, err := peer.ws.ReadMsg() if err != nil { return err } if packet.Code != statusCode { - return fmt.Errorf("peer [%x] sent packet %x before status packet", p.ID(), packet.Code) + return fmt.Errorf("peer [%x] sent packet %x before status packet", peer.ID(), packet.Code) } s := rlp.NewStream(packet.Payload, uint64(packet.Size)) peerVersion, err := s.Uint() if err != nil { - return fmt.Errorf("peer [%x] sent bad status message: %v", p.ID(), err) + return fmt.Errorf("peer [%x] sent bad status message: %v", peer.ID(), err) } if peerVersion != ProtocolVersion { - return fmt.Errorf("peer [%x]: protocol version mismatch %d != %d", p.ID(), peerVersion, ProtocolVersion) + return fmt.Errorf("peer [%x]: protocol version mismatch %d != %d", peer.ID(), peerVersion, ProtocolVersion) } // Wait until out own status is consumed too if err := <-errc; err != nil { - return fmt.Errorf("peer [%x] failed to send status packet: %v", p.ID(), err) + return fmt.Errorf("peer [%x] failed to send status packet: %v", peer.ID(), err) } return nil } // update executes periodic operations on the peer, including message transmission // and expiration. -func (p *Peer) update() { +func (peer *Peer) update() { // Start the tickers for the updates expire := time.NewTicker(expirationCycle) transmit := time.NewTicker(transmissionCycle) @@ -106,15 +106,15 @@ func (p *Peer) update() { for { select { case <-expire.C: - p.expire() + peer.expire() case <-transmit.C: - if err := p.broadcast(); err != nil { - log.Trace("broadcast failed", "reason", err, "peer", p.ID()) + if err := peer.broadcast(); err != nil { + log.Trace("broadcast failed", "reason", err, "peer", peer.ID()) return } - case <-p.quit: + case <-peer.quit: return } } @@ -148,16 +148,16 @@ func (peer *Peer) expire() { // broadcast iterates over the collection of envelopes and transmits yet unknown // ones over the network. -func (p *Peer) broadcast() error { +func (peer *Peer) broadcast() error { var cnt int - envelopes := p.host.Envelopes() + envelopes := peer.host.Envelopes() for _, envelope := range envelopes { - if !p.marked(envelope) { - err := p2p.Send(p.ws, messagesCode, envelope) + if !peer.marked(envelope) { + err := p2p.Send(peer.ws, messagesCode, envelope) if err != nil { return err } else { - p.mark(envelope) + peer.mark(envelope) cnt++ } } @@ -168,7 +168,7 @@ func (p *Peer) broadcast() error { return nil } -func (p *Peer) ID() []byte { - id := p.peer.ID() +func (peer *Peer) ID() []byte { + id := peer.peer.ID() return id[:] } diff --git a/whisper/whisperv5/peer_test.go b/whisper/whisperv5/peer_test.go index 042db47775..75ab525900 100644 --- a/whisper/whisperv5/peer_test.go +++ b/whisper/whisperv5/peer_test.go @@ -32,7 +32,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/p2p/nat" ) -var keys []string = []string{ +var keys = []string{ "d49dcf37238dc8a7aac57dc61b9fee68f0a97f062968978b9fafa7d1033d03a9", "73fd6143c48e80ed3c56ea159fe7494a0b6b393a392227b422f4c3e8f1b54f98", "119dd32adb1daa7a4c7bf77f847fb28730785aa92947edf42fdd997b54de40dc", @@ -84,9 +84,9 @@ type TestNode struct { var result TestData var nodes [NumNodes]*TestNode -var sharedKey []byte = []byte("some arbitrary data here") +var sharedKey = []byte("some arbitrary data here") var sharedTopic TopicType = TopicType{0xF, 0x1, 0x2, 0} -var expectedMessage []byte = []byte("per rectum ad astra") +var expectedMessage = []byte("per rectum ad astra") // This test does the following: // 1. creates a chain of whisper nodes, diff --git a/whisper/whisperv5/topic.go b/whisper/whisperv5/topic.go index 191aa4aaee..4d7d85dbbf 100644 --- a/whisper/whisperv5/topic.go +++ b/whisper/whisperv5/topic.go @@ -23,7 +23,7 @@ import ( "github.com/EthereumCommonwealth/go-callisto/common/hexutil" ) -// Topic represents a cryptographically secure, probabilistic partial +// TopicType represents a cryptographically secure, probabilistic partial // classifications of a message, determined as the first (left) 4 bytes of the // SHA3 hash of some arbitrary data given by the original author of the message. type TopicType [TopicLength]byte diff --git a/whisper/whisperv5/whisper.go b/whisper/whisperv5/whisper.go index 1953d2ba55..780c56ef5a 100644 --- a/whisper/whisperv5/whisper.go +++ b/whisper/whisperv5/whisper.go @@ -469,18 +469,18 @@ func (w *Whisper) Stop() error { // HandlePeer is called by the underlying P2P layer when the whisper sub-protocol // connection is negotiated. -func (wh *Whisper) HandlePeer(peer *p2p.Peer, rw p2p.MsgReadWriter) error { +func (w *Whisper) HandlePeer(peer *p2p.Peer, rw p2p.MsgReadWriter) error { // Create the new peer and start tracking it - whisperPeer := newPeer(wh, peer, rw) + whisperPeer := newPeer(w, peer, rw) - wh.peerMu.Lock() - wh.peers[whisperPeer] = struct{}{} - wh.peerMu.Unlock() + w.peerMu.Lock() + w.peers[whisperPeer] = struct{}{} + w.peerMu.Unlock() defer func() { - wh.peerMu.Lock() - delete(wh.peers, whisperPeer) - wh.peerMu.Unlock() + w.peerMu.Lock() + delete(w.peers, whisperPeer) + w.peerMu.Unlock() }() // Run the peer handshake and state updates @@ -490,11 +490,11 @@ func (wh *Whisper) HandlePeer(peer *p2p.Peer, rw p2p.MsgReadWriter) error { whisperPeer.start() defer whisperPeer.stop() - return wh.runMessageLoop(whisperPeer, rw) + return w.runMessageLoop(whisperPeer, rw) } // runMessageLoop reads and processes inbound messages directly to merge into client-global state. -func (wh *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error { +func (w *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error { for { // fetch the next packet packet, err := rw.ReadMsg() @@ -502,7 +502,7 @@ func (wh *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error { log.Warn("message loop", "peer", p.peer.ID(), "err", err) return err } - if packet.Size > wh.MaxMessageSize() { + if packet.Size > w.MaxMessageSize() { log.Warn("oversized message received", "peer", p.peer.ID()) return errors.New("oversized message received") } @@ -518,7 +518,7 @@ func (wh *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error { log.Warn("failed to decode envelope, peer will be disconnected", "peer", p.peer.ID(), "err", err) return errors.New("invalid envelope") } - cached, err := wh.add(&envelope) + cached, err := w.add(&envelope) if err != nil { log.Warn("bad envelope received, peer will be disconnected", "peer", p.peer.ID(), "err", err) return errors.New("invalid envelope") @@ -537,17 +537,17 @@ func (wh *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error { log.Warn("failed to decode direct message, peer will be disconnected", "peer", p.peer.ID(), "err", err) return errors.New("invalid direct message") } - wh.postEvent(&envelope, true) + w.postEvent(&envelope, true) } case p2pRequestCode: // Must be processed if mail server is implemented. Otherwise ignore. - if wh.mailServer != nil { + if w.mailServer != nil { var request Envelope if err := packet.Decode(&request); err != nil { log.Warn("failed to decode p2p request message, peer will be disconnected", "peer", p.peer.ID(), "err", err) return errors.New("invalid p2p request") } - wh.mailServer.DeliverMail(p, &request) + w.mailServer.DeliverMail(p, &request) } default: // New message types might be implemented in the future versions of Whisper. @@ -561,29 +561,27 @@ func (wh *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error { // add inserts a new envelope into the message pool to be distributed within the // whisper network. It also inserts the envelope into the expiration pool at the // appropriate time-stamp. In case of error, connection should be dropped. -func (wh *Whisper) add(envelope *Envelope) (bool, error) { +func (w *Whisper) add(envelope *Envelope) (bool, error) { now := uint32(time.Now().Unix()) sent := envelope.Expiry - envelope.TTL if sent > now { if sent-SynchAllowance > now { return false, fmt.Errorf("envelope created in the future [%x]", envelope.Hash()) - } else { - // recalculate PoW, adjusted for the time difference, plus one second for latency - envelope.calculatePoW(sent - now + 1) } + // recalculate PoW, adjusted for the time difference, plus one second for latency + envelope.calculatePoW(sent - now + 1) } if envelope.Expiry < now { if envelope.Expiry+SynchAllowance*2 < now { return false, fmt.Errorf("very old message") - } else { - log.Debug("expired envelope dropped", "hash", envelope.Hash().Hex()) - return false, nil // drop envelope without error } + log.Debug("expired envelope dropped", "hash", envelope.Hash().Hex()) + return false, nil // drop envelope without error } - if uint32(envelope.size()) > wh.MaxMessageSize() { + if uint32(envelope.size()) > w.MaxMessageSize() { return false, fmt.Errorf("huge messages are not allowed [%x]", envelope.Hash()) } @@ -598,36 +596,36 @@ func (wh *Whisper) add(envelope *Envelope) (bool, error) { return false, fmt.Errorf("wrong size of AESNonce: %d bytes [env: %x]", aesNonceSize, envelope.Hash()) } - if envelope.PoW() < wh.MinPow() { + if envelope.PoW() < w.MinPow() { log.Debug("envelope with low PoW dropped", "PoW", envelope.PoW(), "hash", envelope.Hash().Hex()) return false, nil // drop envelope without error } hash := envelope.Hash() - wh.poolMu.Lock() - _, alreadyCached := wh.envelopes[hash] + w.poolMu.Lock() + _, alreadyCached := w.envelopes[hash] if !alreadyCached { - wh.envelopes[hash] = envelope - if wh.expirations[envelope.Expiry] == nil { - wh.expirations[envelope.Expiry] = set.NewNonTS() + w.envelopes[hash] = envelope + if w.expirations[envelope.Expiry] == nil { + w.expirations[envelope.Expiry] = set.NewNonTS() } - if !wh.expirations[envelope.Expiry].Has(hash) { - wh.expirations[envelope.Expiry].Add(hash) + if !w.expirations[envelope.Expiry].Has(hash) { + w.expirations[envelope.Expiry].Add(hash) } } - wh.poolMu.Unlock() + w.poolMu.Unlock() if alreadyCached { log.Trace("whisper envelope already cached", "hash", envelope.Hash().Hex()) } else { log.Trace("cached whisper envelope", "hash", envelope.Hash().Hex()) - wh.statsMu.Lock() - wh.stats.memoryUsed += envelope.size() - wh.statsMu.Unlock() - wh.postEvent(envelope, false) // notify the local node about the new message - if wh.mailServer != nil { - wh.mailServer.Archive(envelope) + w.statsMu.Lock() + w.stats.memoryUsed += envelope.size() + w.statsMu.Unlock() + w.postEvent(envelope, false) // notify the local node about the new message + if w.mailServer != nil { + w.mailServer.Archive(envelope) } } return true, nil @@ -838,9 +836,8 @@ func deriveKeyMaterial(key []byte, version uint64) (derivedKey []byte, err error // because it's a once in a session experience derivedKey := pbkdf2.Key(key, nil, 65356, aesKeyLength, sha256.New) return derivedKey, nil - } else { - return nil, unknownVersionError(version) } + return nil, unknownVersionError(version) } // GenerateRandomID generates a random string, which is then returned to be used as a key id