From dadf4d53abd8be13f59fb09820712d74ce8f89a7 Mon Sep 17 00:00:00 2001 From: Vlad Date: Tue, 27 Feb 2018 15:45:00 +0100 Subject: [PATCH 01/21] whisper: mailserver no longer supports the signature vaidation --- whisper/mailserver/mailserver.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/whisper/mailserver/mailserver.go b/whisper/mailserver/mailserver.go index 6555fd5c0b..a889c2bd23 100644 --- a/whisper/mailserver/mailserver.go +++ b/whisper/mailserver/mailserver.go @@ -17,7 +17,6 @@ package mailserver import ( - "bytes" "encoding/binary" "fmt" @@ -175,7 +174,10 @@ func (s *WMailServer) validateRequest(peerID []byte, request *whisper.Envelope) if len(src)-len(peerID) == 1 { src = src[1:] } - if !bytes.Equal(peerID, src) { + + // if you want to check the signature, you can do it here. e.g.: + // if !bytes.Equal(peerID, src) { + if src == nil { log.Warn(fmt.Sprintf("Wrong signature of p2p request")) return false, 0, 0, topic } From 5e30a5f66e126d35c00276abffe32d99182a1bc7 Mon Sep 17 00:00:00 2001 From: Vlad Date: Tue, 27 Feb 2018 15:52:10 +0100 Subject: [PATCH 02/21] whisper: test fixed --- whisper/mailserver/server_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/whisper/mailserver/server_test.go b/whisper/mailserver/server_test.go index c8e0a553a0..aeb317e597 100644 --- a/whisper/mailserver/server_test.go +++ b/whisper/mailserver/server_test.go @@ -167,7 +167,8 @@ func singleRequest(t *testing.T, server *WMailServer, env *whisper.Envelope, p * src[0]++ ok, lower, upper, topic = server.validateRequest(src, request) - if ok { + if !ok { + // request should be valid regardless of signature t.Fatalf("request validation false positive, seed: %d (lower: %d, upper: %d).", seed, lower, upper) } } From 17b0e226d3d6b3829a82ee1c142bd125cc9a0109 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 27 Feb 2018 18:25:56 +0200 Subject: [PATCH 03/21] travis, build, consensus: drop support for Go 1.7 --- .travis.yml | 11 ----- build/ci.go | 6 +-- consensus/ethash/algorithm.go | 43 ++++++++++++++++ consensus/ethash/algorithm_go1.7.go | 47 ------------------ consensus/ethash/algorithm_go1.8.go | 63 ------------------------ consensus/ethash/algorithm_go1.8_test.go | 37 -------------- consensus/ethash/algorithm_test.go | 16 ++++++ consensus/ethash/consensus.go | 10 +--- 8 files changed, 64 insertions(+), 169 deletions(-) delete mode 100644 consensus/ethash/algorithm_go1.7.go delete mode 100644 consensus/ethash/algorithm_go1.8.go delete mode 100644 consensus/ethash/algorithm_go1.8_test.go diff --git a/.travis.yml b/.travis.yml index 3941fa785b..a76a78954d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,17 +3,6 @@ go_import_path: github.com/ethereum/go-ethereum sudo: false matrix: include: - - os: linux - dist: trusty - sudo: required - go: 1.7.x - script: - - sudo modprobe fuse - - sudo chmod 666 /dev/fuse - - sudo chown root:$USER /etc/fuse.conf - - go run build/ci.go install - - go run build/ci.go test -coverage - - os: linux dist: trusty sudo: required diff --git a/build/ci.go b/build/ci.go index 544483c42c..24b58c1ae4 100644 --- a/build/ci.go +++ b/build/ci.go @@ -182,13 +182,13 @@ func doInstall(cmdline []string) { // Check Go version. People regularly open issues about compilation // failure with outdated Go. This should save them the trouble. if !strings.Contains(runtime.Version(), "devel") { - // Figure out the minor version number since we can't textually compare (1.10 < 1.7) + // Figure out the minor version number since we can't textually compare (1.10 < 1.8) var minor int fmt.Sscanf(strings.TrimPrefix(runtime.Version(), "go1."), "%d", &minor) - if minor < 7 { + if minor < 8 { log.Println("You have Go version", runtime.Version()) - log.Println("go-ethereum requires at least Go version 1.7 and cannot") + log.Println("go-ethereum requires at least Go version 1.8 and cannot") log.Println("be compiled with an earlier version. Please upgrade your Go installation.") os.Exit(1) } diff --git a/consensus/ethash/algorithm.go b/consensus/ethash/algorithm.go index 10767bb312..905a7b1ea7 100644 --- a/consensus/ethash/algorithm.go +++ b/consensus/ethash/algorithm.go @@ -19,6 +19,7 @@ package ethash import ( "encoding/binary" "hash" + "math/big" "reflect" "runtime" "sync" @@ -47,6 +48,48 @@ const ( loopAccesses = 64 // Number of accesses in hashimoto loop ) +// cacheSize returns the size of the ethash verification cache that belongs to a certain +// block number. +func cacheSize(block uint64) uint64 { + epoch := int(block / epochLength) + if epoch < maxEpoch { + return cacheSizes[epoch] + } + return calcCacheSize(epoch) +} + +// calcCacheSize calculates the cache size for epoch. The cache size grows linearly, +// however, we always take the highest prime below the linearly growing threshold in order +// to reduce the risk of accidental regularities leading to cyclic behavior. +func calcCacheSize(epoch int) uint64 { + size := cacheInitBytes + cacheGrowthBytes*uint64(epoch) - hashBytes + for !new(big.Int).SetUint64(size / hashBytes).ProbablyPrime(1) { // Always accurate for n < 2^64 + size -= 2 * hashBytes + } + return size +} + +// datasetSize returns the size of the ethash mining dataset that belongs to a certain +// block number. +func datasetSize(block uint64) uint64 { + epoch := int(block / epochLength) + if epoch < maxEpoch { + return datasetSizes[epoch] + } + return calcDatasetSize(epoch) +} + +// calcDatasetSize calculates the dataset size for epoch. The dataset size grows linearly, +// however, we always take the highest prime below the linearly growing threshold in order +// to reduce the risk of accidental regularities leading to cyclic behavior. +func calcDatasetSize(epoch int) uint64 { + size := datasetInitBytes + datasetGrowthBytes*uint64(epoch) - mixBytes + for !new(big.Int).SetUint64(size / mixBytes).ProbablyPrime(1) { // Always accurate for n < 2^64 + size -= 2 * mixBytes + } + return size +} + // hasher is a repetitive hasher allowing the same hash data structures to be // reused between hash runs instead of requiring new ones to be created. type hasher func(dest []byte, data []byte) diff --git a/consensus/ethash/algorithm_go1.7.go b/consensus/ethash/algorithm_go1.7.go deleted file mode 100644 index c7f7f48e41..0000000000 --- a/consensus/ethash/algorithm_go1.7.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// +build !go1.8 - -package ethash - -// cacheSize calculates and returns the size of the ethash verification cache that -// belongs to a certain block number. The cache size grows linearly, however, we -// always take the highest prime below the linearly growing threshold in order to -// reduce the risk of accidental regularities leading to cyclic behavior. -func cacheSize(block uint64) uint64 { - // If we have a pre-generated value, use that - epoch := int(block / epochLength) - if epoch < maxEpoch { - return cacheSizes[epoch] - } - // We don't have a way to verify primes fast before Go 1.8 - panic("fast prime testing unsupported in Go < 1.8") -} - -// datasetSize calculates and returns the size of the ethash mining dataset that -// belongs to a certain block number. The dataset size grows linearly, however, we -// always take the highest prime below the linearly growing threshold in order to -// reduce the risk of accidental regularities leading to cyclic behavior. -func datasetSize(block uint64) uint64 { - // If we have a pre-generated value, use that - epoch := int(block / epochLength) - if epoch < maxEpoch { - return datasetSizes[epoch] - } - // We don't have a way to verify primes fast before Go 1.8 - panic("fast prime testing unsupported in Go < 1.8") -} diff --git a/consensus/ethash/algorithm_go1.8.go b/consensus/ethash/algorithm_go1.8.go deleted file mode 100644 index 975fdffe51..0000000000 --- a/consensus/ethash/algorithm_go1.8.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// +build go1.8 - -package ethash - -import "math/big" - -// cacheSize returns the size of the ethash verification cache that belongs to a certain -// block number. -func cacheSize(block uint64) uint64 { - epoch := int(block / epochLength) - if epoch < maxEpoch { - return cacheSizes[epoch] - } - return calcCacheSize(epoch) -} - -// calcCacheSize calculates the cache size for epoch. The cache size grows linearly, -// however, we always take the highest prime below the linearly growing threshold in order -// to reduce the risk of accidental regularities leading to cyclic behavior. -func calcCacheSize(epoch int) uint64 { - size := cacheInitBytes + cacheGrowthBytes*uint64(epoch) - hashBytes - for !new(big.Int).SetUint64(size / hashBytes).ProbablyPrime(1) { // Always accurate for n < 2^64 - size -= 2 * hashBytes - } - return size -} - -// datasetSize returns the size of the ethash mining dataset that belongs to a certain -// block number. -func datasetSize(block uint64) uint64 { - epoch := int(block / epochLength) - if epoch < maxEpoch { - return datasetSizes[epoch] - } - return calcDatasetSize(epoch) -} - -// calcDatasetSize calculates the dataset size for epoch. The dataset size grows linearly, -// however, we always take the highest prime below the linearly growing threshold in order -// to reduce the risk of accidental regularities leading to cyclic behavior. -func calcDatasetSize(epoch int) uint64 { - size := datasetInitBytes + datasetGrowthBytes*uint64(epoch) - mixBytes - for !new(big.Int).SetUint64(size / mixBytes).ProbablyPrime(1) { // Always accurate for n < 2^64 - size -= 2 * mixBytes - } - return size -} diff --git a/consensus/ethash/algorithm_go1.8_test.go b/consensus/ethash/algorithm_go1.8_test.go deleted file mode 100644 index 6648bd6a97..0000000000 --- a/consensus/ethash/algorithm_go1.8_test.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// +build go1.8 - -package ethash - -import "testing" - -// Tests whether the dataset size calculator works correctly by cross checking the -// hard coded lookup table with the value generated by it. -func TestSizeCalculations(t *testing.T) { - // Verify all the cache and dataset sizes from the lookup table. - for epoch, want := range cacheSizes { - if size := calcCacheSize(epoch); size != want { - t.Errorf("cache %d: cache size mismatch: have %d, want %d", epoch, size, want) - } - } - for epoch, want := range datasetSizes { - if size := calcDatasetSize(epoch); size != want { - t.Errorf("dataset %d: dataset size mismatch: have %d, want %d", epoch, size, want) - } - } -} diff --git a/consensus/ethash/algorithm_test.go b/consensus/ethash/algorithm_test.go index a54f3b582c..841e39233f 100644 --- a/consensus/ethash/algorithm_test.go +++ b/consensus/ethash/algorithm_test.go @@ -30,6 +30,22 @@ import ( "github.com/ethereum/go-ethereum/core/types" ) +// Tests whether the dataset size calculator works correctly by cross checking the +// hard coded lookup table with the value generated by it. +func TestSizeCalculations(t *testing.T) { + // Verify all the cache and dataset sizes from the lookup table. + for epoch, want := range cacheSizes { + if size := calcCacheSize(epoch); size != want { + t.Errorf("cache %d: cache size mismatch: have %d, want %d", epoch, size, want) + } + } + for epoch, want := range datasetSizes { + if size := calcDatasetSize(epoch); size != want { + t.Errorf("dataset %d: dataset size mismatch: have %d, want %d", epoch, size, want) + } + } +} + // Tests that verification caches can be correctly generated. func TestCacheGeneration(t *testing.T) { tests := []struct { diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 92a23d4a4d..3f860bf473 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -53,7 +53,6 @@ var ( errDuplicateUncle = errors.New("duplicate uncle") errUncleIsAncestor = errors.New("uncle is ancestor") errDanglingUncle = errors.New("uncle's parent is not ancestor") - errNonceOutOfRange = errors.New("nonce out of range") errInvalidDifficulty = errors.New("non-positive difficulty") errInvalidMixDigest = errors.New("invalid mix digest") errInvalidPoW = errors.New("invalid proof-of-work") @@ -474,18 +473,13 @@ func (ethash *Ethash) VerifySeal(chain consensus.ChainReader, header *types.Head if ethash.shared != nil { return ethash.shared.VerifySeal(chain, header) } - // Sanity check that the block number is below the lookup table size (60M blocks) - number := header.Number.Uint64() - if number/epochLength >= maxEpoch { - // Go < 1.7 cannot calculate new cache/dataset sizes (no fast prime check) - return errNonceOutOfRange - } // Ensure that we have a valid difficulty for the block if header.Difficulty.Sign() <= 0 { return errInvalidDifficulty } - // Recompute the digest and PoW value and verify against the header + number := header.Number.Uint64() + cache := ethash.cache(number) size := datasetSize(number) if ethash.config.PowMode == ModeTest { From 014d8d98370abf770b89f7987a92999133e3d4ea Mon Sep 17 00:00:00 2001 From: Vlad Date: Tue, 27 Feb 2018 21:16:15 +0100 Subject: [PATCH 04/21] whisper: message filtering optimized --- whisper/whisperv6/filter.go | 66 ++++++++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 11 deletions(-) diff --git a/whisper/whisperv6/filter.go b/whisper/whisperv6/filter.go index eb0c65fa3b..75c31209c9 100644 --- a/whisper/whisperv6/filter.go +++ b/whisper/whisperv6/filter.go @@ -35,6 +35,7 @@ type Filter struct { PoW float64 // Proof of work as described in the Whisper spec AllowP2P bool // Indicates whether this filter is interested in direct peer-to-peer messages SymKeyHash common.Hash // The Keccak256Hash of the symmetric key, needed for optimization + id string // unique identifier Messages map[common.Hash]*ReceivedMessage mutex sync.RWMutex @@ -42,16 +43,21 @@ type Filter struct { // Filters represents a collection of filters type Filters struct { - watchers map[string]*Filter - whisper *Whisper - mutex sync.RWMutex + watchers map[string]*Filter + topicMatcher map[TopicType]map[*Filter]struct{} + allTopicsMatcher map[*Filter]struct{} + + whisper *Whisper + mutex sync.RWMutex } // NewFilters returns a newly created filter collection func NewFilters(w *Whisper) *Filters { return &Filters{ - watchers: make(map[string]*Filter), - whisper: w, + watchers: make(map[string]*Filter), + topicMatcher: make(map[TopicType]map[*Filter]struct{}), + allTopicsMatcher: make(map[*Filter]struct{}), + whisper: w, } } @@ -81,7 +87,9 @@ func (fs *Filters) Install(watcher *Filter) (string, error) { watcher.SymKeyHash = crypto.Keccak256Hash(watcher.KeySym) } + watcher.id = id fs.watchers[id] = watcher + fs.addTopicMatcher(watcher) return id, err } @@ -91,12 +99,49 @@ func (fs *Filters) Uninstall(id string) bool { fs.mutex.Lock() defer fs.mutex.Unlock() if fs.watchers[id] != nil { + fs.removeFromTopicMatchers(fs.watchers[id]) delete(fs.watchers, id) return true } return false } +// addTopicMatcher adds a filter to the topic matchers +func (fs *Filters) addTopicMatcher(watcher *Filter) { + if len(watcher.Topics) == 0 { + fs.allTopicsMatcher[watcher] = struct{}{} + } else { + for _, t := range watcher.Topics { + topic := BytesToTopic(t) + if fs.topicMatcher[topic] == nil { + fs.topicMatcher[topic] = make(map[*Filter]struct{}) + } + fs.topicMatcher[topic][watcher] = struct{}{} + } + } +} + +// removeFromTopicMatchers removes a filter from the topic matchers +func (fs *Filters) removeFromTopicMatchers(watcher *Filter) { + delete(fs.allTopicsMatcher, watcher) + for _, topic := range watcher.Topics { + delete(fs.topicMatcher[BytesToTopic(topic)], watcher) + } +} + +// getWatchersByTopic returns a slice containing the filters that +// match a specific topic +func (fs *Filters) getWatchersByTopic(topic TopicType) []*Filter { + res := make([]*Filter, 0, len(fs.allTopicsMatcher)) + for watcher, _ := range fs.allTopicsMatcher { + res = append(res, watcher) + } + for watcher, _ := range fs.topicMatcher[topic] { + res = append(res, watcher) + } + return res +} + // Get returns a filter from the collection with a specific ID func (fs *Filters) Get(id string) *Filter { fs.mutex.RLock() @@ -112,11 +157,10 @@ func (fs *Filters) NotifyWatchers(env *Envelope, p2pMessage bool) { fs.mutex.RLock() defer fs.mutex.RUnlock() - i := -1 // only used for logging info - for _, watcher := range fs.watchers { - i++ + candidates := fs.getWatchersByTopic(env.Topic) + for _, watcher := range candidates { if p2pMessage && !watcher.AllowP2P { - log.Trace(fmt.Sprintf("msg [%x], filter [%d]: p2p messages are not allowed", env.Hash(), i)) + log.Trace(fmt.Sprintf("msg [%x], filter [%s]: p2p messages are not allowed", env.Hash(), watcher.id)) continue } @@ -128,10 +172,10 @@ func (fs *Filters) NotifyWatchers(env *Envelope, p2pMessage bool) { if match { msg = env.Open(watcher) if msg == nil { - log.Trace("processing message: failed to open", "message", env.Hash().Hex(), "filter", i) + log.Trace("processing message: failed to open", "message", env.Hash().Hex(), "filter", watcher.id) } } else { - log.Trace("processing message: does not match", "message", env.Hash().Hex(), "filter", i) + log.Trace("processing message: does not match", "message", env.Hash().Hex(), "filter", watcher.id) } } From c733792be4b49ec47c5307b1a3b8fa116ea16933 Mon Sep 17 00:00:00 2001 From: Vlad Date: Tue, 27 Feb 2018 23:38:20 +0100 Subject: [PATCH 05/21] whsiper: refactoring --- whisper/whisperv6/filter.go | 29 ++++++----------------------- whisper/whisperv6/filter_test.go | 25 +++++++++++++++---------- whisper/whisperv6/whisper.go | 18 ------------------ whisper/whisperv6/whisper_test.go | 11 +---------- 4 files changed, 22 insertions(+), 61 deletions(-) diff --git a/whisper/whisperv6/filter.go b/whisper/whisperv6/filter.go index 75c31209c9..d7b71795de 100644 --- a/whisper/whisperv6/filter.go +++ b/whisper/whisperv6/filter.go @@ -238,16 +238,17 @@ func (f *Filter) Retrieve() (all []*ReceivedMessage) { // MatchMessage checks if the filter matches an already decrypted // message (i.e. a Message that has already been handled by -// MatchEnvelope when checked by a previous filter) +// MatchEnvelope when checked by a previous filter). +// Topics are not checked here, since this is done by topic matchers. func (f *Filter) MatchMessage(msg *ReceivedMessage) bool { if f.PoW > 0 && msg.PoW < f.PoW { return false } if f.expectsAsymmetricEncryption() && msg.isAsymmetricEncryption() { - return IsPubKeyEqual(&f.KeyAsym.PublicKey, msg.Dst) && f.MatchTopic(msg.Topic) + return IsPubKeyEqual(&f.KeyAsym.PublicKey, msg.Dst) } else if f.expectsSymmetricEncryption() && msg.isSymmetricEncryption() { - return f.SymKeyHash == msg.SymKeyHash && f.MatchTopic(msg.Topic) + return f.SymKeyHash == msg.SymKeyHash } return false } @@ -255,27 +256,9 @@ func (f *Filter) MatchMessage(msg *ReceivedMessage) bool { // MatchEnvelope checks if it's worth decrypting the message. If // it returns `true`, client code is expected to attempt decrypting // the message and subsequently call MatchMessage. +// Topics are not checked here, since this is done by topic matchers. func (f *Filter) MatchEnvelope(envelope *Envelope) bool { - if f.PoW > 0 && envelope.pow < f.PoW { - return false - } - - return f.MatchTopic(envelope.Topic) -} - -// MatchTopic checks that the filter captures a given topic. -func (f *Filter) MatchTopic(topic TopicType) bool { - if len(f.Topics) == 0 { - // any topic matches - return true - } - - for _, bt := range f.Topics { - if matchSingleTopic(topic, bt) { - return true - } - } - return false + return f.PoW <= 0 || envelope.pow >= f.PoW } func matchSingleTopic(topic TopicType, bt []byte) bool { diff --git a/whisper/whisperv6/filter_test.go b/whisper/whisperv6/filter_test.go index e7230ef388..491e137bdb 100644 --- a/whisper/whisperv6/filter_test.go +++ b/whisper/whisperv6/filter_test.go @@ -303,9 +303,8 @@ func TestMatchEnvelope(t *testing.T) { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - params.Topic[0] = 0xFF // ensure mismatch + params.Topic[0] = 0xFF // topic mismatch - // mismatch with pseudo-random data msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) @@ -315,11 +314,13 @@ func TestMatchEnvelope(t *testing.T) { t.Fatalf("failed Wrap with seed %d: %s.", seed, err) } match := fsym.MatchEnvelope(env) - if match { + if !match { + // topic mismatch should have no affect, as topics are handled by topic matchers t.Fatalf("failed MatchEnvelope symmetric with seed %d.", seed) } match = fasym.MatchEnvelope(env) - if match { + if !match { + // topic mismatch should have no affect, as topics are handled by topic matchers t.Fatalf("failed MatchEnvelope asymmetric with seed %d.", seed) } @@ -396,7 +397,7 @@ func TestMatchEnvelope(t *testing.T) { // asymmetric + matching topic: match fasym.Topics[i] = fasym.Topics[i+1] match = fasym.MatchEnvelope(env) - if match { + if !match { t.Fatalf("failed MatchEnvelope(asymmetric + matching topic) with seed %d.", seed) } @@ -431,7 +432,8 @@ func TestMatchEnvelope(t *testing.T) { // filter with topic + envelope without topic: mismatch fasym.Topics = fsym.Topics match = fasym.MatchEnvelope(env) - if match { + if !match { + // topic mismatch should have no affect, as topics are handled by topic matchers t.Fatalf("failed MatchEnvelope(filter without topic + envelope without topic) with seed %d.", seed) } } @@ -487,7 +489,8 @@ func TestMatchMessageSym(t *testing.T) { // topic mismatch f.Topics[index][0]++ - if f.MatchMessage(msg) { + if !f.MatchMessage(msg) { + // topic mismatch should have no affect, as topics are handled by topic matchers t.Fatalf("failed MatchEnvelope(topic mismatch) with seed %d.", seed) } f.Topics[index][0]-- @@ -580,7 +583,8 @@ func TestMatchMessageAsym(t *testing.T) { // topic mismatch f.Topics[index][0]++ - if f.MatchMessage(msg) { + if !f.MatchMessage(msg) { + // topic mismatch should have no affect, as topics are handled by topic matchers t.Fatalf("failed MatchEnvelope(topic mismatch) with seed %d.", seed) } f.Topics[index][0]-- @@ -829,8 +833,9 @@ func TestVariableTopics(t *testing.T) { f.Topics[i][lastTopicByte]++ match = f.MatchEnvelope(env) - if match { - t.Fatalf("MatchEnvelope symmetric with seed %d, step %d: false positive.", seed, i) + if !match { + // topic mismatch should have no affect, as topics are handled by topic matchers + t.Fatalf("MatchEnvelope symmetric with seed %d, step %d.", seed, i) } } } diff --git a/whisper/whisperv6/whisper.go b/whisper/whisperv6/whisper.go index b0b601f032..81f59b3cc7 100644 --- a/whisper/whisperv6/whisper.go +++ b/whisper/whisperv6/whisper.go @@ -928,24 +928,6 @@ func (whisper *Whisper) Envelopes() []*Envelope { return all } -// Messages iterates through all currently floating envelopes -// and retrieves all the messages, that this filter could decrypt. -func (whisper *Whisper) Messages(id string) []*ReceivedMessage { - result := make([]*ReceivedMessage, 0) - whisper.poolMu.RLock() - defer whisper.poolMu.RUnlock() - - if filter := whisper.filters.Get(id); filter != nil { - for _, env := range whisper.envelopes { - msg := filter.processEnvelope(env) - if msg != nil { - result = append(result, msg) - } - } - } - return result -} - // isEnvelopeCached checks if envelope with specific hash has already been received and cached. func (whisper *Whisper) isEnvelopeCached(hash common.Hash) bool { whisper.poolMu.Lock() diff --git a/whisper/whisperv6/whisper_test.go b/whisper/whisperv6/whisper_test.go index 99e5f0bbb4..3a219cd133 100644 --- a/whisper/whisperv6/whisper_test.go +++ b/whisper/whisperv6/whisper_test.go @@ -75,10 +75,6 @@ func TestWhisperBasic(t *testing.T) { if len(mail) != 0 { t.Fatalf("failed w.Envelopes().") } - m := w.Messages("non-existent") - if len(m) != 0 { - t.Fatalf("failed w.Messages.") - } derived := pbkdf2.Key([]byte(peerID), nil, 65356, aesKeyLength, sha256.New) if !validateDataIntegrity(derived, aesKeyLength) { @@ -593,7 +589,7 @@ func TestCustomization(t *testing.T) { } // check w.messages() - id, err := w.Subscribe(f) + _, err = w.Subscribe(f) if err != nil { t.Fatalf("failed subscribe with seed %d: %s.", seed, err) } @@ -602,11 +598,6 @@ func TestCustomization(t *testing.T) { if len(mail) > 0 { t.Fatalf("received premature mail") } - - mail = w.Messages(id) - if len(mail) != 2 { - t.Fatalf("failed to get whisper messages") - } } func TestSymmetricSendCycle(t *testing.T) { From a69cb3b4ff55c6ea2bafb7bd0ab0ff4afafb148f Mon Sep 17 00:00:00 2001 From: Vlad Date: Wed, 28 Feb 2018 00:39:38 +0100 Subject: [PATCH 06/21] whisper: comment updated --- whisper/mailserver/server_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/whisper/mailserver/server_test.go b/whisper/mailserver/server_test.go index aeb317e597..63864c1e6b 100644 --- a/whisper/mailserver/server_test.go +++ b/whisper/mailserver/server_test.go @@ -169,7 +169,7 @@ func singleRequest(t *testing.T, server *WMailServer, env *whisper.Envelope, p * ok, lower, upper, topic = server.validateRequest(src, request) if !ok { // request should be valid regardless of signature - t.Fatalf("request validation false positive, seed: %d (lower: %d, upper: %d).", seed, lower, upper) + t.Fatalf("request validation false negative, seed: %d (lower: %d, upper: %d).", seed, lower, upper) } } From cf52d5c91f4f9c583eecca63bce52c7df6dd2169 Mon Sep 17 00:00:00 2001 From: b00ris Date: Wed, 28 Feb 2018 09:50:36 +0300 Subject: [PATCH 07/21] whisper: fixed datarace --- whisper/whisperv6/peer_test.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/whisper/whisperv6/peer_test.go b/whisper/whisperv6/peer_test.go index 9ce5eed8bc..3d3dbd88aa 100644 --- a/whisper/whisperv6/peer_test.go +++ b/whisper/whisperv6/peer_test.go @@ -23,6 +23,7 @@ import ( mrand "math/rand" "net" "sync" + "sync/atomic" "testing" "time" @@ -71,7 +72,7 @@ var keys = []string{ } type TestData struct { - started int + started int64 counter [NumNodes]int mutex sync.RWMutex } @@ -240,9 +241,7 @@ func startServer(t *testing.T, s *p2p.Server) { t.Fatalf("failed to start the fisrt server.") } - result.mutex.Lock() - defer result.mutex.Unlock() - result.started++ + atomic.AddInt64(&result.started, 1) } func stopServers() { @@ -472,7 +471,10 @@ func checkPowExchange(t *testing.T) { func checkBloomFilterExchangeOnce(t *testing.T, mustPass bool) bool { for i, node := range nodes { for peer := range node.shh.peers { - if !bytes.Equal(peer.bloomFilter, masterBloomFilter) { + peer.bloomMu.Lock() + eqals := bytes.Equal(peer.bloomFilter, masterBloomFilter) + peer.bloomMu.Unlock() + if !eqals { if mustPass { t.Fatalf("node %d: failed to exchange bloom filter requirement in round %d. \n%x expected \n%x got", i, round, masterBloomFilter, peer.bloomFilter) @@ -500,11 +502,13 @@ func checkBloomFilterExchange(t *testing.T) { func waitForServersToStart(t *testing.T) { const iterations = 200 + var started int64 for j := 0; j < iterations; j++ { time.Sleep(50 * time.Millisecond) - if result.started == NumNodes { + started = atomic.LoadInt64(&result.started) + if started == NumNodes { return } } - t.Fatalf("Failed to start all the servers, running: %d", result.started) + t.Fatalf("Failed to start all the servers, running: %d", started) } From 98ec5e50115b47670c1f3ffb2cc890ce4838c4d6 Mon Sep 17 00:00:00 2001 From: Mark Rushakoff Date: Wed, 28 Feb 2018 02:20:07 -0800 Subject: [PATCH 08/21] core/asm: rename isAlphaNumeric to isLetter (#16212) The function would return false for numbers, so isLetter is a more accurate description of the behavior. --- core/asm/lexer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/asm/lexer.go b/core/asm/lexer.go index a34b2cbd8f..4054999504 100644 --- a/core/asm/lexer.go +++ b/core/asm/lexer.go @@ -206,7 +206,7 @@ func lexLine(l *lexer) stateFn { return lexComment case isSpace(r): l.ignore() - case isAlphaNumeric(r) || r == '_': + case isLetter(r) || r == '_': return lexElement case isNumber(r): return lexNumber @@ -278,7 +278,7 @@ func lexElement(l *lexer) stateFn { return lexLine } -func isAlphaNumeric(t rune) bool { +func isLetter(t rune) bool { return unicode.IsLetter(t) } From ba7b384019de17a72a8c24e3aaac8a98ff30dcc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 28 Feb 2018 12:40:15 +0200 Subject: [PATCH 09/21] internal/ethapi: fix getTransactionReceipt --- internal/ethapi/api.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index d021b127c5..20a060e727 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1035,14 +1035,14 @@ func (s *PublicTransactionPoolAPI) GetRawTransactionByHash(ctx context.Context, func (s *PublicTransactionPoolAPI) GetTransactionReceipt(ctx context.Context, hash common.Hash) (map[string]interface{}, error) { tx, blockHash, blockNumber, index := core.GetTransaction(s.b.ChainDb(), hash) if tx == nil { - return nil, errors.New("unknown transaction") + return nil, nil } receipts, err := s.b.GetReceipts(ctx, blockHash) if err != nil { return nil, err } if len(receipts) <= int(index) { - return nil, errors.New("unknown receipt") + return nil, nil } receipt := receipts[index] From 62c239f608ec2ec3721c33d5755ca2287c71f085 Mon Sep 17 00:00:00 2001 From: b00ris Date: Wed, 28 Feb 2018 14:38:42 +0300 Subject: [PATCH 10/21] whisper: fix typo --- whisper/whisperv6/peer_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/whisper/whisperv6/peer_test.go b/whisper/whisperv6/peer_test.go index 3d3dbd88aa..6d7754d790 100644 --- a/whisper/whisperv6/peer_test.go +++ b/whisper/whisperv6/peer_test.go @@ -472,9 +472,9 @@ func checkBloomFilterExchangeOnce(t *testing.T, mustPass bool) bool { for i, node := range nodes { for peer := range node.shh.peers { peer.bloomMu.Lock() - eqals := bytes.Equal(peer.bloomFilter, masterBloomFilter) + equals := bytes.Equal(peer.bloomFilter, masterBloomFilter) peer.bloomMu.Unlock() - if !eqals { + if !equals { if mustPass { t.Fatalf("node %d: failed to exchange bloom filter requirement in round %d. \n%x expected \n%x got", i, round, masterBloomFilter, peer.bloomFilter) From d24d10a764457937f1bda72053c82d98ac95dd7a Mon Sep 17 00:00:00 2001 From: Vlad Date: Wed, 28 Feb 2018 15:05:35 +0100 Subject: [PATCH 11/21] whisper: style fixes --- whisper/whisperv6/filter.go | 15 +++++++++------ whisper/whisperv6/filter_test.go | 12 +----------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/whisper/whisperv6/filter.go b/whisper/whisperv6/filter.go index d7b71795de..e4171f85c9 100644 --- a/whisper/whisperv6/filter.go +++ b/whisper/whisperv6/filter.go @@ -43,9 +43,10 @@ type Filter struct { // Filters represents a collection of filters type Filters struct { - watchers map[string]*Filter - topicMatcher map[TopicType]map[*Filter]struct{} - allTopicsMatcher map[*Filter]struct{} + watchers map[string]*Filter + + topicMatcher map[TopicType]map[*Filter]struct{} // map a topic to the filters that are interested in being notified when a message matches that topic + allTopicsMatcher map[*Filter]struct{} // list all the filters that will be notified of a new message, no matter what its topic is whisper *Whisper mutex sync.RWMutex @@ -106,7 +107,9 @@ func (fs *Filters) Uninstall(id string) bool { return false } -// addTopicMatcher adds a filter to the topic matchers +// addTopicMatcher adds a filter to the topic matchers. +// If the filter's Topics array is empty, it will be tried on every topic. +// Otherwise, it will be tried on the topics specified. func (fs *Filters) addTopicMatcher(watcher *Filter) { if len(watcher.Topics) == 0 { fs.allTopicsMatcher[watcher] = struct{}{} @@ -133,10 +136,10 @@ func (fs *Filters) removeFromTopicMatchers(watcher *Filter) { // match a specific topic func (fs *Filters) getWatchersByTopic(topic TopicType) []*Filter { res := make([]*Filter, 0, len(fs.allTopicsMatcher)) - for watcher, _ := range fs.allTopicsMatcher { + for watcher := range fs.allTopicsMatcher { res = append(res, watcher) } - for watcher, _ := range fs.topicMatcher[topic] { + for watcher := range fs.topicMatcher[topic] { res = append(res, watcher) } return res diff --git a/whisper/whisperv6/filter_test.go b/whisper/whisperv6/filter_test.go index 491e137bdb..0bb7986c39 100644 --- a/whisper/whisperv6/filter_test.go +++ b/whisper/whisperv6/filter_test.go @@ -313,16 +313,6 @@ func TestMatchEnvelope(t *testing.T) { if err != nil { t.Fatalf("failed Wrap with seed %d: %s.", seed, err) } - match := fsym.MatchEnvelope(env) - if !match { - // topic mismatch should have no affect, as topics are handled by topic matchers - t.Fatalf("failed MatchEnvelope symmetric with seed %d.", seed) - } - match = fasym.MatchEnvelope(env) - if !match { - // topic mismatch should have no affect, as topics are handled by topic matchers - t.Fatalf("failed MatchEnvelope asymmetric with seed %d.", seed) - } // encrypt symmetrically i := mrand.Int() % 4 @@ -338,7 +328,7 @@ func TestMatchEnvelope(t *testing.T) { } // symmetric + matching topic: match - match = fsym.MatchEnvelope(env) + match := fsym.MatchEnvelope(env) if !match { t.Fatalf("failed MatchEnvelope() symmetric with seed %d.", seed) } From 5a150e1b7724c91009a237ab0879cd64844b390d Mon Sep 17 00:00:00 2001 From: gluk256 Date: Thu, 1 Mar 2018 09:34:46 +0100 Subject: [PATCH 12/21] whisper: serious security issue fixed (#16219) The diagnostic tool was saving the unencrypted version of the messages, which is an obvious security flaw. As of this commit: * encrypted messages saved instead of plain text. * all messages are stored, even that created by the user of wnode. --- cmd/wnode/main.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/cmd/wnode/main.go b/cmd/wnode/main.go index 0f86adb81c..f8606bf824 100644 --- a/cmd/wnode/main.go +++ b/cmd/wnode/main.go @@ -594,19 +594,22 @@ func writeMessageToFile(dir string, msg *whisper.ReceivedMessage) { address = crypto.PubkeyToAddress(*msg.Src) } - if whisper.IsPubKeyEqual(msg.Src, &asymKey.PublicKey) { - // message from myself: don't save, only report - fmt.Printf("\n%s <%x>: message received: '%s'\n", timestamp, address, name) - } else if len(dir) > 0 { + // this is a sample code; uncomment if you don't want to save your own messages. + //if whisper.IsPubKeyEqual(msg.Src, &asymKey.PublicKey) { + // fmt.Printf("\n%s <%x>: message from myself received, not saved: '%s'\n", timestamp, address, name) + // return + //} + + if len(dir) > 0 { fullpath := filepath.Join(dir, name) - err := ioutil.WriteFile(fullpath, msg.Payload, 0644) + err := ioutil.WriteFile(fullpath, msg.Raw, 0644) if err != nil { fmt.Printf("\n%s {%x}: message received but not saved: %s\n", timestamp, address, err) } else { - fmt.Printf("\n%s {%x}: message received and saved as '%s' (%d bytes)\n", timestamp, address, name, len(msg.Payload)) + fmt.Printf("\n%s {%x}: message received and saved as '%s' (%d bytes)\n", timestamp, address, name, len(msg.Raw)) } } else { - fmt.Printf("\n%s {%x}: big message received (%d bytes), but not saved: %s\n", timestamp, address, len(msg.Payload), name) + fmt.Printf("\n%s {%x}: message received (%d bytes), but not saved: %s\n", timestamp, address, len(msg.Raw), name) } } From ee75a90ab41fd9a2e5676a2371e529ac2908befa Mon Sep 17 00:00:00 2001 From: Vlad Date: Thu, 1 Mar 2018 16:04:09 +0100 Subject: [PATCH 13/21] whisper: topics replaced by bloom filters --- cmd/wnode/main.go | 40 +++++++++++++++++++++++----- whisper/mailserver/mailserver.go | 44 ++++++++++++++++--------------- whisper/mailserver/server_test.go | 20 ++++++++------ whisper/whisperv6/doc.go | 2 +- whisper/whisperv6/envelope.go | 2 +- whisper/whisperv6/peer.go | 14 +++++----- whisper/whisperv6/peer_test.go | 6 ++--- whisper/whisperv6/whisper.go | 22 ++++++++-------- whisper/whisperv6/whisper_test.go | 18 ++++++------- 9 files changed, 100 insertions(+), 68 deletions(-) diff --git a/cmd/wnode/main.go b/cmd/wnode/main.go index f8606bf824..ccfdd46265 100644 --- a/cmd/wnode/main.go +++ b/cmd/wnode/main.go @@ -22,6 +22,7 @@ package main import ( "bufio" "crypto/ecdsa" + crand "crypto/rand" "crypto/sha512" "encoding/binary" "encoding/hex" @@ -48,6 +49,7 @@ import ( ) const quitCommand = "~Q" +const entropySize = 32 // singletons var ( @@ -55,6 +57,7 @@ var ( shh *whisper.Whisper done chan struct{} mailServer mailserver.WMailServer + entropy [entropySize]byte input = bufio.NewReader(os.Stdin) ) @@ -274,6 +277,11 @@ func initialize() { TrustedNodes: peers, }, } + + _, err = crand.Read(entropy[:]) + if err != nil { + utils.Fatalf("crypto/rand failed: %s", err) + } } func startServer() { @@ -614,10 +622,10 @@ func writeMessageToFile(dir string, msg *whisper.ReceivedMessage) { } func requestExpiredMessagesLoop() { - var key, peerID []byte + var key, peerID, bloom []byte var timeLow, timeUpp uint32 var t string - var xt, empty whisper.TopicType + var xt whisper.TopicType keyID, err := shh.AddSymKeyFromPassword(msPassword) if err != nil { @@ -640,18 +648,19 @@ func requestExpiredMessagesLoop() { utils.Fatalf("Failed to parse the topic: %s", err) } xt = whisper.BytesToTopic(x) + bloom = whisper.TopicToBloom(xt) + obfuscateBloom(bloom) + } else { + bloom = whisper.MakeFullNodeBloom() } if timeUpp == 0 { timeUpp = 0xFFFFFFFF } - data := make([]byte, 8+whisper.TopicLength) + data := make([]byte, 8, 8+whisper.BloomFilterSize) binary.BigEndian.PutUint32(data, timeLow) binary.BigEndian.PutUint32(data[4:], timeUpp) - copy(data[8:], xt[:]) - if xt == empty { - data = data[:8] - } + data = append(data, bloom...) var params whisper.MessageParams params.PoW = *argServerPoW @@ -685,3 +694,20 @@ func extractIDFromEnode(s string) []byte { } return n.ID[:] } + +// obfuscateBloom adds 16 random bits to the the bloom +// filter, in order to obfuscate the containing topics. +// it does so deterministically within every session. +// despite additional bits, it will match on average +// 32000 times less messages than full node's bloom filter. +func obfuscateBloom(bloom []byte) { + const half = entropySize / 2 + for i := 0; i < half; i++ { + x := int(entropy[i]) + if entropy[half+i] < 128 { + x += 256 + } + + bloom[x/8] = 1 << uint(x%8) // set the bit number X + } +} diff --git a/whisper/mailserver/mailserver.go b/whisper/mailserver/mailserver.go index a889c2bd23..57e6505ad1 100644 --- a/whisper/mailserver/mailserver.go +++ b/whisper/mailserver/mailserver.go @@ -107,17 +107,16 @@ func (s *WMailServer) DeliverMail(peer *whisper.Peer, request *whisper.Envelope) return } - ok, lower, upper, topic := s.validateRequest(peer.ID(), request) + ok, lower, upper, bloom := s.validateRequest(peer.ID(), request) if ok { - s.processRequest(peer, lower, upper, topic) + s.processRequest(peer, lower, upper, bloom) } } -func (s *WMailServer) processRequest(peer *whisper.Peer, lower, upper uint32, topic whisper.TopicType) []*whisper.Envelope { +func (s *WMailServer) processRequest(peer *whisper.Peer, lower, upper uint32, bloom []byte) []*whisper.Envelope { ret := make([]*whisper.Envelope, 0) var err error var zero common.Hash - var empty whisper.TopicType kl := NewDbKey(lower, zero) ku := NewDbKey(upper, zero) i := s.db.NewIterator(&util.Range{Start: kl.raw, Limit: ku.raw}, nil) @@ -130,7 +129,7 @@ func (s *WMailServer) processRequest(peer *whisper.Peer, lower, upper uint32, to log.Error(fmt.Sprintf("RLP decoding failed: %s", err)) } - if topic == empty || envelope.Topic == topic { + if whisper.BloomFilterMatch(bloom, envelope.Bloom()) { if peer == nil { // used for test purposes ret = append(ret, &envelope) @@ -152,22 +151,16 @@ func (s *WMailServer) processRequest(peer *whisper.Peer, lower, upper uint32, to return ret } -func (s *WMailServer) validateRequest(peerID []byte, request *whisper.Envelope) (bool, uint32, uint32, whisper.TopicType) { - var topic whisper.TopicType +func (s *WMailServer) validateRequest(peerID []byte, request *whisper.Envelope) (bool, uint32, uint32, []byte) { if s.pow > 0.0 && request.PoW() < s.pow { - return false, 0, 0, topic + return false, 0, 0, nil } f := whisper.Filter{KeySym: s.key} decrypted := request.Open(&f) if decrypted == nil { log.Warn(fmt.Sprintf("Failed to decrypt p2p request")) - return false, 0, 0, topic - } - - if len(decrypted.Payload) < 8 { - log.Warn(fmt.Sprintf("Undersized p2p request")) - return false, 0, 0, topic + return false, 0, 0, nil } src := crypto.FromECDSAPub(decrypted.Src) @@ -179,15 +172,24 @@ func (s *WMailServer) validateRequest(peerID []byte, request *whisper.Envelope) // if !bytes.Equal(peerID, src) { if src == nil { log.Warn(fmt.Sprintf("Wrong signature of p2p request")) - return false, 0, 0, topic + return false, 0, 0, nil + } + + var bloom []byte + payloadSize := len(decrypted.Payload) + if payloadSize < 8 { + log.Warn(fmt.Sprintf("Undersized p2p request")) + return false, 0, 0, nil + } else if payloadSize == 8 { + bloom = whisper.MakeFullNodeBloom() + } else if payloadSize < 8+whisper.BloomFilterSize { + log.Warn(fmt.Sprintf("Undersized bloom filter in p2p request")) + return false, 0, 0, nil + } else { + bloom = decrypted.Payload[8 : 8+whisper.BloomFilterSize] } lower := binary.BigEndian.Uint32(decrypted.Payload[:4]) upper := binary.BigEndian.Uint32(decrypted.Payload[4:8]) - - if len(decrypted.Payload) >= 8+whisper.TopicLength { - topic = whisper.BytesToTopic(decrypted.Payload[8:]) - } - - return true, lower, upper, topic + return true, lower, upper, bloom } diff --git a/whisper/mailserver/server_test.go b/whisper/mailserver/server_test.go index 63864c1e6b..d5b993afb9 100644 --- a/whisper/mailserver/server_test.go +++ b/whisper/mailserver/server_test.go @@ -17,6 +17,7 @@ package mailserver import ( + "bytes" "crypto/ecdsa" "encoding/binary" "io/ioutil" @@ -61,7 +62,7 @@ func generateEnvelope(t *testing.T) *whisper.Envelope { h := crypto.Keccak256Hash([]byte("test sample data")) params := &whisper.MessageParams{ KeySym: h[:], - Topic: whisper.TopicType{}, + Topic: whisper.TopicType{0x1F, 0x7E, 0xA1, 0x7F}, Payload: []byte("test payload"), PoW: powRequirement, WorkTime: 2, @@ -121,6 +122,7 @@ func deliverTest(t *testing.T, server *WMailServer, env *whisper.Envelope) { upp: birth + 1, key: testPeerID, } + singleRequest(t, server, env, p, true) p.low, p.upp = birth+1, 0xffffffff @@ -131,14 +133,14 @@ func deliverTest(t *testing.T, server *WMailServer, env *whisper.Envelope) { p.low = birth - 1 p.upp = birth + 1 - p.topic[0]++ + p.topic[0] = 0xFF singleRequest(t, server, env, p, false) } func singleRequest(t *testing.T, server *WMailServer, env *whisper.Envelope, p *ServerTestParams, expect bool) { request := createRequest(t, p) src := crypto.FromECDSAPub(&p.key.PublicKey) - ok, lower, upper, topic := server.validateRequest(src, request) + ok, lower, upper, bloom := server.validateRequest(src, request) if !ok { t.Fatalf("request validation failed, seed: %d.", seed) } @@ -148,12 +150,13 @@ func singleRequest(t *testing.T, server *WMailServer, env *whisper.Envelope, p * if upper != p.upp { t.Fatalf("request validation failed (upper bound), seed: %d.", seed) } - if topic != p.topic { + expectedBloom := whisper.TopicToBloom(p.topic) + if !bytes.Equal(bloom, expectedBloom) { t.Fatalf("request validation failed (topic), seed: %d.", seed) } var exist bool - mail := server.processRequest(nil, p.low, p.upp, p.topic) + mail := server.processRequest(nil, p.low, p.upp, bloom) for _, msg := range mail { if msg.Hash() == env.Hash() { exist = true @@ -166,7 +169,7 @@ func singleRequest(t *testing.T, server *WMailServer, env *whisper.Envelope, p * } src[0]++ - ok, lower, upper, topic = server.validateRequest(src, request) + ok, lower, upper, bloom = server.validateRequest(src, request) if !ok { // request should be valid regardless of signature t.Fatalf("request validation false negative, seed: %d (lower: %d, upper: %d).", seed, lower, upper) @@ -174,10 +177,11 @@ func singleRequest(t *testing.T, server *WMailServer, env *whisper.Envelope, p * } func createRequest(t *testing.T, p *ServerTestParams) *whisper.Envelope { - data := make([]byte, 8+whisper.TopicLength) + bloom := whisper.TopicToBloom(p.topic) + data := make([]byte, 8) binary.BigEndian.PutUint32(data, p.low) binary.BigEndian.PutUint32(data[4:], p.upp) - copy(data[8:], p.topic[:]) + data = append(data, bloom...) key, err := shh.GetSymKey(keyID) if err != nil { diff --git a/whisper/whisperv6/doc.go b/whisper/whisperv6/doc.go index d5d7fed609..066a9766d4 100644 --- a/whisper/whisperv6/doc.go +++ b/whisper/whisperv6/doc.go @@ -60,7 +60,7 @@ const ( aesKeyLength = 32 // in bytes aesNonceLength = 12 // in bytes; for more info please see cipher.gcmStandardNonceSize & aesgcm.NonceSize() keyIDSize = 32 // in bytes - bloomFilterSize = 64 // in bytes + BloomFilterSize = 64 // in bytes flagsLength = 1 EnvelopeHeaderLength = 20 diff --git a/whisper/whisperv6/envelope.go b/whisper/whisperv6/envelope.go index c7bea2bb90..2c80d47bc6 100644 --- a/whisper/whisperv6/envelope.go +++ b/whisper/whisperv6/envelope.go @@ -249,7 +249,7 @@ func (e *Envelope) Bloom() []byte { // TopicToBloom converts the topic (4 bytes) to the bloom filter (64 bytes) func TopicToBloom(topic TopicType) []byte { - b := make([]byte, bloomFilterSize) + b := make([]byte, BloomFilterSize) var index [3]int for j := 0; j < 3; j++ { index[j] = int(topic[j]) diff --git a/whisper/whisperv6/peer.go b/whisper/whisperv6/peer.go index 6d75290fde..2bf1c905b2 100644 --- a/whisper/whisperv6/peer.go +++ b/whisper/whisperv6/peer.go @@ -56,7 +56,7 @@ func newPeer(host *Whisper, remote *p2p.Peer, rw p2p.MsgReadWriter) *Peer { powRequirement: 0.0, known: set.New(), quit: make(chan struct{}), - bloomFilter: makeFullNodeBloom(), + bloomFilter: MakeFullNodeBloom(), fullNode: true, } } @@ -120,7 +120,7 @@ func (peer *Peer) handshake() error { err = s.Decode(&bloom) if err == nil { sz := len(bloom) - if sz != bloomFilterSize && sz != 0 { + if sz != BloomFilterSize && sz != 0 { return fmt.Errorf("peer [%x] sent bad status message: wrong bloom filter size %d", peer.ID(), sz) } peer.setBloomFilter(bloom) @@ -229,7 +229,7 @@ func (peer *Peer) notifyAboutBloomFilterChange(bloom []byte) error { func (peer *Peer) bloomMatch(env *Envelope) bool { peer.bloomMu.Lock() defer peer.bloomMu.Unlock() - return peer.fullNode || bloomFilterMatch(peer.bloomFilter, env.Bloom()) + return peer.fullNode || BloomFilterMatch(peer.bloomFilter, env.Bloom()) } func (peer *Peer) setBloomFilter(bloom []byte) { @@ -238,13 +238,13 @@ func (peer *Peer) setBloomFilter(bloom []byte) { peer.bloomFilter = bloom peer.fullNode = isFullNode(bloom) if peer.fullNode && peer.bloomFilter == nil { - peer.bloomFilter = makeFullNodeBloom() + peer.bloomFilter = MakeFullNodeBloom() } } -func makeFullNodeBloom() []byte { - bloom := make([]byte, bloomFilterSize) - for i := 0; i < bloomFilterSize; i++ { +func MakeFullNodeBloom() []byte { + bloom := make([]byte, BloomFilterSize) + for i := 0; i < BloomFilterSize; i++ { bloom[i] = 0xFF } return bloom diff --git a/whisper/whisperv6/peer_test.go b/whisper/whisperv6/peer_test.go index 6d7754d790..ec985ae65b 100644 --- a/whisper/whisperv6/peer_test.go +++ b/whisper/whisperv6/peer_test.go @@ -152,7 +152,7 @@ func resetParams(t *testing.T) { } func initBloom(t *testing.T) { - masterBloomFilter = make([]byte, bloomFilterSize) + masterBloomFilter = make([]byte, BloomFilterSize) _, err := mrand.Read(masterBloomFilter) if err != nil { t.Fatalf("rand failed: %s.", err) @@ -164,7 +164,7 @@ func initBloom(t *testing.T) { masterBloomFilter[i] = 0xFF } - if !bloomFilterMatch(masterBloomFilter, msgBloom) { + if !BloomFilterMatch(masterBloomFilter, msgBloom) { t.Fatalf("bloom mismatch on initBloom.") } } @@ -178,7 +178,7 @@ func initialize(t *testing.T) { for i := 0; i < NumNodes; i++ { var node TestNode - b := make([]byte, bloomFilterSize) + b := make([]byte, BloomFilterSize) copy(b, masterBloomFilter) node.shh = New(&DefaultConfig) node.shh.SetMinimumPoW(masterPow) diff --git a/whisper/whisperv6/whisper.go b/whisper/whisperv6/whisper.go index 81f59b3cc7..880cced098 100644 --- a/whisper/whisperv6/whisper.go +++ b/whisper/whisperv6/whisper.go @@ -232,11 +232,11 @@ func (whisper *Whisper) SetMaxMessageSize(size uint32) error { // SetBloomFilter sets the new bloom filter func (whisper *Whisper) SetBloomFilter(bloom []byte) error { - if len(bloom) != bloomFilterSize { + if len(bloom) != BloomFilterSize { return fmt.Errorf("invalid bloom filter size: %d", len(bloom)) } - b := make([]byte, bloomFilterSize) + b := make([]byte, BloomFilterSize) copy(b, bloom) whisper.settings.Store(bloomFilterIdx, b) @@ -558,14 +558,14 @@ func (whisper *Whisper) Subscribe(f *Filter) (string, error) { // updateBloomFilter recalculates the new value of bloom filter, // and informs the peers if necessary. func (whisper *Whisper) updateBloomFilter(f *Filter) { - aggregate := make([]byte, bloomFilterSize) + aggregate := make([]byte, BloomFilterSize) for _, t := range f.Topics { top := BytesToTopic(t) b := TopicToBloom(top) aggregate = addBloom(aggregate, b) } - if !bloomFilterMatch(whisper.BloomFilter(), aggregate) { + if !BloomFilterMatch(whisper.BloomFilter(), aggregate) { // existing bloom filter must be updated aggregate = addBloom(whisper.BloomFilter(), aggregate) whisper.SetBloomFilter(aggregate) @@ -701,7 +701,7 @@ func (whisper *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error { case bloomFilterExCode: var bloom []byte err := packet.Decode(&bloom) - if err == nil && len(bloom) != bloomFilterSize { + if err == nil && len(bloom) != BloomFilterSize { err = fmt.Errorf("wrong bloom filter size %d", len(bloom)) } @@ -779,11 +779,11 @@ func (whisper *Whisper) add(envelope *Envelope, isP2P bool) (bool, error) { } } - if !bloomFilterMatch(whisper.BloomFilter(), envelope.Bloom()) { + if !BloomFilterMatch(whisper.BloomFilter(), envelope.Bloom()) { // maybe the value was recently changed, and the peers did not adjust yet. // in this case the previous value is retrieved by BloomFilterTolerance() // for a short period of peer synchronization. - if !bloomFilterMatch(whisper.BloomFilterTolerance(), envelope.Bloom()) { + if !BloomFilterMatch(whisper.BloomFilterTolerance(), envelope.Bloom()) { return false, fmt.Errorf("envelope does not match bloom filter, hash=[%v], bloom: \n%x \n%x \n%x", envelope.Hash().Hex(), whisper.BloomFilter(), envelope.Bloom(), envelope.Topic) } @@ -1025,12 +1025,12 @@ func isFullNode(bloom []byte) bool { return true } -func bloomFilterMatch(filter, sample []byte) bool { +func BloomFilterMatch(filter, sample []byte) bool { if filter == nil { return true } - for i := 0; i < bloomFilterSize; i++ { + for i := 0; i < BloomFilterSize; i++ { f := filter[i] s := sample[i] if (f | s) != f { @@ -1042,8 +1042,8 @@ func bloomFilterMatch(filter, sample []byte) bool { } func addBloom(a, b []byte) []byte { - c := make([]byte, bloomFilterSize) - for i := 0; i < bloomFilterSize; i++ { + c := make([]byte, BloomFilterSize) + for i := 0; i < BloomFilterSize; i++ { c[i] = a[i] | b[i] } return c diff --git a/whisper/whisperv6/whisper_test.go b/whisper/whisperv6/whisper_test.go index 3a219cd133..7fe256309e 100644 --- a/whisper/whisperv6/whisper_test.go +++ b/whisper/whisperv6/whisper_test.go @@ -826,11 +826,11 @@ func TestSymmetricSendKeyMismatch(t *testing.T) { func TestBloom(t *testing.T) { topic := TopicType{0, 0, 255, 6} b := TopicToBloom(topic) - x := make([]byte, bloomFilterSize) + x := make([]byte, BloomFilterSize) x[0] = byte(1) x[32] = byte(1) - x[bloomFilterSize-1] = byte(128) - if !bloomFilterMatch(x, b) || !bloomFilterMatch(b, x) { + x[BloomFilterSize-1] = byte(128) + if !BloomFilterMatch(x, b) || !BloomFilterMatch(b, x) { t.Fatalf("bloom filter does not match the mask") } @@ -842,11 +842,11 @@ func TestBloom(t *testing.T) { if err != nil { t.Fatalf("math rand error") } - if !bloomFilterMatch(b, b) { + if !BloomFilterMatch(b, b) { t.Fatalf("bloom filter does not match self") } x = addBloom(x, b) - if !bloomFilterMatch(x, b) { + if !BloomFilterMatch(x, b) { t.Fatalf("bloom filter does not match combined bloom") } if !isFullNode(nil) { @@ -856,16 +856,16 @@ func TestBloom(t *testing.T) { if isFullNode(x) { t.Fatalf("isFullNode false positive") } - for i := 0; i < bloomFilterSize; i++ { + for i := 0; i < BloomFilterSize; i++ { b[i] = byte(255) } if !isFullNode(b) { t.Fatalf("isFullNode false negative") } - if bloomFilterMatch(x, b) { + if BloomFilterMatch(x, b) { t.Fatalf("bloomFilterMatch false positive") } - if !bloomFilterMatch(b, x) { + if !BloomFilterMatch(b, x) { t.Fatalf("bloomFilterMatch false negative") } @@ -879,7 +879,7 @@ func TestBloom(t *testing.T) { t.Fatalf("failed to set bloom filter: %s", err) } f = w.BloomFilter() - if !bloomFilterMatch(f, x) || !bloomFilterMatch(x, f) { + if !BloomFilterMatch(f, x) || !BloomFilterMatch(x, f) { t.Fatalf("retireved wrong bloom filter") } } From 3ca3fffdf01b94244ef6c2d93ed38a30da9fcb0a Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 1 Mar 2018 18:55:31 +0100 Subject: [PATCH 14/21] metrics: fix flaky Example metrics test (#16222) * metrics: add sleep to test in order to get predictable output * metrics: relax constraints on timer test --- metrics/metrics_test.go | 3 ++- metrics/timer_test.go | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/metrics/metrics_test.go b/metrics/metrics_test.go index 726fba3475..df36da0ade 100644 --- a/metrics/metrics_test.go +++ b/metrics/metrics_test.go @@ -6,6 +6,7 @@ import ( "log" "sync" "testing" + "time" ) const FANOUT = 128 @@ -114,7 +115,7 @@ func Example() { // Threadsafe registration t := GetOrRegisterTimer("db.get.latency", nil) - t.Time(func() {}) + t.Time(func() { time.Sleep(10 * time.Millisecond) }) t.Update(1) fmt.Println(c.Count()) diff --git a/metrics/timer_test.go b/metrics/timer_test.go index f85c9b803d..c1f0ff9388 100644 --- a/metrics/timer_test.go +++ b/metrics/timer_test.go @@ -47,8 +47,8 @@ func TestTimerStop(t *testing.T) { func TestTimerFunc(t *testing.T) { tm := NewTimer() tm.Time(func() { time.Sleep(50e6) }) - if max := tm.Max(); 45e6 > max || max > 55e6 { - t.Errorf("tm.Max(): 45e6 > %v || %v > 55e6\n", max, max) + if max := tm.Max(); 35e6 > max || max > 95e6 { + t.Errorf("tm.Max(): 35e6 > %v || %v > 95e6\n", max, max) } } From d520bf45038ec8f02dd255bea038c5785f7290e0 Mon Sep 17 00:00:00 2001 From: Zhenguo Niu Date: Fri, 2 Mar 2018 17:59:26 +0800 Subject: [PATCH 15/21] cmd/swarm: fix some typos in manifest cmd (#16227) Replace "atleast" with "at least" in the manifest error message. --- cmd/swarm/manifest.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/swarm/manifest.go b/cmd/swarm/manifest.go index aa276e0f9b..41a69a5d05 100644 --- a/cmd/swarm/manifest.go +++ b/cmd/swarm/manifest.go @@ -35,7 +35,7 @@ const bzzManifestJSON = "application/bzz-manifest+json" func add(ctx *cli.Context) { args := ctx.Args() if len(args) < 3 { - utils.Fatalf("Need atleast three arguments []") + utils.Fatalf("Need at least three arguments []") } var ( @@ -69,7 +69,7 @@ func update(ctx *cli.Context) { args := ctx.Args() if len(args) < 3 { - utils.Fatalf("Need atleast three arguments ") + utils.Fatalf("Need at least three arguments ") } var ( @@ -101,7 +101,7 @@ func update(ctx *cli.Context) { func remove(ctx *cli.Context) { args := ctx.Args() if len(args) < 2 { - utils.Fatalf("Need atleast two arguments ") + utils.Fatalf("Need at least two arguments ") } var ( From 6f13e515f40ef84ff277d373488c99de49e33f18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 2 Mar 2018 11:57:11 +0200 Subject: [PATCH 16/21] cmd/faucet: update state in background, skip when busy --- cmd/faucet/faucet.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index 095668c868..5bad09bbd5 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -533,9 +533,11 @@ func (f *faucet) loop() { } defer sub.Unsubscribe() - for { - select { - case head := <-heads: + // Start a goroutine to update the state from head notifications in the background + update := make(chan *types.Header) + + go func() { + for head := range update { // New chain head arrived, query the current stats and stream to clients var ( balance *big.Int @@ -588,6 +590,17 @@ func (f *faucet) loop() { } } f.lock.RUnlock() + } + }() + // Wait for various events and assing to the appropriate background threads + for { + select { + case head := <-heads: + // New head arrived, send if for state update if there's none running + select { + case update <- head: + default: + } case <-f.update: // Pending requests updated, stream to clients From 6219a338225f1a4dfb7e51212ec3dde6e32785ce Mon Sep 17 00:00:00 2001 From: Vlad Date: Fri, 2 Mar 2018 14:54:54 +0100 Subject: [PATCH 17/21] whisper: filereader mode introduced to wnode --- cmd/wnode/main.go | 37 +++++++++++++++++++++++++++++++++++ whisper/whisperv6/envelope.go | 4 ++++ 2 files changed, 41 insertions(+) diff --git a/cmd/wnode/main.go b/cmd/wnode/main.go index ccfdd46265..76590e7f5f 100644 --- a/cmd/wnode/main.go +++ b/cmd/wnode/main.go @@ -86,6 +86,7 @@ var ( asymmetricMode = flag.Bool("asym", false, "use asymmetric encryption") generateKey = flag.Bool("generatekey", false, "generate and show the private key") fileExMode = flag.Bool("fileexchange", false, "file exchange mode") + fileReader = flag.Bool("filereader", false, "load and decrypt messages saved as files, display as plain text") testMode = flag.Bool("test", false, "use of predefined parameters for diagnostics (password, etc.)") echoMode = flag.Bool("echo", false, "echo mode: prints some arguments for diagnostics") @@ -433,6 +434,8 @@ func run() { requestExpiredMessagesLoop() } else if *fileExMode { sendFilesLoop() + } else if *fileReader { + fileReaderLoop() } else { sendLoop() } @@ -483,6 +486,40 @@ func sendFilesLoop() { } } +func fileReaderLoop() { + watcher1 := shh.GetFilter(symFilterID) + watcher2 := shh.GetFilter(asymFilterID) + if watcher1 == nil && watcher2 == nil { + fmt.Println("Error: neither symmetric nor asymmetric filter is installed") + close(done) + return + } + + for { + s := scanLine("") + if s == quitCommand { + fmt.Println("Quit command received") + close(done) + return + } + raw, err := ioutil.ReadFile(s) + if err != nil { + fmt.Printf(">>> Error: %s \n", err) + } else { + env := whisper.Envelope{Data: raw} // the topic is zero + msg := env.Open(watcher1) // force-open envelope regardless of the topic + if msg == nil { + msg = env.Open(watcher2) + } + if msg == nil { + fmt.Printf(">>> Error: failed to decrypt the message \n") + } else { + printMessageInfo(msg) + } + } + } +} + func scanLine(prompt string) string { if len(prompt) > 0 { fmt.Print(prompt) diff --git a/whisper/whisperv6/envelope.go b/whisper/whisperv6/envelope.go index 2c80d47bc6..2f947f1a40 100644 --- a/whisper/whisperv6/envelope.go +++ b/whisper/whisperv6/envelope.go @@ -208,6 +208,10 @@ func (e *Envelope) OpenSymmetric(key []byte) (msg *ReceivedMessage, err error) { // Open tries to decrypt an envelope, and populates the message fields in case of success. func (e *Envelope) Open(watcher *Filter) (msg *ReceivedMessage) { + if watcher == nil { + return nil + } + // The API interface forbids filters doing both symmetric and asymmetric encryption. if watcher.expectsAsymmetricEncryption() && watcher.expectsSymmetricEncryption() { return nil From 12f4d284114a719e9a0779933e8770c352ed1767 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Sat, 3 Mar 2018 00:52:21 +0100 Subject: [PATCH 18/21] internal/debug: add support for mutex profiles (#16230) --- internal/debug/api.go | 27 +++++++++++++++++++++++---- internal/web3ext/web3ext.go | 15 +++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/internal/debug/api.go b/internal/debug/api.go index 3547b05641..048b7d7635 100644 --- a/internal/debug/api.go +++ b/internal/debug/api.go @@ -140,10 +140,9 @@ func (h *HandlerT) GoTrace(file string, nsec uint) error { return nil } -// BlockProfile turns on CPU profiling for nsec seconds and writes -// profile data to file. It uses a profile rate of 1 for most accurate -// information. If a different rate is desired, set the rate -// and write the profile manually. +// BlockProfile turns on goroutine profiling for nsec seconds and writes profile data to +// file. It uses a profile rate of 1 for most accurate information. If a different rate is +// desired, set the rate and write the profile manually. func (*HandlerT) BlockProfile(file string, nsec uint) error { runtime.SetBlockProfileRate(1) time.Sleep(time.Duration(nsec) * time.Second) @@ -162,6 +161,26 @@ func (*HandlerT) WriteBlockProfile(file string) error { return writeProfile("block", file) } +// MutexProfile turns on mutex profiling for nsec seconds and writes profile data to file. +// It uses a profile rate of 1 for most accurate information. If a different rate is +// desired, set the rate and write the profile manually. +func (*HandlerT) MutexProfile(file string, nsec uint) error { + runtime.SetMutexProfileFraction(1) + time.Sleep(time.Duration(nsec) * time.Second) + defer runtime.SetMutexProfileFraction(0) + return writeProfile("mutex", file) +} + +// SetMutexProfileFraction sets the rate of mutex profiling. +func (*HandlerT) SetMutexProfileFraction(rate int) { + runtime.SetMutexProfileFraction(rate) +} + +// WriteMutexProfile writes a goroutine blocking profile to the given file. +func (*HandlerT) WriteMutexProfile(file string) error { + return writeProfile("mutex", file) +} + // WriteMemProfile writes an allocation profile to the given file. // Note that the profiling rate cannot be set through the API, // it must be set on the command line. diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index a6b81b4c2a..9d6ce8c6c8 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -307,6 +307,21 @@ web3._extend({ call: 'debug_writeBlockProfile', params: 1 }), + new web3._extend.Method({ + name: 'mutexProfile', + call: 'debug_mutexProfile', + params: 2 + }), + new web3._extend.Method({ + name: 'setMutexProfileRate', + call: 'debug_setMutexProfileRate', + params: 1 + }), + new web3._extend.Method({ + name: 'writeMutexProfile', + call: 'debug_writeMutexProfile', + params: 1 + }), new web3._extend.Method({ name: 'writeMemProfile', call: 'debug_writeMemProfile', From ca64a122d33008c155c35a9d0e78cfbcafb1820a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Sat, 3 Mar 2018 01:52:39 +0200 Subject: [PATCH 19/21] eth/downloader: save and load trie sync progress (#16224) --- core/database_util.go | 20 ++++++++++++++++++++ eth/downloader/downloader.go | 6 +++++- eth/downloader/statesync.go | 4 ++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/core/database_util.go b/core/database_util.go index 61ab701349..8c46989854 100644 --- a/core/database_util.go +++ b/core/database_util.go @@ -47,6 +47,7 @@ var ( headHeaderKey = []byte("LastHeader") headBlockKey = []byte("LastBlock") headFastKey = []byte("LastFast") + trieSyncKey = []byte("TrieSync") // Data item prefixes (use single byte to avoid mixing data types, avoid `i`). headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header @@ -146,6 +147,16 @@ func GetHeadFastBlockHash(db DatabaseReader) common.Hash { return common.BytesToHash(data) } +// GetTrieSyncProgress retrieves the number of tries nodes fast synced to allow +// reportinc correct numbers across restarts. +func GetTrieSyncProgress(db DatabaseReader) uint64 { + data, _ := db.Get(trieSyncKey) + if len(data) == 0 { + return 0 + } + return new(big.Int).SetBytes(data).Uint64() +} + // GetHeaderRLP retrieves a block header in its raw RLP database encoding, or nil // if the header's not found. func GetHeaderRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue { @@ -374,6 +385,15 @@ func WriteHeadFastBlockHash(db ethdb.Putter, hash common.Hash) error { return nil } +// WriteTrieSyncProgress stores the fast sync trie process counter to support +// retrieving it across restarts. +func WriteTrieSyncProgress(db ethdb.Putter, count uint64) error { + if err := db.Put(trieSyncKey, new(big.Int).SetUint64(count).Bytes()); err != nil { + log.Crit("Failed to store fast sync trie progress", "err", err) + } + return nil +} + // WriteHeader serializes a block header into the database. func WriteHeader(db ethdb.Putter, header *types.Header) error { data, err := rlp.EncodeToBytes(header) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index d13247766a..70febf4cb1 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -27,6 +27,7 @@ import ( ethereum "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -221,7 +222,10 @@ func New(mode SyncMode, stateDb ethdb.Database, mux *event.TypeMux, chain BlockC quitCh: make(chan struct{}), stateCh: make(chan dataPack), stateSyncStart: make(chan *stateSync), - trackStateReq: make(chan *stateReq), + syncStatsState: stateSyncStats{ + processed: core.GetTrieSyncProgress(stateDb), + }, + trackStateReq: make(chan *stateReq), } go dl.qosTuner() go dl.stateFetcher() diff --git a/eth/downloader/statesync.go b/eth/downloader/statesync.go index 9cc65a208c..ee6c7b4914 100644 --- a/eth/downloader/statesync.go +++ b/eth/downloader/statesync.go @@ -23,6 +23,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/ethdb" @@ -466,4 +467,7 @@ func (s *stateSync) updateStats(written, duplicate, unexpected int, duration tim if written > 0 || duplicate > 0 || unexpected > 0 { log.Info("Imported new state entries", "count", written, "elapsed", common.PrettyDuration(duration), "processed", s.d.syncStatsState.processed, "pending", s.d.syncStatsState.pending, "retry", len(s.tasks), "duplicate", s.d.syncStatsState.duplicate, "unexpected", s.d.syncStatsState.unexpected) } + if written > 0 { + core.WriteTrieSyncProgress(s.d.stateDB, s.d.syncStatsState.processed) + } } From 5ad7b9123ce4688a2e3370e7487ef979e97c4373 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Sat, 3 Mar 2018 00:52:54 +0100 Subject: [PATCH 20/21] light: new CHTs (#16233) --- .github/CODEOWNERS | 2 ++ light/postprocess.go | 16 ++++++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6076fe46a4..a7b6176552 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,5 +5,7 @@ accounts/usbwallet @karalabe consensus @karalabe core/ @karalabe @holiman eth/ @karalabe +les/ @zsfelfoldi +light/ @zsfelfoldi mobile/ @karalabe p2p/ @fjl @zsfelfoldi diff --git a/light/postprocess.go b/light/postprocess.go index 84149fdaae..384a635f76 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -58,18 +58,18 @@ type trustedCheckpoint struct { var ( mainnetCheckpoint = trustedCheckpoint{ name: "mainnet", - sectionIdx: 153, - sectionHead: common.HexToHash("04c2114a8cbe49ba5c37a03cc4b4b8d3adfc0bd2c78e0e726405dd84afca1d63"), - chtRoot: common.HexToHash("d7ec603e5d30b567a6e894ee7704e4603232f206d3e5a589794cec0c57bf318e"), - bloomTrieRoot: common.HexToHash("0b139b8fb692e21f663ff200da287192201c28ef5813c1ac6ba02a0a4799eef9"), + sectionIdx: 157, + sectionHead: common.HexToHash("1963c080887ca7f406c2bb114293eea83e54f783f94df24b447f7e3b6317c747"), + chtRoot: common.HexToHash("42abc436567dfb678a38fa6a9f881aa4c8a4cc8eaa2def08359292c3d0bd48ec"), + bloomTrieRoot: common.HexToHash("281c9f8fb3cb8b37ae45e9907ef8f3b19cd22c54e297c2d6c09c1db1593dce42"), } ropstenCheckpoint = trustedCheckpoint{ name: "ropsten", - sectionIdx: 79, - sectionHead: common.HexToHash("1b1ba890510e06411fdee9bb64ca7705c56a1a4ce3559ddb34b3680c526cb419"), - chtRoot: common.HexToHash("71d60207af74e5a22a3e1cfbfc89f9944f91b49aa980c86fba94d568369eaf44"), - bloomTrieRoot: common.HexToHash("70aca4b3b6d08dde8704c95cedb1420394453c1aec390947751e69ff8c436360"), + sectionIdx: 83, + sectionHead: common.HexToHash("3ca623586bc0da35f1fc8d9b6b55950f3b1f69be9c6501846a2df672adb61236"), + chtRoot: common.HexToHash("8f08ec7783969768c6ef06e5fe3398223cbf4ae2907b676da7b6fe6c7f55b059"), + bloomTrieRoot: common.HexToHash("02d86d3c6a87f8f8a92c2a59bbba2132ff6f9f61b0915a5dc28a9d8279219fd0"), } ) From fa375955ad52bc7936f33c8d8cec68fb9007baaa Mon Sep 17 00:00:00 2001 From: gluk256 Date: Sat, 3 Mar 2018 00:54:15 +0100 Subject: [PATCH 21/21] whisper/whisperv6: delete unused function (#16234) --- whisper/whisperv6/filter.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/whisper/whisperv6/filter.go b/whisper/whisperv6/filter.go index e4171f85c9..2f170ddebe 100644 --- a/whisper/whisperv6/filter.go +++ b/whisper/whisperv6/filter.go @@ -191,20 +191,6 @@ func (fs *Filters) NotifyWatchers(env *Envelope, p2pMessage bool) { } } -func (f *Filter) processEnvelope(env *Envelope) *ReceivedMessage { - if f.MatchEnvelope(env) { - msg := env.Open(f) - if msg != nil { - return msg - } - - log.Trace("processing envelope: failed to open", "hash", env.Hash().Hex()) - } else { - log.Trace("processing envelope: does not match", "hash", env.Hash().Hex()) - } - return nil -} - func (f *Filter) expectsAsymmetricEncryption() bool { return f.KeyAsym != nil }