From 03116ba7b092d9658b42c87764ade010d425d2ef Mon Sep 17 00:00:00 2001 From: b00ris Date: Wed, 21 Feb 2018 11:55:17 +0300 Subject: [PATCH 1/8] whisper: bottleneck on matching messages --- whisper/whisperv5/filter.go | 113 ++++++++------ whisper/whisperv5/filter_test.go | 243 ++++++++++++++++++------------- 2 files changed, 208 insertions(+), 148 deletions(-) diff --git a/whisper/whisperv5/filter.go b/whisper/whisperv5/filter.go index 3190334ebb..920bcf20b8 100644 --- a/whisper/whisperv5/filter.go +++ b/whisper/whisperv5/filter.go @@ -26,6 +26,8 @@ import ( "github.com/ethereum/go-ethereum/log" ) +const ALL_TOPICS = "" + type Filter struct { Src *ecdsa.PublicKey // Sender of the message KeyAsym *ecdsa.PrivateKey // Private Key of recipient @@ -40,16 +42,20 @@ type Filter struct { } type Filters struct { - watchers map[string]*Filter - whisper *Whisper - mutex sync.RWMutex + watchers map[string]*Filter + watchersTopics map[string]map[string]struct{} + whisper *Whisper + mutex sync.RWMutex } func NewFilters(w *Whisper) *Filters { - return &Filters{ - watchers: make(map[string]*Filter), - whisper: w, + fs := &Filters{ + watchers: make(map[string]*Filter), + watchersTopics: make(map[string]map[string]struct{}), + whisper: w, } + fs.watchersTopics[ALL_TOPICS] = make(map[string]struct{}) + return fs } func (fs *Filters) Install(watcher *Filter) (string, error) { @@ -74,14 +80,56 @@ func (fs *Filters) Install(watcher *Filter) (string, error) { } fs.watchers[id] = watcher + fs.addFilterToTopicsMapping(watcher, id) return id, err } +func (fs *Filters) addFilterToTopicsMapping(watcher *Filter, id string) { + for i := range fs.prepareTopicsMapping(watcher) { + topicMapping, ok := fs.watchersTopics[i] + if !ok { + fs.watchersTopics[i] = make(map[string]struct{}) + topicMapping = fs.watchersTopics[i] + } + topicMapping[id] = struct{}{} + } +} + +func (fs *Filters) removeTopicFromTopicMapping(id string) { + for i := range fs.watchersTopics { + delete(fs.watchersTopics[i], id) + } +} + +func (fs *Filters) prepareTopicsMapping(watcher *Filter) map[string]struct{} { + topics := make(map[string]struct{}, len(watcher.Topics)) + + if len(watcher.Topics) == 0 { + topics[ALL_TOPICS] = struct{}{} + return topics + } + + for _, topic := range watcher.Topics { + topics[common.ToHex(topic)] = struct{}{} + } + + return topics +} + +func (fs *Filters) matchedTopics(topic TopicType) map[string]struct{} { + m := fs.watchersTopics[ALL_TOPICS] + for i := range fs.watchersTopics[topic.String()] { + m[i] = struct{}{} + } + return m +} + func (fs *Filters) Uninstall(id string) bool { fs.mutex.Lock() defer fs.mutex.Unlock() if fs.watchers[id] != nil { delete(fs.watchers, id) + fs.removeTopicFromTopicMapping(id) return true } return false @@ -99,11 +147,15 @@ func (fs *Filters) NotifyWatchers(env *Envelope, p2pMessage bool) { fs.mutex.RLock() defer fs.mutex.RUnlock() - i := -1 // only used for logging info - for _, watcher := range fs.watchers { - i++ + for watcherID := range fs.matchedTopics(env.Topic) { + watcher, ok := fs.watchers[watcherID] + if !ok { + log.Trace(fmt.Sprintf("msg [%x], filter [%s]: filter not exists", env.Hash(), watcherID)) + continue + } + if p2pMessage && !watcher.AllowP2P { - log.Trace(fmt.Sprintf("msg [%x], filter [%d]: p2p messages are not allowed", env.Hash(), i)) + log.Trace(fmt.Sprintf("msg [%x], filter [%s]: p2p messages are not allowed", env.Hash(), watcherID)) continue } @@ -115,10 +167,10 @@ func (fs *Filters) NotifyWatchers(env *Envelope, p2pMessage bool) { if match { msg = env.Open(watcher) if msg == nil { - log.Trace("processing message: failed to open", "message", env.Hash().Hex(), "filter", i) + log.Trace("processing message: failed to open", "message", env.Hash().Hex(), "filter", watcherID) } } else { - log.Trace("processing message: does not match", "message", env.Hash().Hex(), "filter", i) + log.Trace("processing message: does not match", "message", env.Hash().Hex(), "filter", watcherID) } } @@ -181,9 +233,9 @@ func (f *Filter) MatchMessage(msg *ReceivedMessage) bool { } if f.expectsAsymmetricEncryption() && msg.isAsymmetricEncryption() { - return IsPubKeyEqual(&f.KeyAsym.PublicKey, msg.Dst) && f.MatchTopic(msg.Topic) + return IsPubKeyEqual(&f.KeyAsym.PublicKey, msg.Dst) } else if f.expectsSymmetricEncryption() && msg.isSymmetricEncryption() { - return f.SymKeyHash == msg.SymKeyHash && f.MatchTopic(msg.Topic) + return f.SymKeyHash == msg.SymKeyHash } return false } @@ -194,44 +246,13 @@ func (f *Filter) MatchEnvelope(envelope *Envelope) bool { } if f.expectsAsymmetricEncryption() && envelope.isAsymmetric() { - return f.MatchTopic(envelope.Topic) + return true } else if f.expectsSymmetricEncryption() && envelope.IsSymmetric() { - return f.MatchTopic(envelope.Topic) - } - return false -} - -func (f *Filter) MatchTopic(topic TopicType) bool { - if len(f.Topics) == 0 { - // any topic matches return true } - - for _, bt := range f.Topics { - if matchSingleTopic(topic, bt) { - return true - } - } return false } -func matchSingleTopic(topic TopicType, bt []byte) bool { - if len(bt) > TopicLength { - bt = bt[:TopicLength] - } - - if len(bt) < TopicLength { - return false - } - - for j, b := range bt { - if topic[j] != b { - return false - } - } - return true -} - func IsPubKeyEqual(a, b *ecdsa.PublicKey) bool { if !ValidatePublicKey(a) { return false diff --git a/whisper/whisperv5/filter_test.go b/whisper/whisperv5/filter_test.go index 01034a3513..f50c970f5f 100644 --- a/whisper/whisperv5/filter_test.go +++ b/whisper/whisperv5/filter_test.go @@ -83,6 +83,15 @@ func generateFilter(t *testing.T, symmetric bool) (*Filter, error) { return &f, nil } +func generateFilters() *Filters { + fs := Filters{ + watchers: make(map[string]*Filter), + watchersTopics: make(map[string]map[string]struct{}), + } + fs.watchersTopics[ALL_TOPICS] = make(map[string]struct{}) + return &fs +} + func generateTestCases(t *testing.T, SizeTestFilters int) []FilterTestCase { cases := make([]FilterTestCase, SizeTestFilters) for i := 0; i < SizeTestFilters; i++ { @@ -284,19 +293,7 @@ func TestMatchEnvelope(t *testing.T) { if err != nil { t.Fatalf("failed Wrap with seed %d: %s.", seed, err) } - match := fsym.MatchEnvelope(env) - if match { - t.Fatalf("failed MatchEnvelope symmetric with seed %d.", seed) - } - match = fasym.MatchEnvelope(env) - if match { - t.Fatalf("failed MatchEnvelope asymmetric with seed %d.", seed) - } - // encrypt symmetrically - i := mrand.Int() % 4 - fsym.Topics[i] = params.Topic[:] - fasym.Topics[i] = params.Topic[:] msg, err = NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) @@ -306,41 +303,20 @@ func TestMatchEnvelope(t *testing.T) { t.Fatalf("failed Wrap() with seed %d: %s.", seed, err) } - // symmetric + matching topic: match - match = fsym.MatchEnvelope(env) - if !match { - t.Fatalf("failed MatchEnvelope() symmetric with seed %d.", seed) - } - - // asymmetric + matching topic: mismatch - match = fasym.MatchEnvelope(env) - if match { - t.Fatalf("failed MatchEnvelope() asymmetric with seed %d.", seed) - } - - // symmetric + matching topic + insufficient PoW: mismatch + // symmetric + insufficient PoW: mismatch fsym.PoW = env.PoW() + 1.0 - match = fsym.MatchEnvelope(env) + match := fsym.MatchEnvelope(env) if match { t.Fatalf("failed MatchEnvelope(symmetric + matching topic + insufficient PoW) asymmetric with seed %d.", seed) } - // symmetric + matching topic + sufficient PoW: match + // symmetric + sufficient PoW: match fsym.PoW = env.PoW() / 2 match = fsym.MatchEnvelope(env) if !match { t.Fatalf("failed MatchEnvelope(symmetric + matching topic + sufficient PoW) with seed %d.", seed) } - // symmetric + topics are nil (wildcard): match - prevTopics := fsym.Topics - fsym.Topics = nil - match = fsym.MatchEnvelope(env) - if !match { - t.Fatalf("failed MatchEnvelope(symmetric + topics are nil) with seed %d.", seed) - } - fsym.Topics = prevTopics - // encrypt asymmetrically key, err := crypto.GenerateKey() if err != nil { @@ -363,26 +339,6 @@ func TestMatchEnvelope(t *testing.T) { t.Fatalf("failed MatchEnvelope(encryption method mismatch) with seed %d.", seed) } - // asymmetric + mismatching topic: mismatch - match = fasym.MatchEnvelope(env) - if !match { - t.Fatalf("failed MatchEnvelope(asymmetric + mismatching topic) with seed %d.", seed) - } - - // asymmetric + matching topic: match - fasym.Topics[i] = fasym.Topics[i+1] - match = fasym.MatchEnvelope(env) - if match { - t.Fatalf("failed MatchEnvelope(asymmetric + matching topic) with seed %d.", seed) - } - - // asymmetric + filter without topic (wildcard): match - fasym.Topics = nil - match = fasym.MatchEnvelope(env) - if !match { - t.Fatalf("failed MatchEnvelope(asymmetric + filter without topic) with seed %d.", seed) - } - // asymmetric + insufficient PoW: mismatch fasym.PoW = env.PoW() + 1.0 match = fasym.MatchEnvelope(env) @@ -397,19 +353,6 @@ func TestMatchEnvelope(t *testing.T) { t.Fatalf("failed MatchEnvelope(asymmetric + sufficient PoW) with seed %d.", seed) } - // filter without topic + envelope without topic: match - env.Topic = TopicType{} - match = fasym.MatchEnvelope(env) - if !match { - t.Fatalf("failed MatchEnvelope(filter without topic + envelope without topic) with seed %d.", seed) - } - - // filter with topic + envelope without topic: mismatch - fasym.Topics = fsym.Topics - match = fasym.MatchEnvelope(env) - if match { - t.Fatalf("failed MatchEnvelope(filter without topic + envelope without topic) with seed %d.", seed) - } } func TestMatchMessageSym(t *testing.T) { @@ -461,13 +404,6 @@ func TestMatchMessageSym(t *testing.T) { t.Fatalf("failed MatchEnvelope(sufficient PoW) with seed %d.", seed) } - // topic mismatch - f.Topics[index][0]++ - if f.MatchMessage(msg) { - t.Fatalf("failed MatchEnvelope(topic mismatch) with seed %d.", seed) - } - f.Topics[index][0]-- - // key mismatch f.SymKeyHash[0]++ if f.MatchMessage(msg) { @@ -554,9 +490,29 @@ func TestMatchMessageAsym(t *testing.T) { t.Fatalf("failed MatchEnvelope(sufficient PoW) with seed %d.", seed) } + fs := generateFilters() + filterID, err := fs.Install(f) + if err != nil { + t.Fatalf("failed filter install with seed %d: %s.", seed, err) + } + + m := fs.matchedTopics(env.Topic) + _, matchedTopic := m[filterID] + + if !matchedTopic { + t.Fatalf("failed MatchEnvelope(topic mismatch) with seed %d.", seed) + } + // topic mismatch + if !fs.Uninstall(filterID) { + t.Fatal("failed to uninstall filter") + } f.Topics[index][0]++ - if f.MatchMessage(msg) { + filterID, err = fs.Install(f) + m = fs.matchedTopics(env.Topic) + _, matchedTopic = m[filterID] + + if matchedTopic { t.Fatalf("failed MatchEnvelope(topic mismatch) with seed %d.", seed) } f.Topics[index][0]-- @@ -795,54 +751,137 @@ func TestVariableTopics(t *testing.T) { if err != nil { t.Fatalf("failed generateFilter with seed %d: %s.", seed, err) } + fs := generateFilters() + + filterID, err := fs.Install(f) + if err != nil { + t.Fatal(err) + } for i := 0; i < 4; i++ { env.Topic = BytesToTopic(f.Topics[i]) + + //test match + m := fs.matchedTopics(env.Topic) + _, ok := m[filterID] match = f.MatchEnvelope(env) - if !match { + if !(match && ok) { t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed, i) } - f.Topics[i][lastTopicByte]++ + //change envelop topic + env.Topic[lastTopicByte]++ + + //false positive test match = f.MatchEnvelope(env) - if match { + m = fs.matchedTopics(env.Topic) + _, ok = m[filterID] + if !(match && ok) { t.Fatalf("MatchEnvelope symmetric with seed %d, step %d: false positive.", seed, i) } } } -func TestMatchSingleTopic_ReturnTrue(t *testing.T) { - bt := []byte("test") - topic := BytesToTopic(bt) +func TestTopicsMapping(t *testing.T) { + InitSingleTest() - if !matchSingleTopic(topic, bt) { - t.FailNow() + const lastTopicByte = 3 + params, err := generateMessageParams() + if err != nil { + t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) + } + msg, err := NewSentMessage(params) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } + env, err := msg.Wrap(params) + if err != nil { + t.Fatalf("failed Wrap with seed %d: %s.", seed, err) + } + + f, err := generateFilter(t, true) + if err != nil { + t.Fatalf("failed generateFilter with seed %d: %s.", seed, err) + } + fs := generateFilters() + + for i := 0; i < len(f.Topics); i++ { + env.Topic = BytesToTopic(f.Topics[i]) + + //test match + filterID, err := fs.Install(f) + if err != nil { + t.Fatal(err) + } + m := fs.matchedTopics(env.Topic) + if _, matchTopic := m[filterID]; !matchTopic { + t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed, i) + } + + //test match without filter + if !fs.Uninstall(filterID) { + t.Fatal("Failed to uninstall filter") + } + m = fs.matchedTopics(env.Topic) + if _, matchTopic := m[filterID]; matchTopic { + t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed, i) + } + + //test match with changed topic + f.Topics[i][lastTopicByte]++ + filterID, err = fs.Install(f) + if err != nil { + t.Fatal(err) + } + m = fs.matchedTopics(env.Topic) + if _, matchTopic := m[filterID]; matchTopic { + t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed, i) + } + if !fs.Uninstall(filterID) { + t.Fatal("Failed to uninstall filter") + } } } -func TestMatchSingleTopic_WithTail_ReturnTrue(t *testing.T) { - bt := []byte("test with tail") - topic := BytesToTopic([]byte("test")) +func TestTopicsMapping_MatchAllTopics_Success(t *testing.T) { + InitSingleTest() - if !matchSingleTopic(topic, bt) { - t.FailNow() + f, err := generateFilter(t, true) + if err != nil { + t.Fatalf("failed generateFilter with seed %d: %s.", seed, err) } -} + fs := generateFilters() -func TestMatchSingleTopic_NotEquals_ReturnFalse(t *testing.T) { - bt := []byte("tes") - topic := BytesToTopic(bt) + f.Topics = [][]byte{} - if matchSingleTopic(topic, bt) { - t.FailNow() + filterID, err := fs.Install(f) + if err != nil { + t.Fatal(err) } -} -func TestMatchSingleTopic_InsufficientLength_ReturnFalse(t *testing.T) { - bt := []byte("test") - topic := BytesToTopic([]byte("not_equal")) + //generate topic + topic := TopicType{} + mrand.Read(topic[:]) - if matchSingleTopic(topic, bt) { - t.FailNow() + m := fs.matchedTopics(topic) + if _, matchTopic := m[filterID]; !matchTopic { + t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed) } + if _, ok := fs.watchersTopics[ALL_TOPICS][filterID]; !ok { + t.Fatal("watcher mapping incorrect") + } + + ////test match without filter + if !fs.Uninstall(filterID) { + t.Fatal("Failed to uninstall filter") + } + m = fs.matchedTopics(topic) + if _, matchTopic := m[filterID]; matchTopic { + t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed) + } + + if _, ok := fs.watchersTopics[ALL_TOPICS][filterID]; ok { + t.Fatal("watcher mapping incorrect") + } + } From 376e7953dfbc13bfc68589e490fc3182f6c8f2d4 Mon Sep 17 00:00:00 2001 From: b00ris Date: Wed, 21 Feb 2018 16:55:53 +0300 Subject: [PATCH 2/8] whisper: add mutex to mapper --- whisper/whisperv5/filter.go | 116 +++++++++++++++++++------------ whisper/whisperv5/filter_test.go | 32 ++++----- 2 files changed, 87 insertions(+), 61 deletions(-) diff --git a/whisper/whisperv5/filter.go b/whisper/whisperv5/filter.go index 920bcf20b8..7e6c87a47c 100644 --- a/whisper/whisperv5/filter.go +++ b/whisper/whisperv5/filter.go @@ -44,6 +44,7 @@ type Filter struct { type Filters struct { watchers map[string]*Filter watchersTopics map[string]map[string]struct{} + topicMatcher *topicMatcher whisper *Whisper mutex sync.RWMutex } @@ -52,9 +53,9 @@ func NewFilters(w *Whisper) *Filters { fs := &Filters{ watchers: make(map[string]*Filter), watchersTopics: make(map[string]map[string]struct{}), + topicMatcher: newTopicMatcher(), whisper: w, } - fs.watchersTopics[ALL_TOPICS] = make(map[string]struct{}) return fs } @@ -80,56 +81,16 @@ func (fs *Filters) Install(watcher *Filter) (string, error) { } fs.watchers[id] = watcher - fs.addFilterToTopicsMapping(watcher, id) + fs.topicMatcher.addFilterToTopicsMapping(watcher, id) return id, err } -func (fs *Filters) addFilterToTopicsMapping(watcher *Filter, id string) { - for i := range fs.prepareTopicsMapping(watcher) { - topicMapping, ok := fs.watchersTopics[i] - if !ok { - fs.watchersTopics[i] = make(map[string]struct{}) - topicMapping = fs.watchersTopics[i] - } - topicMapping[id] = struct{}{} - } -} - -func (fs *Filters) removeTopicFromTopicMapping(id string) { - for i := range fs.watchersTopics { - delete(fs.watchersTopics[i], id) - } -} - -func (fs *Filters) prepareTopicsMapping(watcher *Filter) map[string]struct{} { - topics := make(map[string]struct{}, len(watcher.Topics)) - - if len(watcher.Topics) == 0 { - topics[ALL_TOPICS] = struct{}{} - return topics - } - - for _, topic := range watcher.Topics { - topics[common.ToHex(topic)] = struct{}{} - } - - return topics -} - -func (fs *Filters) matchedTopics(topic TopicType) map[string]struct{} { - m := fs.watchersTopics[ALL_TOPICS] - for i := range fs.watchersTopics[topic.String()] { - m[i] = struct{}{} - } - return m -} - func (fs *Filters) Uninstall(id string) bool { fs.mutex.Lock() defer fs.mutex.Unlock() if fs.watchers[id] != nil { delete(fs.watchers, id) - fs.removeTopicFromTopicMapping(id) + fs.topicMatcher.removeTopicFromTopicMapping(id) return true } return false @@ -147,7 +108,7 @@ func (fs *Filters) NotifyWatchers(env *Envelope, p2pMessage bool) { fs.mutex.RLock() defer fs.mutex.RUnlock() - for watcherID := range fs.matchedTopics(env.Topic) { + for watcherID := range fs.topicMatcher.matchedTopics(env.Topic) { watcher, ok := fs.watchers[watcherID] if !ok { log.Trace(fmt.Sprintf("msg [%x], filter [%s]: filter not exists", env.Hash(), watcherID)) @@ -262,3 +223,70 @@ func IsPubKeyEqual(a, b *ecdsa.PublicKey) bool { // the curve is always the same, just compare the points return a.X.Cmp(b.X) == 0 && a.Y.Cmp(b.Y) == 0 } + +func newTopicMatcher() *topicMatcher { + tm := new(topicMatcher) + tm.mapper = make(map[string]map[string]struct{}) + tm.mapper[ALL_TOPICS] = make(map[string]struct{}) + return tm +} + +type topicMatcher struct { + mapper map[string]map[string]struct{} + mx sync.RWMutex +} + +func (fs *topicMatcher) addFilterToTopicsMapping(watcher *Filter, id string) { + fs.mx.Lock() + defer fs.mx.Unlock() + + for i := range fs.prepareTopicsMapping(watcher) { + topicMapping, ok := fs.mapper[i] + if !ok { + fs.mapper[i] = make(map[string]struct{}) + topicMapping = fs.mapper[i] + } + topicMapping[id] = struct{}{} + } +} + +func (fs *topicMatcher) removeTopicFromTopicMapping(id string) { + fs.mx.Lock() + defer fs.mx.Unlock() + for i := range fs.mapper { + delete(fs.mapper[i], id) + } +} + +func (fs *topicMatcher) prepareTopicsMapping(watcher *Filter) map[string]struct{} { + fs.mx.RLock() + defer fs.mx.RUnlock() + topics := make(map[string]struct{}, len(watcher.Topics)) + + if len(watcher.Topics) == 0 { + topics[ALL_TOPICS] = struct{}{} + return topics + } + + for _, topic := range watcher.Topics { + topics[common.ToHex(topic)] = struct{}{} + } + + return topics +} + +func (fs *topicMatcher) matchedTopics(topic TopicType) map[string]struct{} { + fs.mx.RLock() + defer fs.mx.RUnlock() + + m := make(map[string]struct{}, len(fs.mapper[ALL_TOPICS])+len(fs.mapper[topic.String()])) + + for i := range fs.mapper[ALL_TOPICS] { + m[i] = struct{}{} + } + + for i := range fs.mapper[topic.String()] { + m[i] = struct{}{} + } + return m +} diff --git a/whisper/whisperv5/filter_test.go b/whisper/whisperv5/filter_test.go index f50c970f5f..e0db720dbb 100644 --- a/whisper/whisperv5/filter_test.go +++ b/whisper/whisperv5/filter_test.go @@ -85,10 +85,9 @@ func generateFilter(t *testing.T, symmetric bool) (*Filter, error) { func generateFilters() *Filters { fs := Filters{ - watchers: make(map[string]*Filter), - watchersTopics: make(map[string]map[string]struct{}), + watchers: make(map[string]*Filter), + topicMatcher: newTopicMatcher(), } - fs.watchersTopics[ALL_TOPICS] = make(map[string]struct{}) return &fs } @@ -496,7 +495,7 @@ func TestMatchMessageAsym(t *testing.T) { t.Fatalf("failed filter install with seed %d: %s.", seed, err) } - m := fs.matchedTopics(env.Topic) + m := fs.topicMatcher.matchedTopics(env.Topic) _, matchedTopic := m[filterID] if !matchedTopic { @@ -509,7 +508,7 @@ func TestMatchMessageAsym(t *testing.T) { } f.Topics[index][0]++ filterID, err = fs.Install(f) - m = fs.matchedTopics(env.Topic) + m = fs.topicMatcher.matchedTopics(env.Topic) _, matchedTopic = m[filterID] if matchedTopic { @@ -762,7 +761,7 @@ func TestVariableTopics(t *testing.T) { env.Topic = BytesToTopic(f.Topics[i]) //test match - m := fs.matchedTopics(env.Topic) + m := fs.topicMatcher.matchedTopics(env.Topic) _, ok := m[filterID] match = f.MatchEnvelope(env) if !(match && ok) { @@ -774,9 +773,9 @@ func TestVariableTopics(t *testing.T) { //false positive test match = f.MatchEnvelope(env) - m = fs.matchedTopics(env.Topic) + m = fs.topicMatcher.matchedTopics(env.Topic) _, ok = m[filterID] - if !(match && ok) { + if match && ok { t.Fatalf("MatchEnvelope symmetric with seed %d, step %d: false positive.", seed, i) } } @@ -813,7 +812,7 @@ func TestTopicsMapping(t *testing.T) { if err != nil { t.Fatal(err) } - m := fs.matchedTopics(env.Topic) + m := fs.topicMatcher.matchedTopics(env.Topic) if _, matchTopic := m[filterID]; !matchTopic { t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed, i) } @@ -822,7 +821,7 @@ func TestTopicsMapping(t *testing.T) { if !fs.Uninstall(filterID) { t.Fatal("Failed to uninstall filter") } - m = fs.matchedTopics(env.Topic) + m = fs.topicMatcher.matchedTopics(env.Topic) if _, matchTopic := m[filterID]; matchTopic { t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed, i) } @@ -833,7 +832,7 @@ func TestTopicsMapping(t *testing.T) { if err != nil { t.Fatal(err) } - m = fs.matchedTopics(env.Topic) + m = fs.topicMatcher.matchedTopics(env.Topic) if _, matchTopic := m[filterID]; matchTopic { t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed, i) } @@ -863,25 +862,24 @@ func TestTopicsMapping_MatchAllTopics_Success(t *testing.T) { topic := TopicType{} mrand.Read(topic[:]) - m := fs.matchedTopics(topic) + m := fs.topicMatcher.matchedTopics(topic) if _, matchTopic := m[filterID]; !matchTopic { t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed) } - if _, ok := fs.watchersTopics[ALL_TOPICS][filterID]; !ok { + if _, ok := fs.topicMatcher.mapper[ALL_TOPICS][filterID]; !ok { t.Fatal("watcher mapping incorrect") } - ////test match without filter + //test match without filter if !fs.Uninstall(filterID) { t.Fatal("Failed to uninstall filter") } - m = fs.matchedTopics(topic) + m = fs.topicMatcher.matchedTopics(topic) if _, matchTopic := m[filterID]; matchTopic { t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed) } - if _, ok := fs.watchersTopics[ALL_TOPICS][filterID]; ok { + if _, ok := fs.topicMatcher.mapper[ALL_TOPICS][filterID]; ok { t.Fatal("watcher mapping incorrect") } - } From 41353474f7256e7c27c1e235ade1efb7f8c08ce9 Mon Sep 17 00:00:00 2001 From: b00ris Date: Wed, 21 Feb 2018 18:51:55 +0300 Subject: [PATCH 3/8] whisper: add pool to matched --- whisper/whisperv5/filter.go | 36 ++++++++++------ whisper/whisperv5/filter_test.go | 74 +++++++++++++------------------- 2 files changed, 55 insertions(+), 55 deletions(-) diff --git a/whisper/whisperv5/filter.go b/whisper/whisperv5/filter.go index 7e6c87a47c..b38ef3bce1 100644 --- a/whisper/whisperv5/filter.go +++ b/whisper/whisperv5/filter.go @@ -104,11 +104,14 @@ func (fs *Filters) Get(id string) *Filter { func (fs *Filters) NotifyWatchers(env *Envelope, p2pMessage bool) { var msg *ReceivedMessage + matchedTopics := fs.topicMatcher.take() + defer fs.topicMatcher.resolve(matchedTopics) fs.mutex.RLock() defer fs.mutex.RUnlock() - for watcherID := range fs.topicMatcher.matchedTopics(env.Topic) { + fs.topicMatcher.matchedTopics(env.Topic, &matchedTopics) + for _, watcherID := range matchedTopics { watcher, ok := fs.watchers[watcherID] if !ok { log.Trace(fmt.Sprintf("msg [%x], filter [%s]: filter not exists", env.Hash(), watcherID)) @@ -224,16 +227,30 @@ func IsPubKeyEqual(a, b *ecdsa.PublicKey) bool { return a.X.Cmp(b.X) == 0 && a.Y.Cmp(b.Y) == 0 } +type topicMatcher struct { + mapper map[string]map[string]struct{} + mx sync.RWMutex + pool sync.Pool +} + func newTopicMatcher() *topicMatcher { tm := new(topicMatcher) tm.mapper = make(map[string]map[string]struct{}) tm.mapper[ALL_TOPICS] = make(map[string]struct{}) + tm.pool.New = func() interface{} { + return []string{} + } return tm } -type topicMatcher struct { - mapper map[string]map[string]struct{} - mx sync.RWMutex +func (fs *topicMatcher) take() []string { + return fs.pool.Get().([]string) +} +func (fs *topicMatcher) resolve(s []string) { + if cap(s) > 1000 { + return + } + fs.pool.Put(s[:0]) } func (fs *topicMatcher) addFilterToTopicsMapping(watcher *Filter, id string) { @@ -259,8 +276,6 @@ func (fs *topicMatcher) removeTopicFromTopicMapping(id string) { } func (fs *topicMatcher) prepareTopicsMapping(watcher *Filter) map[string]struct{} { - fs.mx.RLock() - defer fs.mx.RUnlock() topics := make(map[string]struct{}, len(watcher.Topics)) if len(watcher.Topics) == 0 { @@ -275,18 +290,15 @@ func (fs *topicMatcher) prepareTopicsMapping(watcher *Filter) map[string]struct{ return topics } -func (fs *topicMatcher) matchedTopics(topic TopicType) map[string]struct{} { +func (fs *topicMatcher) matchedTopics(topic TopicType, matched *[]string) { fs.mx.RLock() defer fs.mx.RUnlock() - m := make(map[string]struct{}, len(fs.mapper[ALL_TOPICS])+len(fs.mapper[topic.String()])) - for i := range fs.mapper[ALL_TOPICS] { - m[i] = struct{}{} + *matched = append(*matched, i) } for i := range fs.mapper[topic.String()] { - m[i] = struct{}{} + *matched = append(*matched, i) } - return m } diff --git a/whisper/whisperv5/filter_test.go b/whisper/whisperv5/filter_test.go index e0db720dbb..3e2a4f54df 100644 --- a/whisper/whisperv5/filter_test.go +++ b/whisper/whisperv5/filter_test.go @@ -489,33 +489,6 @@ func TestMatchMessageAsym(t *testing.T) { t.Fatalf("failed MatchEnvelope(sufficient PoW) with seed %d.", seed) } - fs := generateFilters() - filterID, err := fs.Install(f) - if err != nil { - t.Fatalf("failed filter install with seed %d: %s.", seed, err) - } - - m := fs.topicMatcher.matchedTopics(env.Topic) - _, matchedTopic := m[filterID] - - if !matchedTopic { - t.Fatalf("failed MatchEnvelope(topic mismatch) with seed %d.", seed) - } - - // topic mismatch - if !fs.Uninstall(filterID) { - t.Fatal("failed to uninstall filter") - } - f.Topics[index][0]++ - filterID, err = fs.Install(f) - m = fs.topicMatcher.matchedTopics(env.Topic) - _, matchedTopic = m[filterID] - - if matchedTopic { - t.Fatalf("failed MatchEnvelope(topic mismatch) with seed %d.", seed) - } - f.Topics[index][0]-- - // key mismatch prev := *f.KeyAsym.PublicKey.X zero := *big.NewInt(0) @@ -761,10 +734,10 @@ func TestVariableTopics(t *testing.T) { env.Topic = BytesToTopic(f.Topics[i]) //test match - m := fs.topicMatcher.matchedTopics(env.Topic) - _, ok := m[filterID] + matched := []string{} + fs.topicMatcher.matchedTopics(env.Topic, &matched) match = f.MatchEnvelope(env) - if !(match && ok) { + if !(match && hasFilterID(matched, filterID)) { t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed, i) } @@ -773,9 +746,10 @@ func TestVariableTopics(t *testing.T) { //false positive test match = f.MatchEnvelope(env) - m = fs.topicMatcher.matchedTopics(env.Topic) - _, ok = m[filterID] - if match && ok { + + matched = matched[:0] + fs.topicMatcher.matchedTopics(env.Topic, &matched) + if match && hasFilterID(matched, filterID) { t.Fatalf("MatchEnvelope symmetric with seed %d, step %d: false positive.", seed, i) } } @@ -812,8 +786,9 @@ func TestTopicsMapping(t *testing.T) { if err != nil { t.Fatal(err) } - m := fs.topicMatcher.matchedTopics(env.Topic) - if _, matchTopic := m[filterID]; !matchTopic { + matched := []string{} + fs.topicMatcher.matchedTopics(env.Topic, &matched) + if !hasFilterID(matched, filterID) { t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed, i) } @@ -821,8 +796,9 @@ func TestTopicsMapping(t *testing.T) { if !fs.Uninstall(filterID) { t.Fatal("Failed to uninstall filter") } - m = fs.topicMatcher.matchedTopics(env.Topic) - if _, matchTopic := m[filterID]; matchTopic { + matched = matched[:0] + fs.topicMatcher.matchedTopics(env.Topic, &matched) + if hasFilterID(matched, filterID) { t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed, i) } @@ -832,8 +808,9 @@ func TestTopicsMapping(t *testing.T) { if err != nil { t.Fatal(err) } - m = fs.topicMatcher.matchedTopics(env.Topic) - if _, matchTopic := m[filterID]; matchTopic { + matched = matched[:0] + fs.topicMatcher.matchedTopics(env.Topic, &matched) + if hasFilterID(matched, filterID) { t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed, i) } if !fs.Uninstall(filterID) { @@ -862,8 +839,9 @@ func TestTopicsMapping_MatchAllTopics_Success(t *testing.T) { topic := TopicType{} mrand.Read(topic[:]) - m := fs.topicMatcher.matchedTopics(topic) - if _, matchTopic := m[filterID]; !matchTopic { + matched := []string{} + fs.topicMatcher.matchedTopics(topic, &matched) + if !hasFilterID(matched, filterID) { t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed) } if _, ok := fs.topicMatcher.mapper[ALL_TOPICS][filterID]; !ok { @@ -874,8 +852,9 @@ func TestTopicsMapping_MatchAllTopics_Success(t *testing.T) { if !fs.Uninstall(filterID) { t.Fatal("Failed to uninstall filter") } - m = fs.topicMatcher.matchedTopics(topic) - if _, matchTopic := m[filterID]; matchTopic { + matched = matched[:0] + fs.topicMatcher.matchedTopics(topic, &matched) + if hasFilterID(matched, filterID) { t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed) } @@ -883,3 +862,12 @@ func TestTopicsMapping_MatchAllTopics_Success(t *testing.T) { t.Fatal("watcher mapping incorrect") } } + +func hasFilterID(matched []string, filterID string) bool { + for i := range matched { + if matched[i] == filterID { + return true + } + } + return false +} From c58ecf1a0bf416a4ef7881af8b8d6925d6827034 Mon Sep 17 00:00:00 2001 From: b00ris Date: Wed, 21 Feb 2018 23:10:00 +0300 Subject: [PATCH 4/8] whisper: add matching to v6 --- whisper/whisperv5/filter.go | 16 ++- whisper/whisperv5/filter_test.go | 1 + whisper/whisperv6/filter.go | 153 +++++++++++++++------ whisper/whisperv6/filter_test.go | 228 +++++++++++++++++++------------ 4 files changed, 256 insertions(+), 142 deletions(-) diff --git a/whisper/whisperv5/filter.go b/whisper/whisperv5/filter.go index b38ef3bce1..d8c558db26 100644 --- a/whisper/whisperv5/filter.go +++ b/whisper/whisperv5/filter.go @@ -26,7 +26,10 @@ import ( "github.com/ethereum/go-ethereum/log" ) -const ALL_TOPICS = "" +const ( + ALL_TOPICS = "" + MAX_POOL_CAPACITY = 1000 +) type Filter struct { Src *ecdsa.PublicKey // Sender of the message @@ -43,18 +46,16 @@ type Filter struct { type Filters struct { watchers map[string]*Filter - watchersTopics map[string]map[string]struct{} - topicMatcher *topicMatcher whisper *Whisper mutex sync.RWMutex + topicMatcher *topicMatcher } func NewFilters(w *Whisper) *Filters { fs := &Filters{ watchers: make(map[string]*Filter), - watchersTopics: make(map[string]map[string]struct{}), - topicMatcher: newTopicMatcher(), whisper: w, + topicMatcher: newTopicMatcher(), } return fs } @@ -228,6 +229,9 @@ func IsPubKeyEqual(a, b *ecdsa.PublicKey) bool { } type topicMatcher struct { + //structure - map[topic]map[filterID] + //mapping for topics + //"" topic means that the filter allows all topic values mapper map[string]map[string]struct{} mx sync.RWMutex pool sync.Pool @@ -247,7 +251,7 @@ func (fs *topicMatcher) take() []string { return fs.pool.Get().([]string) } func (fs *topicMatcher) resolve(s []string) { - if cap(s) > 1000 { + if cap(s) > MAX_POOL_CAPACITY { return } fs.pool.Put(s[:0]) diff --git a/whisper/whisperv5/filter_test.go b/whisper/whisperv5/filter_test.go index 3e2a4f54df..475884bde3 100644 --- a/whisper/whisperv5/filter_test.go +++ b/whisper/whisperv5/filter_test.go @@ -723,6 +723,7 @@ func TestVariableTopics(t *testing.T) { if err != nil { t.Fatalf("failed generateFilter with seed %d: %s.", seed, err) } + fs := generateFilters() filterID, err := fs.Install(f) diff --git a/whisper/whisperv6/filter.go b/whisper/whisperv6/filter.go index eb0c65fa3b..aca2a297c7 100644 --- a/whisper/whisperv6/filter.go +++ b/whisper/whisperv6/filter.go @@ -26,6 +26,11 @@ import ( "github.com/ethereum/go-ethereum/log" ) +const ( + ALL_TOPICS = "" + MAX_POOL_CAPACITY = 1000 +) + // Filter represents a Whisper message filter type Filter struct { Src *ecdsa.PublicKey // Sender of the message @@ -42,16 +47,18 @@ type Filter struct { // Filters represents a collection of filters type Filters struct { - watchers map[string]*Filter - whisper *Whisper - mutex sync.RWMutex + watchers map[string]*Filter + whisper *Whisper + mutex sync.RWMutex + topicMatcher *topicMatcher } // NewFilters returns a newly created filter collection func NewFilters(w *Whisper) *Filters { return &Filters{ - watchers: make(map[string]*Filter), - whisper: w, + watchers: make(map[string]*Filter), + whisper: w, + topicMatcher: newTopicMatcher(), } } @@ -82,6 +89,7 @@ func (fs *Filters) Install(watcher *Filter) (string, error) { } fs.watchers[id] = watcher + fs.topicMatcher.addFilterToTopicsMapping(watcher, id) return id, err } @@ -92,6 +100,7 @@ func (fs *Filters) Uninstall(id string) bool { defer fs.mutex.Unlock() if fs.watchers[id] != nil { delete(fs.watchers, id) + fs.topicMatcher.removeTopicFromTopicMapping(id) return true } return false @@ -108,15 +117,22 @@ func (fs *Filters) Get(id string) *Filter { // for the envelope's topic. func (fs *Filters) NotifyWatchers(env *Envelope, p2pMessage bool) { var msg *ReceivedMessage + matchedTopics := fs.topicMatcher.take() + defer fs.topicMatcher.resolve(matchedTopics) fs.mutex.RLock() defer fs.mutex.RUnlock() - i := -1 // only used for logging info - for _, watcher := range fs.watchers { - i++ + fs.topicMatcher.matchedTopics(env.Topic, &matchedTopics) + for _, watcherID := range matchedTopics { + watcher, ok := fs.watchers[watcherID] + if !ok { + log.Trace(fmt.Sprintf("msg [%x], filter [%s]: filter not exists", env.Hash(), watcherID)) + continue + } + if p2pMessage && !watcher.AllowP2P { - log.Trace(fmt.Sprintf("msg [%x], filter [%d]: p2p messages are not allowed", env.Hash(), i)) + log.Trace(fmt.Sprintf("msg [%x], filter [%s]: p2p messages are not allowed", env.Hash(), watcherID)) continue } @@ -128,10 +144,10 @@ func (fs *Filters) NotifyWatchers(env *Envelope, p2pMessage bool) { if match { msg = env.Open(watcher) if msg == nil { - log.Trace("processing message: failed to open", "message", env.Hash().Hex(), "filter", i) + log.Trace("processing message: failed to open", "message", env.Hash().Hex(), "filter", watcherID) } } else { - log.Trace("processing message: does not match", "message", env.Hash().Hex(), "filter", i) + log.Trace("processing message: does not match", "message", env.Hash().Hex(), "filter", watcherID) } } @@ -201,9 +217,9 @@ func (f *Filter) MatchMessage(msg *ReceivedMessage) bool { } if f.expectsAsymmetricEncryption() && msg.isAsymmetricEncryption() { - return IsPubKeyEqual(&f.KeyAsym.PublicKey, msg.Dst) && f.MatchTopic(msg.Topic) + return IsPubKeyEqual(&f.KeyAsym.PublicKey, msg.Dst) } else if f.expectsSymmetricEncryption() && msg.isSymmetricEncryption() { - return f.SymKeyHash == msg.SymKeyHash && f.MatchTopic(msg.Topic) + return f.SymKeyHash == msg.SymKeyHash } return false } @@ -216,38 +232,6 @@ func (f *Filter) MatchEnvelope(envelope *Envelope) bool { return false } - return f.MatchTopic(envelope.Topic) -} - -// MatchTopic checks that the filter captures a given topic. -func (f *Filter) MatchTopic(topic TopicType) bool { - if len(f.Topics) == 0 { - // any topic matches - return true - } - - for _, bt := range f.Topics { - if matchSingleTopic(topic, bt) { - return true - } - } - return false -} - -func matchSingleTopic(topic TopicType, bt []byte) bool { - if len(bt) > TopicLength { - bt = bt[:TopicLength] - } - - if len(bt) < TopicLength { - return false - } - - for j, b := range bt { - if topic[j] != b { - return false - } - } return true } @@ -261,3 +245,82 @@ func IsPubKeyEqual(a, b *ecdsa.PublicKey) bool { // the curve is always the same, just compare the points return a.X.Cmp(b.X) == 0 && a.Y.Cmp(b.Y) == 0 } + +type topicMatcher struct { + //structure - map[topic]map[filterID] + //mapping for topics + //"" topic means that the filter allows all topic values + mapper map[string]map[string]struct{} + mx sync.RWMutex + pool sync.Pool +} + +func newTopicMatcher() *topicMatcher { + tm := new(topicMatcher) + tm.mapper = make(map[string]map[string]struct{}) + tm.mapper[ALL_TOPICS] = make(map[string]struct{}) + tm.pool.New = func() interface{} { + return []string{} + } + return tm +} + +func (fs *topicMatcher) take() []string { + return fs.pool.Get().([]string) +} +func (fs *topicMatcher) resolve(s []string) { + if cap(s) > MAX_POOL_CAPACITY { + return + } + fs.pool.Put(s[:0]) +} + +func (fs *topicMatcher) addFilterToTopicsMapping(watcher *Filter, id string) { + fs.mx.Lock() + defer fs.mx.Unlock() + + for i := range fs.prepareTopicsMapping(watcher) { + topicMapping, ok := fs.mapper[i] + if !ok { + fs.mapper[i] = make(map[string]struct{}) + topicMapping = fs.mapper[i] + } + topicMapping[id] = struct{}{} + } +} + +func (fs *topicMatcher) removeTopicFromTopicMapping(id string) { + fs.mx.Lock() + defer fs.mx.Unlock() + for i := range fs.mapper { + delete(fs.mapper[i], id) + } +} + +func (fs *topicMatcher) prepareTopicsMapping(watcher *Filter) map[string]struct{} { + topics := make(map[string]struct{}, len(watcher.Topics)) + + if len(watcher.Topics) == 0 { + topics[ALL_TOPICS] = struct{}{} + return topics + } + + for _, topic := range watcher.Topics { + topics[common.ToHex(topic)] = struct{}{} + } + + return topics +} + +func (fs *topicMatcher) matchedTopics(topic TopicType, matched *[]string) { + fs.mx.RLock() + defer fs.mx.RUnlock() + + for i := range fs.mapper[ALL_TOPICS] { + *matched = append(*matched, i) + } + + for i := range fs.mapper[topic.String()] { + *matched = append(*matched, i) + } +} diff --git a/whisper/whisperv6/filter_test.go b/whisper/whisperv6/filter_test.go index e7230ef388..4324470a9a 100644 --- a/whisper/whisperv6/filter_test.go +++ b/whisper/whisperv6/filter_test.go @@ -83,6 +83,14 @@ func generateFilter(t *testing.T, symmetric bool) (*Filter, error) { return &f, nil } +func generateFilters() *Filters { + fs := Filters{ + watchers: make(map[string]*Filter), + topicMatcher: newTopicMatcher(), + } + return &fs +} + func generateTestCases(t *testing.T, SizeTestFilters int) []FilterTestCase { cases := make([]FilterTestCase, SizeTestFilters) for i := 0; i < SizeTestFilters; i++ { @@ -314,19 +322,8 @@ func TestMatchEnvelope(t *testing.T) { if err != nil { t.Fatalf("failed Wrap with seed %d: %s.", seed, err) } - match := fsym.MatchEnvelope(env) - if match { - t.Fatalf("failed MatchEnvelope symmetric with seed %d.", seed) - } - match = fasym.MatchEnvelope(env) - if match { - t.Fatalf("failed MatchEnvelope asymmetric with seed %d.", seed) - } // encrypt symmetrically - i := mrand.Int() % 4 - fsym.Topics[i] = params.Topic[:] - fasym.Topics[i] = params.Topic[:] msg, err = NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) @@ -336,20 +333,20 @@ func TestMatchEnvelope(t *testing.T) { t.Fatalf("failed Wrap() with seed %d: %s.", seed, err) } - // symmetric + matching topic: match - match = fsym.MatchEnvelope(env) + // symmetric + match := fsym.MatchEnvelope(env) if !match { t.Fatalf("failed MatchEnvelope() symmetric with seed %d.", seed) } - // symmetric + matching topic + insufficient PoW: mismatch + // symmetric + insufficient PoW: mismatch fsym.PoW = env.PoW() + 1.0 match = fsym.MatchEnvelope(env) if match { t.Fatalf("failed MatchEnvelope(symmetric + matching topic + insufficient PoW) asymmetric with seed %d.", seed) } - // symmetric + matching topic + sufficient PoW: match + // symmetric + sufficient PoW: match fsym.PoW = env.PoW() / 2 match = fsym.MatchEnvelope(env) if !match { @@ -387,26 +384,6 @@ func TestMatchEnvelope(t *testing.T) { t.Fatalf("failed MatchEnvelope(encryption method mismatch) with seed %d.", seed) } - // asymmetric + mismatching topic: mismatch - match = fasym.MatchEnvelope(env) - if !match { - t.Fatalf("failed MatchEnvelope(asymmetric + mismatching topic) with seed %d.", seed) - } - - // asymmetric + matching topic: match - fasym.Topics[i] = fasym.Topics[i+1] - match = fasym.MatchEnvelope(env) - if match { - t.Fatalf("failed MatchEnvelope(asymmetric + matching topic) with seed %d.", seed) - } - - // asymmetric + filter without topic (wildcard): match - fasym.Topics = nil - match = fasym.MatchEnvelope(env) - if !match { - t.Fatalf("failed MatchEnvelope(asymmetric + filter without topic) with seed %d.", seed) - } - // asymmetric + insufficient PoW: mismatch fasym.PoW = env.PoW() + 1.0 match = fasym.MatchEnvelope(env) @@ -420,20 +397,6 @@ func TestMatchEnvelope(t *testing.T) { if !match { t.Fatalf("failed MatchEnvelope(asymmetric + sufficient PoW) with seed %d.", seed) } - - // filter without topic + envelope without topic: match - env.Topic = TopicType{} - match = fasym.MatchEnvelope(env) - if !match { - t.Fatalf("failed MatchEnvelope(filter without topic + envelope without topic) with seed %d.", seed) - } - - // filter with topic + envelope without topic: mismatch - fasym.Topics = fsym.Topics - match = fasym.MatchEnvelope(env) - if match { - t.Fatalf("failed MatchEnvelope(filter without topic + envelope without topic) with seed %d.", seed) - } } func TestMatchMessageSym(t *testing.T) { @@ -485,13 +448,6 @@ func TestMatchMessageSym(t *testing.T) { t.Fatalf("failed MatchEnvelope(sufficient PoW) with seed %d.", seed) } - // topic mismatch - f.Topics[index][0]++ - if f.MatchMessage(msg) { - t.Fatalf("failed MatchEnvelope(topic mismatch) with seed %d.", seed) - } - f.Topics[index][0]-- - // key mismatch f.SymKeyHash[0]++ if f.MatchMessage(msg) { @@ -578,13 +534,6 @@ func TestMatchMessageAsym(t *testing.T) { t.Fatalf("failed MatchEnvelope(sufficient PoW) with seed %d.", seed) } - // topic mismatch - f.Topics[index][0]++ - if f.MatchMessage(msg) { - t.Fatalf("failed MatchEnvelope(topic mismatch) with seed %d.", seed) - } - f.Topics[index][0]-- - // key mismatch prev := *f.KeyAsym.PublicKey.X zero := *big.NewInt(0) @@ -820,53 +769,150 @@ func TestVariableTopics(t *testing.T) { t.Fatalf("failed generateFilter with seed %d: %s.", seed, err) } + fs := generateFilters() + + filterID, err := fs.Install(f) + if err != nil { + t.Fatal(err) + } + for i := 0; i < 4; i++ { env.Topic = BytesToTopic(f.Topics[i]) + //test match + matched := []string{} + fs.topicMatcher.matchedTopics(env.Topic, &matched) match = f.MatchEnvelope(env) - if !match { + if !(match && hasFilterID(matched, filterID)) { t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed, i) } - f.Topics[i][lastTopicByte]++ + //change envelop topic + env.Topic[lastTopicByte]++ + + //false positive test match = f.MatchEnvelope(env) - if match { + + matched = matched[:0] + fs.topicMatcher.matchedTopics(env.Topic, &matched) + if match && hasFilterID(matched, filterID) { t.Fatalf("MatchEnvelope symmetric with seed %d, step %d: false positive.", seed, i) } } } -func TestMatchSingleTopic_ReturnTrue(t *testing.T) { - bt := []byte("test") - topic := BytesToTopic(bt) +func TestTopicsMapping(t *testing.T) { + InitSingleTest() - if !matchSingleTopic(topic, bt) { - t.FailNow() + const lastTopicByte = 3 + params, err := generateMessageParams() + if err != nil { + t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) + } + msg, err := NewSentMessage(params) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } + env, err := msg.Wrap(params) + if err != nil { + t.Fatalf("failed Wrap with seed %d: %s.", seed, err) + } + + f, err := generateFilter(t, true) + if err != nil { + t.Fatalf("failed generateFilter with seed %d: %s.", seed, err) + } + fs := generateFilters() + + for i := 0; i < len(f.Topics); i++ { + env.Topic = BytesToTopic(f.Topics[i]) + + //test match + filterID, err := fs.Install(f) + if err != nil { + t.Fatal(err) + } + matched := []string{} + fs.topicMatcher.matchedTopics(env.Topic, &matched) + if !hasFilterID(matched, filterID) { + t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed, i) + } + + //test match without filter + if !fs.Uninstall(filterID) { + t.Fatal("Failed to uninstall filter") + } + matched = matched[:0] + fs.topicMatcher.matchedTopics(env.Topic, &matched) + if hasFilterID(matched, filterID) { + t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed, i) + } + + //test match with changed topic + f.Topics[i][lastTopicByte]++ + filterID, err = fs.Install(f) + if err != nil { + t.Fatal(err) + } + matched = matched[:0] + fs.topicMatcher.matchedTopics(env.Topic, &matched) + if hasFilterID(matched, filterID) { + t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed, i) + } + if !fs.Uninstall(filterID) { + t.Fatal("Failed to uninstall filter") + } } } -func TestMatchSingleTopic_WithTail_ReturnTrue(t *testing.T) { - bt := []byte("test with tail") - topic := BytesToTopic([]byte("test")) +func TestTopicsMapping_MatchAllTopics_Success(t *testing.T) { + InitSingleTest() - if !matchSingleTopic(topic, bt) { - t.FailNow() + f, err := generateFilter(t, true) + if err != nil { + t.Fatalf("failed generateFilter with seed %d: %s.", seed, err) + } + fs := generateFilters() + + f.Topics = [][]byte{} + + filterID, err := fs.Install(f) + if err != nil { + t.Fatal(err) + } + + //generate topic + topic := TopicType{} + mrand.Read(topic[:]) + + matched := []string{} + fs.topicMatcher.matchedTopics(topic, &matched) + if !hasFilterID(matched, filterID) { + t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed) + } + if _, ok := fs.topicMatcher.mapper[ALL_TOPICS][filterID]; !ok { + t.Fatal("watcher mapping incorrect") + } + + //test match without filter + if !fs.Uninstall(filterID) { + t.Fatal("Failed to uninstall filter") + } + matched = matched[:0] + fs.topicMatcher.matchedTopics(topic, &matched) + if hasFilterID(matched, filterID) { + t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed) + } + + if _, ok := fs.topicMatcher.mapper[ALL_TOPICS][filterID]; ok { + t.Fatal("watcher mapping incorrect") } } -func TestMatchSingleTopic_NotEquals_ReturnFalse(t *testing.T) { - bt := []byte("tes") - topic := BytesToTopic(bt) - - if matchSingleTopic(topic, bt) { - t.FailNow() - } -} - -func TestMatchSingleTopic_InsufficientLength_ReturnFalse(t *testing.T) { - bt := []byte("test") - topic := BytesToTopic([]byte("not_equal")) - - if matchSingleTopic(topic, bt) { - t.FailNow() +func hasFilterID(matched []string, filterID string) bool { + for i := range matched { + if matched[i] == filterID { + return true + } } + return false } From db2baf8886041bbad29026ad9ceb32fc98787c71 Mon Sep 17 00:00:00 2001 From: b00ris Date: Thu, 22 Feb 2018 10:28:06 +0300 Subject: [PATCH 5/8] whisper: add comments --- whisper/whisperv5/filter.go | 25 +++++++++++++++++-------- whisper/whisperv6/filter.go | 14 ++++++++++++-- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/whisper/whisperv5/filter.go b/whisper/whisperv5/filter.go index d8c558db26..4f7a0861f4 100644 --- a/whisper/whisperv5/filter.go +++ b/whisper/whisperv5/filter.go @@ -27,7 +27,7 @@ import ( ) const ( - ALL_TOPICS = "" + ALL_TOPICS = "" MAX_POOL_CAPACITY = 1000 ) @@ -45,17 +45,17 @@ type Filter struct { } type Filters struct { - watchers map[string]*Filter - whisper *Whisper - mutex sync.RWMutex - topicMatcher *topicMatcher + watchers map[string]*Filter + whisper *Whisper + mutex sync.RWMutex + topicMatcher *topicMatcher } func NewFilters(w *Whisper) *Filters { fs := &Filters{ - watchers: make(map[string]*Filter), - whisper: w, - topicMatcher: newTopicMatcher(), + watchers: make(map[string]*Filter), + whisper: w, + topicMatcher: newTopicMatcher(), } return fs } @@ -228,6 +228,7 @@ func IsPubKeyEqual(a, b *ecdsa.PublicKey) bool { return a.X.Cmp(b.X) == 0 && a.Y.Cmp(b.Y) == 0 } +//topicMatcher keeps topic->watcher mapping type topicMatcher struct { //structure - map[topic]map[filterID] //mapping for topics @@ -237,6 +238,7 @@ type topicMatcher struct { pool sync.Pool } +//newTopicMatcher returns a newly created topic matcher func newTopicMatcher() *topicMatcher { tm := new(topicMatcher) tm.mapper = make(map[string]map[string]struct{}) @@ -247,9 +249,12 @@ func newTopicMatcher() *topicMatcher { return tm } +//take returns []string from pool func (fs *topicMatcher) take() []string { return fs.pool.Get().([]string) } + +//resolve put []string to pool func (fs *topicMatcher) resolve(s []string) { if cap(s) > MAX_POOL_CAPACITY { return @@ -257,6 +262,7 @@ func (fs *topicMatcher) resolve(s []string) { fs.pool.Put(s[:0]) } +//addFilterToTopicsMapping fill topic->watcher mapping for current watcher func (fs *topicMatcher) addFilterToTopicsMapping(watcher *Filter, id string) { fs.mx.Lock() defer fs.mx.Unlock() @@ -271,6 +277,7 @@ func (fs *topicMatcher) addFilterToTopicsMapping(watcher *Filter, id string) { } } +//removeTopicFromTopicMapping removes mapping info by filterID func (fs *topicMatcher) removeTopicFromTopicMapping(id string) { fs.mx.Lock() defer fs.mx.Unlock() @@ -279,6 +286,7 @@ func (fs *topicMatcher) removeTopicFromTopicMapping(id string) { } } +//prepareTopicsMapping returns set of topics for watcher func (fs *topicMatcher) prepareTopicsMapping(watcher *Filter) map[string]struct{} { topics := make(map[string]struct{}, len(watcher.Topics)) @@ -294,6 +302,7 @@ func (fs *topicMatcher) prepareTopicsMapping(watcher *Filter) map[string]struct{ return topics } +//matchedTopics write all matched topics to matched func (fs *topicMatcher) matchedTopics(topic TopicType, matched *[]string) { fs.mx.RLock() defer fs.mx.RUnlock() diff --git a/whisper/whisperv6/filter.go b/whisper/whisperv6/filter.go index aca2a297c7..a59da6ddae 100644 --- a/whisper/whisperv6/filter.go +++ b/whisper/whisperv6/filter.go @@ -89,6 +89,7 @@ func (fs *Filters) Install(watcher *Filter) (string, error) { } fs.watchers[id] = watcher + //add topic matching for watcher fs.topicMatcher.addFilterToTopicsMapping(watcher, id) return id, err } @@ -246,6 +247,7 @@ func IsPubKeyEqual(a, b *ecdsa.PublicKey) bool { return a.X.Cmp(b.X) == 0 && a.Y.Cmp(b.Y) == 0 } +//topicMatcher keeps topic->watcher mapping type topicMatcher struct { //structure - map[topic]map[filterID] //mapping for topics @@ -255,6 +257,7 @@ type topicMatcher struct { pool sync.Pool } +//newTopicMatcher returns a newly created topic matcher func newTopicMatcher() *topicMatcher { tm := new(topicMatcher) tm.mapper = make(map[string]map[string]struct{}) @@ -265,9 +268,12 @@ func newTopicMatcher() *topicMatcher { return tm } +//take returns []string from pool func (fs *topicMatcher) take() []string { return fs.pool.Get().([]string) } + +//resolve put []string to pool func (fs *topicMatcher) resolve(s []string) { if cap(s) > MAX_POOL_CAPACITY { return @@ -275,6 +281,7 @@ func (fs *topicMatcher) resolve(s []string) { fs.pool.Put(s[:0]) } +//addFilterToTopicsMapping fill topic->watcher mapping for current watcher func (fs *topicMatcher) addFilterToTopicsMapping(watcher *Filter, id string) { fs.mx.Lock() defer fs.mx.Unlock() @@ -289,14 +296,16 @@ func (fs *topicMatcher) addFilterToTopicsMapping(watcher *Filter, id string) { } } -func (fs *topicMatcher) removeTopicFromTopicMapping(id string) { +//removeTopicFromTopicMapping removes mapping info by filterID +func (fs *topicMatcher) removeTopicFromTopicMapping(filterID string) { fs.mx.Lock() defer fs.mx.Unlock() for i := range fs.mapper { - delete(fs.mapper[i], id) + delete(fs.mapper[i], filterID) } } +//prepareTopicsMapping returns set of topics for watcher func (fs *topicMatcher) prepareTopicsMapping(watcher *Filter) map[string]struct{} { topics := make(map[string]struct{}, len(watcher.Topics)) @@ -312,6 +321,7 @@ func (fs *topicMatcher) prepareTopicsMapping(watcher *Filter) map[string]struct{ return topics } +//matchedTopics write all matched topics to matched func (fs *topicMatcher) matchedTopics(topic TopicType, matched *[]string) { fs.mx.RLock() defer fs.mx.RUnlock() From 0315c192755feefbb95428854ef8266e343007f9 Mon Sep 17 00:00:00 2001 From: b00ris Date: Mon, 26 Feb 2018 08:27:54 +0300 Subject: [PATCH 6/8] whisper: add benchmarks --- whisper/whisperv5/filter_test.go | 105 +++++++++++++++++++++++------ whisper/whisperv5/whisper_test.go | 8 +-- whisper/whisperv6/filter_test.go | 108 ++++++++++++++++++++++++------ whisper/whisperv6/whisper_test.go | 8 +-- 4 files changed, 183 insertions(+), 46 deletions(-) diff --git a/whisper/whisperv5/filter_test.go b/whisper/whisperv5/filter_test.go index 475884bde3..5ba2a2cd0b 100644 --- a/whisper/whisperv5/filter_test.go +++ b/whisper/whisperv5/filter_test.go @@ -22,6 +22,7 @@ import ( "testing" "time" + "fmt" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" ) @@ -48,7 +49,7 @@ type FilterTestCase struct { msgCnt int } -func generateFilter(t *testing.T, symmetric bool) (*Filter, error) { +func generateFilter(symmetric bool) (*Filter, error) { var f Filter f.Messages = make(map[common.Hash]*ReceivedMessage) @@ -62,8 +63,8 @@ func generateFilter(t *testing.T, symmetric bool) (*Filter, error) { key, err := crypto.GenerateKey() if err != nil { - t.Fatalf("generateFilter 1 failed with seed %d.", seed) - return nil, err + return nil, fmt.Errorf("generateFilter 1 failed with seed %d. Error: %s", seed, err.Error()) + } f.Src = &key.PublicKey @@ -74,8 +75,7 @@ func generateFilter(t *testing.T, symmetric bool) (*Filter, error) { } else { f.KeyAsym, err = crypto.GenerateKey() if err != nil { - t.Fatalf("generateFilter 2 failed with seed %d.", seed) - return nil, err + return nil, fmt.Errorf("generateFilter 2 failed with seed %d. Error: %s", seed, err.Error()) } } @@ -94,7 +94,10 @@ func generateFilters() *Filters { func generateTestCases(t *testing.T, SizeTestFilters int) []FilterTestCase { cases := make([]FilterTestCase, SizeTestFilters) for i := 0; i < SizeTestFilters; i++ { - f, _ := generateFilter(t, true) + f, err := generateFilter(true) + if err != nil { + t.Fatal(err) + } cases[i].f = f cases[i].alive = mrand.Int()&int(1) == 0 } @@ -145,8 +148,10 @@ func TestInstallSymKeyGeneratesHash(t *testing.T) { w := New(&Config{}) filters := NewFilters(w) - filter, _ := generateFilter(t, true) - + filter, err := generateFilter(true) + if err != nil { + t.Fatal(err) + } // save the current SymKeyHash for comparison initialSymKeyHash := filter.SymKeyHash @@ -154,7 +159,7 @@ func TestInstallSymKeyGeneratesHash(t *testing.T) { var invalid common.Hash filter.SymKeyHash = invalid - _, err := filters.Install(filter) + _, err = filters.Install(filter) if err != nil { t.Fatalf("Error installing the filter: %s", err) @@ -172,8 +177,10 @@ func TestInstallIdenticalFilters(t *testing.T) { w := New(&Config{}) filters := NewFilters(w) - filter1, _ := generateFilter(t, true) - + filter1, err := generateFilter(true) + if err != nil { + t.Fatal(err) + } // Copy the first filter since some of its fields // are randomly gnerated. filter2 := &Filter{ @@ -184,7 +191,7 @@ func TestInstallIdenticalFilters(t *testing.T) { Messages: make(map[common.Hash]*ReceivedMessage), } - _, err := filters.Install(filter1) + _, err = filters.Install(filter1) if err != nil { t.Fatalf("Error installing the first filter with seed %d: %s", seed, err) @@ -266,12 +273,12 @@ func TestComparePubKey(t *testing.T) { func TestMatchEnvelope(t *testing.T) { InitSingleTest() - fsym, err := generateFilter(t, true) + fsym, err := generateFilter(true) if err != nil { t.Fatalf("failed generateFilter with seed %d: %s.", seed, err) } - fasym, err := generateFilter(t, false) + fasym, err := generateFilter(false) if err != nil { t.Fatalf("failed generateFilter() with seed %d: %s.", seed, err) } @@ -362,7 +369,7 @@ func TestMatchMessageSym(t *testing.T) { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - f, err := generateFilter(t, true) + f, err := generateFilter(true) if err != nil { t.Fatalf("failed generateFilter with seed %d: %s.", seed, err) } @@ -441,7 +448,7 @@ func TestMatchMessageSym(t *testing.T) { func TestMatchMessageAsym(t *testing.T) { InitSingleTest() - f, err := generateFilter(t, false) + f, err := generateFilter(false) if err != nil { t.Fatalf("failed generateFilter with seed %d: %s.", seed, err) } @@ -719,7 +726,7 @@ func TestVariableTopics(t *testing.T) { t.Fatalf("failed Wrap with seed %d: %s.", seed, err) } - f, err := generateFilter(t, true) + f, err := generateFilter(true) if err != nil { t.Fatalf("failed generateFilter with seed %d: %s.", seed, err) } @@ -773,7 +780,7 @@ func TestTopicsMapping(t *testing.T) { t.Fatalf("failed Wrap with seed %d: %s.", seed, err) } - f, err := generateFilter(t, true) + f, err := generateFilter(true) if err != nil { t.Fatalf("failed generateFilter with seed %d: %s.", seed, err) } @@ -823,7 +830,7 @@ func TestTopicsMapping(t *testing.T) { func TestTopicsMapping_MatchAllTopics_Success(t *testing.T) { InitSingleTest() - f, err := generateFilter(t, true) + f, err := generateFilter(true) if err != nil { t.Fatalf("failed generateFilter with seed %d: %s.", seed, err) } @@ -872,3 +879,63 @@ func hasFilterID(matched []string, filterID string) bool { } return false } + +func BenchmarkFilter_MatchEnvelope_5Filters(b *testing.B) { + InitSingleTest() + benchFilter_MatchMessage(b, 5) +} +func BenchmarkFilter_MatchEnvelope_10Filters(b *testing.B) { + InitSingleTest() + benchFilter_MatchMessage(b, 10) +} +func BenchmarkFilter_MatchEnvelope_20Filters(b *testing.B) { + InitSingleTest() + benchFilter_MatchMessage(b, 20) +} +func BenchmarkFilter_MatchEnvelope_50Filters(b *testing.B) { + InitSingleTest() + benchFilter_MatchMessage(b, 50) +} +func BenchmarkFilter_MatchEnvelope_100Filters(b *testing.B) { + InitSingleTest() + benchFilter_MatchMessage(b, 100) +} + +func benchFilter_MatchMessage(b *testing.B, numOfFilters int) { + params, err := generateMessageParams() + if err != nil { + b.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) + } + msg, err := NewSentMessage(params) + if err != nil { + b.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } + env, err := msg.Wrap(params) + if err != nil { + b.Fatalf("failed Wrap with seed %d: %s.", seed, err) + } + + fs := generateFilters() + + for i := 0; i < numOfFilters; i++ { + f, err := generateFilter(true) + if err != nil { + b.Fatalf("failed generateFilter with seed %d: %s.", seed, err) + } + + _, err = fs.Install(f) + if err != nil { + b.Fatalf("failed generateFilter with seed %d: %s.", seed, err) + } + + } + + var topic TopicType + b.ResetTimer() + for i := 0; i < b.N; i++ { + mrand.Read(topic[:]) + env.Topic = topic + + fs.NotifyWatchers(env, false) + } +} diff --git a/whisper/whisperv5/whisper_test.go b/whisper/whisperv5/whisper_test.go index 8af085292a..0fc50b2ab4 100644 --- a/whisper/whisperv5/whisper_test.go +++ b/whisper/whisperv5/whisper_test.go @@ -542,7 +542,7 @@ func TestCustomization(t *testing.T) { const smallPoW = 0.00001 - f, err := generateFilter(t, true) + f, err := generateFilter(true) if err != nil { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } @@ -636,7 +636,7 @@ func TestSymmetricSendCycle(t *testing.T) { w.Start(nil) defer w.Stop() - filter1, err := generateFilter(t, true) + filter1, err := generateFilter(true) if err != nil { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } @@ -725,7 +725,7 @@ func TestSymmetricSendWithoutAKey(t *testing.T) { w.Start(nil) defer w.Stop() - filter, err := generateFilter(t, true) + filter, err := generateFilter(true) if err != nil { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } @@ -793,7 +793,7 @@ func TestSymmetricSendKeyMismatch(t *testing.T) { w.Start(nil) defer w.Stop() - filter, err := generateFilter(t, true) + filter, err := generateFilter(true) if err != nil { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } diff --git a/whisper/whisperv6/filter_test.go b/whisper/whisperv6/filter_test.go index 4324470a9a..45c4dd54dc 100644 --- a/whisper/whisperv6/filter_test.go +++ b/whisper/whisperv6/filter_test.go @@ -22,6 +22,7 @@ import ( "testing" "time" + "fmt" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" ) @@ -48,7 +49,7 @@ type FilterTestCase struct { msgCnt int } -func generateFilter(t *testing.T, symmetric bool) (*Filter, error) { +func generateFilter(symmetric bool) (*Filter, error) { var f Filter f.Messages = make(map[common.Hash]*ReceivedMessage) @@ -62,8 +63,7 @@ func generateFilter(t *testing.T, symmetric bool) (*Filter, error) { key, err := crypto.GenerateKey() if err != nil { - t.Fatalf("generateFilter 1 failed with seed %d.", seed) - return nil, err + return nil, fmt.Errorf("generateFilter 1 failed with seed %d. Error: %s", seed, err.Error()) } f.Src = &key.PublicKey @@ -74,8 +74,7 @@ func generateFilter(t *testing.T, symmetric bool) (*Filter, error) { } else { f.KeyAsym, err = crypto.GenerateKey() if err != nil { - t.Fatalf("generateFilter 2 failed with seed %d.", seed) - return nil, err + return nil, fmt.Errorf("generateFilter 2 failed with seed %d. Error: %s", seed, err.Error()) } } @@ -94,7 +93,10 @@ func generateFilters() *Filters { func generateTestCases(t *testing.T, SizeTestFilters int) []FilterTestCase { cases := make([]FilterTestCase, SizeTestFilters) for i := 0; i < SizeTestFilters; i++ { - f, _ := generateFilter(t, true) + f, err := generateFilter(true) + if err != nil { + t.Fatal(err) + } cases[i].f = f cases[i].alive = mrand.Int()&int(1) == 0 } @@ -145,8 +147,10 @@ func TestInstallSymKeyGeneratesHash(t *testing.T) { w := New(&Config{}) filters := NewFilters(w) - filter, _ := generateFilter(t, true) - + filter, err := generateFilter(true) + if err != nil { + t.Fatal(err) + } // save the current SymKeyHash for comparison initialSymKeyHash := filter.SymKeyHash @@ -154,7 +158,7 @@ func TestInstallSymKeyGeneratesHash(t *testing.T) { var invalid common.Hash filter.SymKeyHash = invalid - _, err := filters.Install(filter) + _, err = filters.Install(filter) if err != nil { t.Fatalf("Error installing the filter: %s", err) @@ -172,7 +176,10 @@ func TestInstallIdenticalFilters(t *testing.T) { w := New(&Config{}) filters := NewFilters(w) - filter1, _ := generateFilter(t, true) + filter1, err := generateFilter(true) + if err != nil { + t.Fatal(err) + } // Copy the first filter since some of its fields // are randomly gnerated. @@ -184,7 +191,7 @@ func TestInstallIdenticalFilters(t *testing.T) { Messages: make(map[common.Hash]*ReceivedMessage), } - _, err := filters.Install(filter1) + _, err = filters.Install(filter1) if err != nil { t.Fatalf("Error installing the first filter with seed %d: %s", seed, err) @@ -242,7 +249,10 @@ func TestInstallFilterWithSymAndAsymKeys(t *testing.T) { w := New(&Config{}) filters := NewFilters(w) - filter1, _ := generateFilter(t, true) + filter1, err := generateFilter(true) + if err != nil { + t.Fatal(err) + } asymKey, err := crypto.GenerateKey() if err != nil { @@ -296,12 +306,12 @@ func TestComparePubKey(t *testing.T) { func TestMatchEnvelope(t *testing.T) { InitSingleTest() - fsym, err := generateFilter(t, true) + fsym, err := generateFilter(true) if err != nil { t.Fatalf("failed generateFilter with seed %d: %s.", seed, err) } - fasym, err := generateFilter(t, false) + fasym, err := generateFilter(false) if err != nil { t.Fatalf("failed generateFilter() with seed %d: %s.", seed, err) } @@ -407,7 +417,7 @@ func TestMatchMessageSym(t *testing.T) { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - f, err := generateFilter(t, true) + f, err := generateFilter(true) if err != nil { t.Fatalf("failed generateFilter with seed %d: %s.", seed, err) } @@ -486,7 +496,7 @@ func TestMatchMessageSym(t *testing.T) { func TestMatchMessageAsym(t *testing.T) { InitSingleTest() - f, err := generateFilter(t, false) + f, err := generateFilter(false) if err != nil { t.Fatalf("failed generateFilter with seed %d: %s.", seed, err) } @@ -764,7 +774,7 @@ func TestVariableTopics(t *testing.T) { t.Fatalf("failed Wrap with seed %d: %s.", seed, err) } - f, err := generateFilter(t, true) + f, err := generateFilter(true) if err != nil { t.Fatalf("failed generateFilter with seed %d: %s.", seed, err) } @@ -817,7 +827,7 @@ func TestTopicsMapping(t *testing.T) { t.Fatalf("failed Wrap with seed %d: %s.", seed, err) } - f, err := generateFilter(t, true) + f, err := generateFilter(true) if err != nil { t.Fatalf("failed generateFilter with seed %d: %s.", seed, err) } @@ -867,7 +877,7 @@ func TestTopicsMapping(t *testing.T) { func TestTopicsMapping_MatchAllTopics_Success(t *testing.T) { InitSingleTest() - f, err := generateFilter(t, true) + f, err := generateFilter(true) if err != nil { t.Fatalf("failed generateFilter with seed %d: %s.", seed, err) } @@ -916,3 +926,63 @@ func hasFilterID(matched []string, filterID string) bool { } return false } + +func BenchmarkFilter_MatchEnvelope_5Filters(b *testing.B) { + InitSingleTest() + benchFilter_MatchMessage(b, 5) +} +func BenchmarkFilter_MatchEnvelope_10Filters(b *testing.B) { + InitSingleTest() + benchFilter_MatchMessage(b, 10) +} +func BenchmarkFilter_MatchEnvelope_20Filters(b *testing.B) { + InitSingleTest() + benchFilter_MatchMessage(b, 20) +} +func BenchmarkFilter_MatchEnvelope_50Filters(b *testing.B) { + InitSingleTest() + benchFilter_MatchMessage(b, 50) +} +func BenchmarkFilter_MatchEnvelope_100Filters(b *testing.B) { + InitSingleTest() + benchFilter_MatchMessage(b, 100) +} + +func benchFilter_MatchMessage(b *testing.B, numOfFilters int) { + params, err := generateMessageParams() + if err != nil { + b.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) + } + msg, err := NewSentMessage(params) + if err != nil { + b.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } + env, err := msg.Wrap(params) + if err != nil { + b.Fatalf("failed Wrap with seed %d: %s.", seed, err) + } + + fs := generateFilters() + + for i := 0; i < numOfFilters; i++ { + f, err := generateFilter(true) + if err != nil { + b.Fatalf("failed generateFilter with seed %d: %s.", seed, err) + } + + _, err = fs.Install(f) + if err != nil { + b.Fatalf("failed generateFilter with seed %d: %s.", seed, err) + } + + } + + var topic TopicType + b.ResetTimer() + for i := 0; i < b.N; i++ { + mrand.Read(topic[:]) + env.Topic = topic + + fs.NotifyWatchers(env, false) + } +} diff --git a/whisper/whisperv6/whisper_test.go b/whisper/whisperv6/whisper_test.go index 99e5f0bbb4..422e601e7a 100644 --- a/whisper/whisperv6/whisper_test.go +++ b/whisper/whisperv6/whisper_test.go @@ -524,7 +524,7 @@ func TestCustomization(t *testing.T) { const smallPoW = 0.00001 - f, err := generateFilter(t, true) + f, err := generateFilter(true) if err != nil { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } @@ -618,7 +618,7 @@ func TestSymmetricSendCycle(t *testing.T) { w.Start(nil) defer w.Stop() - filter1, err := generateFilter(t, true) + filter1, err := generateFilter(true) if err != nil { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } @@ -707,7 +707,7 @@ func TestSymmetricSendWithoutAKey(t *testing.T) { w.Start(nil) defer w.Stop() - filter, err := generateFilter(t, true) + filter, err := generateFilter(true) if err != nil { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } @@ -775,7 +775,7 @@ func TestSymmetricSendKeyMismatch(t *testing.T) { w.Start(nil) defer w.Stop() - filter, err := generateFilter(t, true) + filter, err := generateFilter(true) if err != nil { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } From 1d092fec7c8779c76b95cdf3bd2f647c6a3b7b32 Mon Sep 17 00:00:00 2001 From: b00ris Date: Mon, 26 Feb 2018 11:18:37 +0300 Subject: [PATCH 7/8] whisper: changed error text --- whisper/whisperv5/filter_test.go | 12 ++++++------ whisper/whisperv6/filter_test.go | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/whisper/whisperv5/filter_test.go b/whisper/whisperv5/filter_test.go index 5ba2a2cd0b..c5f2412050 100644 --- a/whisper/whisperv5/filter_test.go +++ b/whisper/whisperv5/filter_test.go @@ -797,7 +797,7 @@ func TestTopicsMapping(t *testing.T) { matched := []string{} fs.topicMatcher.matchedTopics(env.Topic, &matched) if !hasFilterID(matched, filterID) { - t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed, i) + t.Fatalf("failed matchedTopics, step %d.", i) } //test match without filter @@ -807,7 +807,7 @@ func TestTopicsMapping(t *testing.T) { matched = matched[:0] fs.topicMatcher.matchedTopics(env.Topic, &matched) if hasFilterID(matched, filterID) { - t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed, i) + t.Fatalf("failed match without filter, step %d.", i) } //test match with changed topic @@ -819,7 +819,7 @@ func TestTopicsMapping(t *testing.T) { matched = matched[:0] fs.topicMatcher.matchedTopics(env.Topic, &matched) if hasFilterID(matched, filterID) { - t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed, i) + t.Fatalf("failed match with changed topic, step %d.", i) } if !fs.Uninstall(filterID) { t.Fatal("Failed to uninstall filter") @@ -850,7 +850,7 @@ func TestTopicsMapping_MatchAllTopics_Success(t *testing.T) { matched := []string{} fs.topicMatcher.matchedTopics(topic, &matched) if !hasFilterID(matched, filterID) { - t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed) + t.Fatal("failed matchedTopics") } if _, ok := fs.topicMatcher.mapper[ALL_TOPICS][filterID]; !ok { t.Fatal("watcher mapping incorrect") @@ -863,7 +863,7 @@ func TestTopicsMapping_MatchAllTopics_Success(t *testing.T) { matched = matched[:0] fs.topicMatcher.matchedTopics(topic, &matched) if hasFilterID(matched, filterID) { - t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed) + t.Fatal("failed match without filter") } if _, ok := fs.topicMatcher.mapper[ALL_TOPICS][filterID]; ok { @@ -925,7 +925,7 @@ func benchFilter_MatchMessage(b *testing.B, numOfFilters int) { _, err = fs.Install(f) if err != nil { - b.Fatalf("failed generateFilter with seed %d: %s.", seed, err) + b.Fatalf("failed install filter with seed %d: %s.", seed, err) } } diff --git a/whisper/whisperv6/filter_test.go b/whisper/whisperv6/filter_test.go index 45c4dd54dc..20dda1680a 100644 --- a/whisper/whisperv6/filter_test.go +++ b/whisper/whisperv6/filter_test.go @@ -844,7 +844,7 @@ func TestTopicsMapping(t *testing.T) { matched := []string{} fs.topicMatcher.matchedTopics(env.Topic, &matched) if !hasFilterID(matched, filterID) { - t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed, i) + t.Fatalf("failed matchedTopics, step %d.", seed, i) } //test match without filter @@ -854,7 +854,7 @@ func TestTopicsMapping(t *testing.T) { matched = matched[:0] fs.topicMatcher.matchedTopics(env.Topic, &matched) if hasFilterID(matched, filterID) { - t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed, i) + t.Fatalf("failed match without filter, step %d.", i) } //test match with changed topic @@ -866,7 +866,7 @@ func TestTopicsMapping(t *testing.T) { matched = matched[:0] fs.topicMatcher.matchedTopics(env.Topic, &matched) if hasFilterID(matched, filterID) { - t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed, i) + t.Fatalf("failed match with changed topic, step %d.", i) } if !fs.Uninstall(filterID) { t.Fatal("Failed to uninstall filter") @@ -897,7 +897,7 @@ func TestTopicsMapping_MatchAllTopics_Success(t *testing.T) { matched := []string{} fs.topicMatcher.matchedTopics(topic, &matched) if !hasFilterID(matched, filterID) { - t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed) + t.Fatal("failed matchedTopics") } if _, ok := fs.topicMatcher.mapper[ALL_TOPICS][filterID]; !ok { t.Fatal("watcher mapping incorrect") @@ -910,7 +910,7 @@ func TestTopicsMapping_MatchAllTopics_Success(t *testing.T) { matched = matched[:0] fs.topicMatcher.matchedTopics(topic, &matched) if hasFilterID(matched, filterID) { - t.Fatalf("failed MatchEnvelope symmetric with seed %d, step %d.", seed) + t.Fatal("failed match without filter") } if _, ok := fs.topicMatcher.mapper[ALL_TOPICS][filterID]; ok { @@ -972,7 +972,7 @@ func benchFilter_MatchMessage(b *testing.B, numOfFilters int) { _, err = fs.Install(f) if err != nil { - b.Fatalf("failed generateFilter with seed %d: %s.", seed, err) + b.Fatalf("failed install filter with seed %d: %s.", seed, err) } } From 37f0f2c824b85a467a97f5bbfc028191e5356e8b Mon Sep 17 00:00:00 2001 From: b00ris Date: Mon, 26 Feb 2018 11:39:24 +0300 Subject: [PATCH 8/8] whisper: fix num of params --- whisper/whisperv6/filter_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/whisper/whisperv6/filter_test.go b/whisper/whisperv6/filter_test.go index 20dda1680a..42e1f05018 100644 --- a/whisper/whisperv6/filter_test.go +++ b/whisper/whisperv6/filter_test.go @@ -844,7 +844,7 @@ func TestTopicsMapping(t *testing.T) { matched := []string{} fs.topicMatcher.matchedTopics(env.Topic, &matched) if !hasFilterID(matched, filterID) { - t.Fatalf("failed matchedTopics, step %d.", seed, i) + t.Fatalf("failed matchedTopics, step %d.", i) } //test match without filter