diff --git a/core/bloombits/matcher.go b/core/bloombits/matcher.go index f3ed405a61..d95dc0187a 100644 --- a/core/bloombits/matcher.go +++ b/core/bloombits/matcher.go @@ -80,7 +80,8 @@ type Matcher struct { } // NewMatcher creates a new pipeline for retrieving bloom bit streams and doing -// address and topic filtering on them. +// address and topic filtering on them. Setting a filter component to `nil` is +// allowed and will result in that filter rule being skipped (OR 0x11...1). func NewMatcher(sectionSize uint64, filters [][][]byte) *Matcher { // Create the matcher instance m := &Matcher{ @@ -95,11 +96,19 @@ func NewMatcher(sectionSize uint64, filters [][][]byte) *Matcher { m.filters = nil for _, filter := range filters { + // Gather the bit indexes of the filter rule, special casing the nil filter bloomBits := make([]bloomIndexes, len(filter)) for i, clause := range filter { + if clause == nil { + bloomBits = nil + break + } bloomBits[i] = calcBloomIndexes(clause) } - m.filters = append(m.filters, bloomBits) + // Accumulate the filter rules if no nil rule was within + if bloomBits != nil { + m.filters = append(m.filters, bloomBits) + } } // For every bit, create a scheduler to load/download the bit vectors for _, bloomIndexLists := range m.filters { diff --git a/core/bloombits/matcher_test.go b/core/bloombits/matcher_test.go index f0198c4e38..3098814c65 100644 --- a/core/bloombits/matcher_test.go +++ b/core/bloombits/matcher_test.go @@ -21,10 +21,36 @@ import ( "sync/atomic" "testing" "time" + + "github.com/ethereum/go-ethereum/common" ) const testSectionSize = 4096 +// Tests that wildcard filter rules (nil) can be specified and are handled well. +func TestMatcherWildcards(t *testing.T) { + matcher := NewMatcher(testSectionSize, [][][]byte{ + [][]byte{common.Address{}.Bytes(), common.Address{0x01}.Bytes()}, // Default address is not a wildcard + [][]byte{common.Hash{}.Bytes(), common.Hash{0x01}.Bytes()}, // Default hash is not a wildcard + [][]byte{common.Hash{0x01}.Bytes()}, // Plain rule, sanity check + [][]byte{common.Hash{0x01}.Bytes(), nil}, // Wildcard suffix, drop rule + [][]byte{nil, common.Hash{0x01}.Bytes()}, // Wildcard prefix, drop rule + [][]byte{nil, nil}, // Wildcard rule, drop rule + }) + if len(matcher.filters) != 3 { + t.Fatalf("filter system size mismatch: have %d, want %d", len(matcher.filters), 3) + } + if len(matcher.filters[0]) != 2 { + t.Fatalf("address clause size mismatch: have %d, want %d", len(matcher.filters[0]), 2) + } + if len(matcher.filters[1]) != 2 { + t.Fatalf("combo topic clause size mismatch: have %d, want %d", len(matcher.filters[1]), 2) + } + if len(matcher.filters[2]) != 1 { + t.Fatalf("singletone topic clause size mismatch: have %d, want %d", len(matcher.filters[2]), 1) + } +} + // Tests the matcher pipeline on a single continuous workflow without interrupts. func TestMatcherContinuous(t *testing.T) { testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 100000, false, 75) diff --git a/eth/filters/api.go b/eth/filters/api.go index 6e1d48adb6..bf3b44623b 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -271,7 +271,7 @@ type FilterCriteria struct { FromBlock *big.Int ToBlock *big.Int Addresses []common.Address - Topics [][]common.Hash + Topics [][]*common.Hash } // NewFilter creates a new filter and returns the filter id. It can be @@ -493,12 +493,12 @@ func (args *FilterCriteria) UnmarshalJSON(data []byte) error { // topics is an array consisting of strings and/or arrays of strings. // JSON null values are converted to common.Hash{} and ignored by the filter manager. if len(raw.Topics) > 0 { - args.Topics = make([][]common.Hash, len(raw.Topics)) + args.Topics = make([][]*common.Hash, len(raw.Topics)) for i, t := range raw.Topics { switch topic := t.(type) { case nil: // ignore topic when matching logs - args.Topics[i] = []common.Hash{{}} + args.Topics[i] = []*common.Hash{nil} case string: // match specific topic @@ -506,18 +506,19 @@ func (args *FilterCriteria) UnmarshalJSON(data []byte) error { if err != nil { return err } - args.Topics[i] = []common.Hash{top} + args.Topics[i] = []*common.Hash{&top} + case []interface{}: // or case e.g. [null, "topic0", "topic1"] for _, rawTopic := range topic { if rawTopic == nil { - args.Topics[i] = append(args.Topics[i], common.Hash{}) + args.Topics[i] = append(args.Topics[i], nil) } else if topic, ok := rawTopic.(string); ok { parsed, err := decodeTopic(topic) if err != nil { return err } - args.Topics[i] = append(args.Topics[i], parsed) + args.Topics[i] = append(args.Topics[i], &parsed) } else { return fmt.Errorf("invalid topic(s)") } diff --git a/eth/filters/api_test.go b/eth/filters/api_test.go index 068a5ea243..b9b397abb3 100644 --- a/eth/filters/api_test.go +++ b/eth/filters/api_test.go @@ -34,7 +34,6 @@ func TestUnmarshalJSONNewFilterArgs(t *testing.T) { topic0 = common.HexToHash("3ac225168df54212a25c1c01fd35bebfea408fdac2e31ddd6f80a4bbf9a5f1ca") topic1 = common.HexToHash("9084a792d2f8b16a62b882fd56f7860c07bf5fa91dd8a2ae7e809e5180fef0b3") topic2 = common.HexToHash("6ccae1c4af4152f460ff510e573399795dfab5dcf1fa60d1f33ac8fdc1e480ce") - nullTopic = common.Hash{} ) // default values @@ -109,7 +108,7 @@ func TestUnmarshalJSONNewFilterArgs(t *testing.T) { if len(test4.Topics[0]) != 1 { t.Fatalf("expected len(topics[0]) to be 1, got %d", len(test4.Topics[0])) } - if test4.Topics[0][0] != topic0 { + if *test4.Topics[0][0] != topic0 { t.Fatalf("got %x, expected %x", test4.Topics[0][0], topic0) } @@ -125,13 +124,13 @@ func TestUnmarshalJSONNewFilterArgs(t *testing.T) { if len(test5.Topics[0]) != 1 { t.Fatalf("expected 1 topic, got %d", len(test5.Topics[0])) } - if test5.Topics[0][0] != topic0 { + if *test5.Topics[0][0] != topic0 { t.Fatalf("got %x, expected %x", test5.Topics[0][0], topic0) } if len(test5.Topics[1]) != 1 { t.Fatalf("expected 1 topic, got %d", len(test5.Topics[1])) } - if test5.Topics[1][0] != topic1 { + if *test5.Topics[1][0] != topic1 { t.Fatalf("got %x, expected %x", test5.Topics[1][0], topic1) } @@ -147,19 +146,19 @@ func TestUnmarshalJSONNewFilterArgs(t *testing.T) { if len(test6.Topics[0]) != 1 { t.Fatalf("expected 1 topic, got %d", len(test6.Topics[0])) } - if test6.Topics[0][0] != topic0 { + if *test6.Topics[0][0] != topic0 { t.Fatalf("got %x, expected %x", test6.Topics[0][0], topic0) } if len(test6.Topics[1]) != 1 { t.Fatalf("expected 1 topic, got %d", len(test6.Topics[1])) } - if test6.Topics[1][0] != nullTopic { + if test6.Topics[1][0] != nil { t.Fatalf("got %x, expected empty hash", test6.Topics[1][0]) } if len(test6.Topics[2]) != 1 { t.Fatalf("expected 1 topic, got %d", len(test6.Topics[2])) } - if test6.Topics[2][0] != topic2 { + if *test6.Topics[2][0] != topic2 { t.Fatalf("got %x, expected %x", test6.Topics[2][0], topic2) } @@ -175,7 +174,7 @@ func TestUnmarshalJSONNewFilterArgs(t *testing.T) { if len(test7.Topics[0]) != 2 { t.Fatalf("expected 2 topics, got %d topics", len(test7.Topics[0])) } - if test7.Topics[0][0] != topic0 || test7.Topics[0][1] != topic1 { + if *test7.Topics[0][0] != topic0 || *test7.Topics[0][1] != topic1 { t.Fatalf("invalid topics expected [%x,%x], got [%x,%x]", topic0, topic1, test7.Topics[0][0], test7.Topics[0][1], ) @@ -183,15 +182,15 @@ func TestUnmarshalJSONNewFilterArgs(t *testing.T) { if len(test7.Topics[1]) != 1 { t.Fatalf("expected 1 topic, got %d topics", len(test7.Topics[1])) } - if test7.Topics[1][0] != nullTopic { + if test7.Topics[1][0] != nil { t.Fatalf("expected empty hash, got %x", test7.Topics[1][0]) } if len(test7.Topics[2]) != 2 { t.Fatalf("expected 2 topics, got %d topics", len(test7.Topics[2])) } - if test7.Topics[2][0] != topic2 || test7.Topics[2][1] != nullTopic { - t.Fatalf("invalid topics expected [%x,%x], got [%x,%x]", - topic2, nullTopic, test7.Topics[2][0], test7.Topics[2][1], + if *test7.Topics[2][0] != topic2 || test7.Topics[2][1] != nil { + t.Fatalf("invalid topics expected [%x,nil], got [%x,%x]", + topic2, test7.Topics[2][0], test7.Topics[2][1], ) } } diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 4f6c30058e..6ed3643c7f 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -52,15 +52,17 @@ type Filter struct { db ethdb.Database begin, end int64 addresses []common.Address - topics [][]common.Hash + topics [][]*common.Hash matcher *bloombits.Matcher } // New creates a new filter which uses a bloom filter on blocks to figure out whether // a particular block is interesting or not. -func New(backend Backend, begin, end int64, addresses []common.Address, topics [][]common.Hash) *Filter { - // Flatten the address and topic filter clauses into a single filter system +func New(backend Backend, begin, end int64, addresses []common.Address, topics [][]*common.Hash) *Filter { + // Flatten the address and topic filter clauses into a single bloombits filter + // system. Since the bloombits are not positional, nil topics are permitted, + // which get flattened into a nil byte slice. var filters [][][]byte if len(addresses) > 0 { filter := make([][]byte, len(addresses)) @@ -72,7 +74,9 @@ func New(backend Backend, begin, end int64, addresses []common.Address, topics [ for _, topicList := range topics { filter := make([][]byte, len(topicList)) for i, topic := range topicList { - filter[i] = topic.Bytes() + if topic != nil { + filter[i] = topic.Bytes() + } } filters = append(filters, filter) } @@ -221,7 +225,7 @@ func includes(addresses []common.Address, a common.Address) bool { } // filterLogs creates a slice of logs matching the given criteria. -func filterLogs(logs []*types.Log, fromBlock, toBlock *big.Int, addresses []common.Address, topics [][]common.Hash) []*types.Log { +func filterLogs(logs []*types.Log, fromBlock, toBlock *big.Int, addresses []common.Address, topics [][]*common.Hash) []*types.Log { var ret []*types.Log Logs: for _, log := range logs { @@ -235,10 +239,6 @@ Logs: if len(addresses) > 0 && !includes(addresses, log.Address) { continue } - - logTopics := make([]common.Hash, len(topics)) - copy(logTopics, log.Topics) - // If the to filtered topics is greater than the amount of topics in logs, skip. if len(topics) > len(log.Topics) { continue Logs @@ -247,24 +247,21 @@ Logs: for i, topics := range topics { var match bool for _, topic := range topics { - // common.Hash{} is a match all (wildcard) - if (topic == common.Hash{}) || log.Topics[i] == topic { + if topic == nil || log.Topics[i] == *topic { // nil topic is a match all (wildcard) match = true break } } - if !match { continue Logs } } ret = append(ret, log) } - return ret } -func bloomFilter(bloom types.Bloom, addresses []common.Address, topics [][]common.Hash) bool { +func bloomFilter(bloom types.Bloom, addresses []common.Address, topics [][]*common.Hash) bool { if len(addresses) > 0 { var included bool for _, addr := range addresses { @@ -273,7 +270,6 @@ func bloomFilter(bloom types.Bloom, addresses []common.Address, topics [][]commo break } } - if !included { return false } @@ -282,7 +278,7 @@ func bloomFilter(bloom types.Bloom, addresses []common.Address, topics [][]commo for _, sub := range topics { var included bool for _, topic := range sub { - if (topic == common.Hash{}) || types.BloomLookup(bloom, topic) { + if topic == nil || types.BloomLookup(bloom, *topic) { // nil topic is a match all (wildcard) included = true break } @@ -291,6 +287,5 @@ func bloomFilter(bloom types.Bloom, addresses []common.Address, topics [][]commo return false } } - return true } diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index 00ade0ffb2..3f4fe8d726 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -212,7 +212,6 @@ func (es *EventSystem) subscribeMinedPendingLogs(crit FilterCriteria, logs chan installed: make(chan struct{}), err: make(chan error), } - return es.subscribe(sub) } @@ -230,7 +229,6 @@ func (es *EventSystem) subscribeLogs(crit FilterCriteria, logs chan []*types.Log installed: make(chan struct{}), err: make(chan error), } - return es.subscribe(sub) } @@ -248,7 +246,6 @@ func (es *EventSystem) subscribePendingLogs(crit FilterCriteria, logs chan []*ty installed: make(chan struct{}), err: make(chan error), } - return es.subscribe(sub) } @@ -265,7 +262,6 @@ func (es *EventSystem) SubscribeNewHeads(headers chan *types.Header) *Subscripti installed: make(chan struct{}), err: make(chan error), } - return es.subscribe(sub) } @@ -282,7 +278,6 @@ func (es *EventSystem) SubscribePendingTxEvents(hashes chan common.Hash) *Subscr installed: make(chan struct{}), err: make(chan error), } - return es.subscribe(sub) } @@ -374,7 +369,7 @@ func (es *EventSystem) lightFilterNewHead(newHeader *types.Header, callBack func } // filter logs of a single header in light client mode -func (es *EventSystem) lightFilterLogs(header *types.Header, addresses []common.Address, topics [][]common.Hash, remove bool) []*types.Log { +func (es *EventSystem) lightFilterLogs(header *types.Header, addresses []common.Address, topics [][]*common.Hash, remove bool) []*types.Log { if bloomFilter(header.Bloom, addresses, topics) { // Get the logs of the block ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 664ce07a5f..7e86cc7be6 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -363,15 +363,15 @@ func TestLogFilter(t *testing.T) { // match all 0: {FilterCriteria{}, allLogs, ""}, // match none due to no matching addresses - 1: {FilterCriteria{Addresses: []common.Address{{}, notUsedAddress}, Topics: [][]common.Hash{allLogs[0].Topics}}, []*types.Log{}, ""}, + 1: {FilterCriteria{Addresses: []common.Address{{}, notUsedAddress}, Topics: [][]*common.Hash{nil}}, []*types.Log{}, ""}, // match logs based on addresses, ignore topics 2: {FilterCriteria{Addresses: []common.Address{firstAddr}}, allLogs[:2], ""}, // match none due to no matching topics (match with address) - 3: {FilterCriteria{Addresses: []common.Address{secondAddr}, Topics: [][]common.Hash{{notUsedTopic}}}, []*types.Log{}, ""}, + 3: {FilterCriteria{Addresses: []common.Address{secondAddr}, Topics: [][]*common.Hash{{¬UsedTopic}}}, []*types.Log{}, ""}, // match logs based on addresses and topics - 4: {FilterCriteria{Addresses: []common.Address{thirdAddress}, Topics: [][]common.Hash{{firstTopic, secondTopic}}}, allLogs[3:5], ""}, + 4: {FilterCriteria{Addresses: []common.Address{thirdAddress}, Topics: [][]*common.Hash{{&firstTopic, &secondTopic}}}, allLogs[3:5], ""}, // match logs based on multiple addresses and "or" topics - 5: {FilterCriteria{Addresses: []common.Address{secondAddr, thirdAddress}, Topics: [][]common.Hash{{firstTopic, secondTopic}}}, allLogs[2:5], ""}, + 5: {FilterCriteria{Addresses: []common.Address{secondAddr, thirdAddress}, Topics: [][]*common.Hash{{&firstTopic, &secondTopic}}}, allLogs[2:5], ""}, // logs in the pending block 6: {FilterCriteria{Addresses: []common.Address{firstAddr}, FromBlock: big.NewInt(rpc.PendingBlockNumber.Int64()), ToBlock: big.NewInt(rpc.PendingBlockNumber.Int64())}, allLogs[:2], ""}, // mined logs with block num >= 2 or pending logs @@ -381,9 +381,11 @@ func TestLogFilter(t *testing.T) { // all "mined" logs 9: {FilterCriteria{ToBlock: big.NewInt(rpc.LatestBlockNumber.Int64())}, allLogs, ""}, // all "mined" logs with 1>= block num <=2 and topic secondTopic - 10: {FilterCriteria{FromBlock: big.NewInt(1), ToBlock: big.NewInt(2), Topics: [][]common.Hash{{secondTopic}}}, allLogs[3:4], ""}, + 10: {FilterCriteria{FromBlock: big.NewInt(1), ToBlock: big.NewInt(2), Topics: [][]*common.Hash{{&secondTopic}}}, allLogs[3:4], ""}, // all "mined" and pending logs with topic firstTopic - 11: {FilterCriteria{FromBlock: big.NewInt(rpc.LatestBlockNumber.Int64()), ToBlock: big.NewInt(rpc.PendingBlockNumber.Int64()), Topics: [][]common.Hash{{firstTopic}}}, expectedCase11, ""}, + 11: {FilterCriteria{FromBlock: big.NewInt(rpc.LatestBlockNumber.Int64()), ToBlock: big.NewInt(rpc.PendingBlockNumber.Int64()), Topics: [][]*common.Hash{{&firstTopic}}}, expectedCase11, ""}, + // match all logs due to wildcard topic + 12: {FilterCriteria{Topics: [][]*common.Hash{{nil}}}, allLogs[1:], ""}, } ) @@ -493,19 +495,19 @@ func TestPendingLogsSubscription(t *testing.T) { // match all {FilterCriteria{}, convertLogs(allLogs), nil, nil}, // match none due to no matching addresses - {FilterCriteria{Addresses: []common.Address{{}, notUsedAddress}, Topics: [][]common.Hash{{}}}, []*types.Log{}, nil, nil}, + {FilterCriteria{Addresses: []common.Address{{}, notUsedAddress}, Topics: [][]*common.Hash{nil}}, []*types.Log{}, nil, nil}, // match logs based on addresses, ignore topics {FilterCriteria{Addresses: []common.Address{firstAddr}}, append(convertLogs(allLogs[:2]), allLogs[5].Logs[3]), nil, nil}, // match none due to no matching topics (match with address) - {FilterCriteria{Addresses: []common.Address{secondAddr}, Topics: [][]common.Hash{{notUsedTopic}}}, []*types.Log{}, nil, nil}, + {FilterCriteria{Addresses: []common.Address{secondAddr}, Topics: [][]*common.Hash{{¬UsedTopic}}}, []*types.Log{}, nil, nil}, // match logs based on addresses and topics - {FilterCriteria{Addresses: []common.Address{thirdAddress}, Topics: [][]common.Hash{{firstTopic, secondTopic}}}, append(convertLogs(allLogs[3:5]), allLogs[5].Logs[0]), nil, nil}, + {FilterCriteria{Addresses: []common.Address{thirdAddress}, Topics: [][]*common.Hash{{&firstTopic, &secondTopic}}}, append(convertLogs(allLogs[3:5]), allLogs[5].Logs[0]), nil, nil}, // match logs based on multiple addresses and "or" topics - {FilterCriteria{Addresses: []common.Address{secondAddr, thirdAddress}, Topics: [][]common.Hash{{firstTopic, secondTopic}}}, append(convertLogs(allLogs[2:5]), allLogs[5].Logs[0]), nil, nil}, + {FilterCriteria{Addresses: []common.Address{secondAddr, thirdAddress}, Topics: [][]*common.Hash{{&firstTopic, &secondTopic}}}, append(convertLogs(allLogs[2:5]), allLogs[5].Logs[0]), nil, nil}, // block numbers are ignored for filters created with New***Filter, these return all logs that match the given criteria when the state changes {FilterCriteria{Addresses: []common.Address{firstAddr}, FromBlock: big.NewInt(2), ToBlock: big.NewInt(3)}, append(convertLogs(allLogs[:2]), allLogs[5].Logs[3]), nil, nil}, // multiple pending logs, should match only 2 topics from the logs in block 5 - {FilterCriteria{Addresses: []common.Address{thirdAddress}, Topics: [][]common.Hash{{firstTopic, forthTopic}}}, []*types.Log{allLogs[5].Logs[0], allLogs[5].Logs[2]}, nil, nil}, + {FilterCriteria{Addresses: []common.Address{thirdAddress}, Topics: [][]*common.Hash{{&firstTopic, &forthTopic}}}, []*types.Log{allLogs[5].Logs[0], allLogs[5].Logs[2]}, nil, nil}, } ) diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index 11235e95a7..fcb4fe7dc0 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -185,14 +185,14 @@ func TestFilters(t *testing.T) { } } - filter := New(backend, 0, -1, []common.Address{addr}, [][]common.Hash{{hash1, hash2, hash3, hash4}}) + filter := New(backend, 0, -1, []common.Address{addr}, [][]*common.Hash{{&hash1, &hash2, &hash3, &hash4}}) logs, _ := filter.Logs(context.Background()) if len(logs) != 4 { t.Error("expected 4 log, got", len(logs)) } - filter = New(backend, 900, 999, []common.Address{addr}, [][]common.Hash{{hash3}}) + filter = New(backend, 900, 999, []common.Address{addr}, [][]*common.Hash{{&hash3}}) logs, _ = filter.Logs(context.Background()) if len(logs) != 1 { t.Error("expected 1 log, got", len(logs)) @@ -201,7 +201,7 @@ func TestFilters(t *testing.T) { t.Errorf("expected log[0].Topics[0] to be %x, got %x", hash3, logs[0].Topics[0]) } - filter = New(backend, 990, -1, []common.Address{addr}, [][]common.Hash{{hash3}}) + filter = New(backend, 990, -1, []common.Address{addr}, [][]*common.Hash{{&hash3}}) logs, _ = filter.Logs(context.Background()) if len(logs) != 1 { t.Error("expected 1 log, got", len(logs)) @@ -210,7 +210,7 @@ func TestFilters(t *testing.T) { t.Errorf("expected log[0].Topics[0] to be %x, got %x", hash3, logs[0].Topics[0]) } - filter = New(backend, 1, 10, nil, [][]common.Hash{{hash1, hash2}}) + filter = New(backend, 1, 10, nil, [][]*common.Hash{{&hash1, &hash2}}) logs, _ = filter.Logs(context.Background()) if len(logs) != 2 { @@ -218,7 +218,7 @@ func TestFilters(t *testing.T) { } failHash := common.BytesToHash([]byte("fail")) - filter = New(backend, 0, -1, nil, [][]common.Hash{{failHash}}) + filter = New(backend, 0, -1, nil, [][]*common.Hash{{&failHash}}) logs, _ = filter.Logs(context.Background()) if len(logs) != 0 { @@ -233,7 +233,7 @@ func TestFilters(t *testing.T) { t.Error("expected 0 log, got", len(logs)) } - filter = New(backend, 0, -1, nil, [][]common.Hash{{failHash}, {hash1}}) + filter = New(backend, 0, -1, nil, [][]*common.Hash{{&failHash}, {&hash1}}) logs, _ = filter.Logs(context.Background()) if len(logs) != 0 {