Merge 4f2b101d27 into 07aae19e5d

core/blockchain.go

@@ -104,6 +104,8 @@ type BlockChain struct {
 	procInterrupt int32          // interrupt signaler for block processing
 	wg            sync.WaitGroup // chain processing wait group for shutting down
 
+	chainProcessors []ChainProcessor // NewHead function called under chain lock after a new head has been written to the db
+
 	engine    consensus.Engine
 	processor Processor // block processor interface
 	validator Validator // block and state validator interface
@@ -168,6 +170,15 @@ func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, engine co
 	return bc, nil
 }
 
+// AddChainProcessor adds a new head callback function that will be
+// called under chain lock after a new head has been written to the db
+func (self *BlockChain) AddChainProcessor(cp ChainProcessor) {
+	self.chainmu.Lock()
+	self.chainProcessors = append(self.chainProcessors, cp)
+	cp.NewHead(self.hc.CurrentHeader().Number.Uint64(), false)
+	self.chainmu.Unlock()
+}
+
 func (self *BlockChain) getProcInterrupt() bool {
 	return atomic.LoadInt32(&self.procInterrupt) == 1
 }
@@ -243,6 +254,7 @@ func (bc *BlockChain) SetHead(head uint64) error {
 	bc.mu.Lock()
 	defer bc.mu.Unlock()
 
+	oldHead := bc.hc.CurrentHeader().Number.Uint64()
 	// Rewind the header chain, deleting all block bodies until then
 	delFn := func(hash common.Hash, num uint64) {
 		DeleteBody(bc.chainDb, hash, num)
@@ -256,6 +268,11 @@ func (bc *BlockChain) SetHead(head uint64) error {
 	bc.blockCache.Purge()
 	bc.futureBlocks.Purge()
 
+	if head < oldHead {
+		for _, cp := range bc.chainProcessors {
+			cp.NewHead(head, true)
+		}
+	}
 	// Rewind the block chain, ensuring we don't end up with a stateless head block
 	if bc.currentBlock != nil && currentHeader.Number.Uint64() < bc.currentBlock.NumberU64() {
 		bc.currentBlock = bc.GetBlock(currentHeader.Hash(), currentHeader.Number.Uint64())
@@ -637,12 +654,15 @@ func (self *BlockChain) Rollback(chain []common.Hash) {
 	self.mu.Lock()
 	defer self.mu.Unlock()
 
+	var rollbackHead *types.Header
+
 	for i := len(chain) - 1; i >= 0; i-- {
 		hash := chain[i]
 
 		currentHeader := self.hc.CurrentHeader()
 		if currentHeader.Hash() == hash {
-			self.hc.SetCurrentHeader(self.GetHeader(currentHeader.ParentHash, currentHeader.Number.Uint64()-1))
+			rollbackHead = self.GetHeader(currentHeader.ParentHash, currentHeader.Number.Uint64()-1)
+			self.hc.SetCurrentHeader(rollbackHead)
 		}
 		if self.currentFastBlock.Hash() == hash {
 			self.currentFastBlock = self.GetBlock(self.currentFastBlock.ParentHash(), self.currentFastBlock.NumberU64()-1)
@@ -653,6 +673,12 @@ func (self *BlockChain) Rollback(chain []common.Hash) {
 			WriteHeadBlockHash(self.chainDb, self.currentBlock.Hash())
 		}
 	}
+
+	if rollbackHead != nil {
+		for _, cp := range self.chainProcessors {
+			cp.NewHead(rollbackHead.Number.Uint64(), true)
+		}
+	}
 }
 
 // SetReceiptsData computes all the non-consensus fields of the receipts
@@ -757,12 +783,6 @@ func (self *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain
 				log.Crit("Failed to write block receipts", "err", err)
 				return
 			}
-			if err := WriteMipmapBloom(self.chainDb, block.NumberU64(), receipts); err != nil {
-				errs[index] = fmt.Errorf("failed to write log blooms: %v", err)
-				atomic.AddInt32(&failed, 1)
-				log.Crit("Failed to write log blooms", "err", err)
-				return
-			}
 			if err := WriteTransactions(self.chainDb, block); err != nil {
 				errs[index] = fmt.Errorf("failed to write individual transactions: %v", err)
 				atomic.AddInt32(&failed, 1)
@@ -1015,10 +1035,6 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) {
 			if err := WriteReceipts(self.chainDb, receipts); err != nil {
 				return i, err
 			}
-			// Write map map bloom filters
-			if err := WriteMipmapBloom(self.chainDb, block.NumberU64(), receipts); err != nil {
-				return i, err
-			}
 			// Write hash preimages
 			if err := WritePreimages(self.chainDb, block.NumberU64(), self.stateCache.Preimages()); err != nil {
 				return i, err
@@ -1181,10 +1197,6 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error {
 		if err := WriteReceipts(self.chainDb, receipts); err != nil {
 			return err
 		}
-		// Write map map bloom filters
-		if err := WriteMipmapBloom(self.chainDb, block.NumberU64(), receipts); err != nil {
-			return err
-		}
 		addedTxs = append(addedTxs, block.Transactions()...)
 	}
 
@@ -1316,7 +1328,16 @@ func (self *BlockChain) InsertHeaderChain(chain []*types.Header, checkFreq int)
 		self.mu.Lock()
 		defer self.mu.Unlock()
 
-		_, err := self.hc.WriteHeader(header)
+		_, err := self.hc.WriteHeader(header, func(head *types.Header) {
+			for _, cp := range self.chainProcessors {
+				cp.NewHead(head.Number.Uint64(), true)
+			}
+		})
+
+		for _, cp := range self.chainProcessors {
+			cp.NewHead(header.Number.Uint64(), false)
+		}
+
 		return err
 	}
 
@@ -1339,7 +1360,7 @@ func (self *BlockChain) writeHeader(header *types.Header) error {
 	self.mu.Lock()
 	defer self.mu.Unlock()
 
-	_, err := self.hc.WriteHeader(header)
+	_, err := self.hc.WriteHeader(header, nil)
 	return err
 }
 

core/bloombits/fetcher_test.go

@@ -0,0 +1,101 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+package bloombits
+
+import (
+	"bytes"
+	"encoding/binary"
+	"math/rand"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+const testFetcherReqCount = 50000
+
+func fetcherTestVector(b uint, s uint64) []byte {
+	r := make([]byte, 10)
+	binary.BigEndian.PutUint16(r[0:2], uint16(b))
+	binary.BigEndian.PutUint64(r[2:10], s)
+	return r
+}
+
+func TestFetcher(t *testing.T) {
+	testFetcher(t, 1)
+}
+
+func TestFetcherMultipleReaders(t *testing.T) {
+	testFetcher(t, 10)
+}
+
+func testFetcher(t *testing.T, cnt int) {
+	f := &fetcher{
+		reqMap: make(map[uint64]req),
+	}
+	distCh := make(chan distReq, channelCap)
+	stop := make(chan struct{})
+	var reqCount uint32
+
+	for i := 0; i < 10; i++ {
+		go func() {
+			for {
+				req, ok := <-distCh
+				if !ok {
+					return
+				}
+				time.Sleep(time.Duration(rand.Intn(1000000)))
+				atomic.AddUint32(&reqCount, 1)
+				f.deliver([]uint64{req.sectionIdx}, [][]byte{fetcherTestVector(req.bitIdx, req.sectionIdx)})
+			}
+		}()
+	}
+
+	var wg, wg2 sync.WaitGroup
+	for cc := 0; cc < cnt; cc++ {
+		wg.Add(1)
+		in := make(chan uint64, channelCap)
+		out := f.fetch(in, distCh, stop, &wg2)
+
+		time.Sleep(time.Millisecond * 100 * time.Duration(cc))
+		go func() {
+			for i := uint64(0); i < testFetcherReqCount; i++ {
+				in <- i
+			}
+		}()
+
+		go func() {
+			for i := uint64(0); i < testFetcherReqCount; i++ {
+				bv := <-out
+				if !bytes.Equal(bv, fetcherTestVector(0, i)) {
+					if len(bv) != 10 {
+						t.Errorf("Vector #%d length is %d, expected 10", i, len(bv))
+					} else {
+						j := binary.BigEndian.Uint64(bv[2:10])
+						t.Errorf("Expected vector #%d, fetched #%d", i, j)
+					}
+				}
+			}
+			wg.Done()
+		}()
+	}
+
+	wg.Wait()
+	close(stop)
+	if reqCount != testFetcherReqCount {
+		t.Errorf("Request count mismatch: expected %v, got %v", testFetcherReqCount, reqCount)
+	}
+}

core/bloombits/matcher.go

@@ -0,0 +1,559 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+package bloombits
+
+import (
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/bitutil"
+	"github.com/ethereum/go-ethereum/core/types"
+)
+
+const (
+	maxRequestLength = 16
+	channelCap       = 100
+)
+
+// fetcher handles bit vector retrieval pipelines for a single bit index
+type fetcher struct {
+	bitIdx  uint
+	reqMap  map[uint64]req
+	reqLock sync.RWMutex
+}
+
+type req struct {
+	data    []byte
+	queued  bool
+	fetched chan struct{}
+}
+
+type distReq struct {
+	bitIdx     uint
+	sectionIdx uint64
+}
+
+// fetch creates a retrieval pipeline, receiving section indexes from sectionCh and returning the results
+// in the same order through the returned channel. Multiple fetch instances of the same fetcher are allowed
+// to run in parallel, in case the same bit index appears multiple times in the filter structure. Each section
+// is requested only once, requests are sent to the request distributor (part of Matcher) through distCh.
+func (f *fetcher) fetch(sectionCh chan uint64, distCh chan distReq, stop chan struct{}, wg *sync.WaitGroup) chan []byte {
+	dataCh := make(chan []byte, channelCap)
+	returnCh := make(chan uint64, channelCap)
+	wg.Add(2)
+
+	go func() {
+		defer wg.Done()
+		defer close(returnCh)
+
+		for {
+			select {
+			case <-stop:
+				return
+			case idx, ok := <-sectionCh:
+				if !ok {
+					return
+				}
+
+				req := false
+				f.reqLock.Lock()
+				r := f.reqMap[idx]
+				if r.data == nil {
+					req = !r.queued
+					r.queued = true
+					if r.fetched == nil {
+						r.fetched = make(chan struct{})
+					}
+					f.reqMap[idx] = r
+				}
+				f.reqLock.Unlock()
+				if req {
+					distCh <- distReq{bitIdx: f.bitIdx, sectionIdx: idx} // success is guaranteed, distibuteRequests shuts down after fetch
+				}
+				select {
+				case <-stop:
+					return
+				case returnCh <- idx:
+				}
+			}
+		}
+	}()
+
+	go func() {
+		defer wg.Done()
+		defer close(dataCh)
+
+		for {
+			select {
+			case <-stop:
+				return
+			case idx, ok := <-returnCh:
+				if !ok {
+					return
+				}
+
+				f.reqLock.RLock()
+				r := f.reqMap[idx]
+				f.reqLock.RUnlock()
+
+				if r.data == nil {
+					select {
+					case <-stop:
+						return
+					case <-r.fetched:
+						f.reqLock.RLock()
+						r = f.reqMap[idx]
+						f.reqLock.RUnlock()
+					}
+				}
+				select {
+				case <-stop:
+					return
+				case dataCh <- r.data:
+				}
+			}
+		}
+	}()
+
+	return dataCh
+}
+
+// deliver is called by the request distributor when a reply to a request has
+// arrived
+func (f *fetcher) deliver(sectionIdxList []uint64, data [][]byte) {
+	f.reqLock.Lock()
+	defer f.reqLock.Unlock()
+
+	for i, idx := range sectionIdxList {
+		r := f.reqMap[idx]
+		if r.data != nil {
+			panic("BloomBits section data delivered twice")
+		}
+		r.data = data[i]
+		close(r.fetched)
+		f.reqMap[idx] = r
+	}
+}
+
+// Matcher is a pipelined structure of fetchers and logic matchers which perform
+// binary AND/OR operations on the bitstreams, finally creating a stream of potential matches.
+type Matcher struct {
+	addresses   []types.BloomIndexList
+	topics      [][]types.BloomIndexList
+	fetchers    map[uint]*fetcher
+	sectionSize uint64
+
+	distCh       chan distReq
+	reqs         map[uint][]uint64
+	getNextReqCh chan chan nextRequests
+	wg, distWg   sync.WaitGroup
+}
+
+// NewMatcher creates a new Matcher instance
+func NewMatcher(sectionSize uint64, addresses []common.Address, topics [][]common.Hash) *Matcher {
+	m := &Matcher{fetchers: make(map[uint]*fetcher), reqs: make(map[uint][]uint64), distCh: make(chan distReq, channelCap), sectionSize: sectionSize}
+	m.setAddresses(addresses)
+	m.setTopics(topics)
+	return m
+}
+
+// setAddresses matches only logs that are generated from addresses that are included
+// in the given addresses.
+func (m *Matcher) setAddresses(addr []common.Address) {
+	m.addresses = make([]types.BloomIndexList, len(addr))
+	for i, b := range addr {
+		m.addresses[i] = types.BloomIndexes(b.Bytes())
+	}
+
+	for _, idxs := range m.addresses {
+		for _, idx := range idxs {
+			m.newFetcher(idx)
+		}
+	}
+}
+
+// setTopics matches only logs that have topics matching the given topics.
+func (m *Matcher) setTopics(topics [][]common.Hash) {
+	m.topics = nil
+loop:
+	for _, topicList := range topics {
+		t := make([]types.BloomIndexList, len(topicList))
+		for i, b := range topicList {
+			if (b == common.Hash{}) {
+				continue loop
+			}
+			t[i] = types.BloomIndexes(b.Bytes())
+		}
+		m.topics = append(m.topics, t)
+	}
+
+	for _, idxss := range m.topics {
+		for _, idxs := range idxss {
+			for _, idx := range idxs {
+				m.newFetcher(idx)
+			}
+		}
+	}
+}
+
+// match creates a daisy-chain of sub-matchers, one for the address set and one for each topic set, each
+// sub-matcher receiving a section only if the previous ones have all found a potential match in one of
+// the blocks of the section, then binary AND-ing its own matches and forwaring the result to the next one
+func (m *Matcher) match(processCh chan partialMatches, stop chan struct{}) chan partialMatches {
+	subIdx := m.topics
+	if len(m.addresses) > 0 {
+		subIdx = append([][]types.BloomIndexList{m.addresses}, subIdx...)
+	}
+	m.getNextReqCh = make(chan chan nextRequests) // should be a blocking channel
+	m.distributeRequests(stop)
+
+	for _, idx := range subIdx {
+		processCh = m.subMatch(processCh, idx, stop)
+	}
+	return processCh
+}
+
+// partialMatches with a non-nil vector represents a section in which some sub-matchers have already
+// found potential matches. Subsequent sub-matchers will binary AND their matches with this vector.
+// If vector is nil, it represents a section to be processed by the first sub-matcher.
+type partialMatches struct {
+	sectionIdx uint64
+	vector     []byte
+}
+
+// newFetcher adds a fetcher for the given bit index if it has not existed before
+func (m *Matcher) newFetcher(idx uint) {
+	if _, ok := m.fetchers[idx]; ok {
+		return
+	}
+	f := &fetcher{
+		bitIdx: idx,
+		reqMap: make(map[uint64]req),
+	}
+	m.fetchers[idx] = f
+}
+
+// subMatch creates a sub-matcher that filters for a set of addresses or topics, binary OR-s those matches, then
+// binary AND-s the result to the daisy-chain input (processCh) and forwards it to the daisy-chain output.
+// The matches of each address/topic are calculated by fetching the given sections of the three bloom bit indexes belonging to
+// that address/topic, and binary AND-ing those vectors together.
+func (m *Matcher) subMatch(processCh chan partialMatches, idxs []types.BloomIndexList, stop chan struct{}) chan partialMatches {
+	// set up fetchers
+	fetchIdx := make([][3]chan uint64, len(idxs))
+	fetchData := make([][3]chan []byte, len(idxs))
+	for i, idx := range idxs {
+		for j, ii := range idx {
+			fetchIdx[i][j] = make(chan uint64, channelCap)
+			fetchData[i][j] = m.fetchers[ii].fetch(fetchIdx[i][j], m.distCh, stop, &m.wg)
+		}
+	}
+
+	fetchedCh := make(chan partialMatches, channelCap) // entries from processCh are forwarded here after fetches have been initiated
+	resultsCh := make(chan partialMatches, channelCap)
+
+	m.wg.Add(2)
+	// goroutine for starting retrievals
+	go func() {
+		defer m.wg.Done()
+
+		for {
+			select {
+			case <-stop:
+				return
+			case s, ok := <-processCh:
+				if !ok {
+					close(fetchedCh)
+					for _, ff := range fetchIdx {
+						for _, f := range ff {
+							close(f)
+						}
+					}
+					return
+				}
+
+				for _, ff := range fetchIdx {
+					for _, f := range ff {
+						select {
+						case <-stop:
+							return
+						case f <- s.sectionIdx:
+						}
+					}
+				}
+				select {
+				case <-stop:
+					return
+				case fetchedCh <- s:
+				}
+			}
+		}
+	}()
+
+	// goroutine for processing retrieved data
+	go func() {
+		defer m.wg.Done()
+
+		for {
+			select {
+			case <-stop:
+				return
+			case s, ok := <-fetchedCh:
+				if !ok {
+					close(resultsCh)
+					return
+				}
+
+				var orVector []byte
+				for _, ff := range fetchData {
+					var andVector []byte
+					for _, f := range ff {
+						var data []byte
+						select {
+						case <-stop:
+							return
+						case data = <-f:
+						}
+						if andVector == nil {
+							andVector = make([]byte, int(m.sectionSize/8))
+							copy(andVector, data)
+						} else {
+							bitutil.ANDBytes(andVector, andVector, data)
+						}
+					}
+					if orVector == nil {
+						orVector = andVector
+					} else {
+						bitutil.ORBytes(orVector, orVector, andVector)
+					}
+				}
+
+				if orVector == nil {
+					orVector = make([]byte, int(m.sectionSize/8))
+				}
+				if s.vector != nil {
+					bitutil.ANDBytes(orVector, orVector, s.vector)
+				}
+				if bitutil.TestBytes(orVector) {
+					select {
+					case <-stop:
+						return
+					case resultsCh <- partialMatches{s.sectionIdx, orVector}:
+					}
+				}
+			}
+		}
+	}()
+
+	return resultsCh
+}
+
+// GetMatches returns a stream of bloom matches in a given range of blocks.
+// It returns a results channel immediately and stops if the stop channel is closed or
+// there are no more matches in the range (in which case the results channel is closed).
+// GetMatches can be called multiple times for different ranges, in which case already
+// delivered bit vectors are not requested again.
+func (m *Matcher) GetMatches(start, end uint64, stop chan struct{}) chan uint64 {
+	m.distWg.Wait()
+
+	processCh := make(chan partialMatches, channelCap)
+	resultsCh := make(chan uint64, channelCap)
+
+	res := m.match(processCh, stop)
+
+	startSection := start / m.sectionSize
+	endSection := end / m.sectionSize
+
+	m.wg.Add(2)
+	go func() {
+		defer m.wg.Done()
+		defer close(processCh)
+
+		for i := startSection; i <= endSection; i++ {
+			select {
+			case processCh <- partialMatches{i, nil}:
+			case <-stop:
+				return
+			}
+		}
+	}()
+
+	go func() {
+		defer m.wg.Done()
+		defer close(resultsCh)
+
+		for {
+			select {
+			case r, ok := <-res:
+				if !ok {
+					return
+				}
+				sectionStart := r.sectionIdx * m.sectionSize
+				s := sectionStart
+				if start > s {
+					s = start
+				}
+				e := sectionStart + m.sectionSize - 1
+				if end < e {
+					e = end
+				}
+				for i := s; i <= e; i++ {
+					b := r.vector[(i-sectionStart)/8]
+					bit := 7 - i%8
+					if b != 0 {
+						if b&(1<<bit) != 0 {
+							select {
+							case <-stop:
+								return
+							case resultsCh <- i:
+							}
+						}
+					} else {
+						i += bit
+					}
+				}
+
+			case <-stop:
+				return
+			}
+		}
+	}()
+
+	return resultsCh
+}
+
+type nextRequests struct {
+	bitIdx         uint
+	sectionIdxList []uint64
+}
+
+// distributeRequests receives requests from the fetchers and either queues them
+// or immediately forwards them to one of the waiting NextRequest functions.
+// Requests with a lower section idx are always prioritized.
+func (m *Matcher) distributeRequests(stop chan struct{}) {
+	m.distWg.Add(1)
+	stopDist := make(chan struct{})
+	go func() {
+		<-stop
+		m.wg.Wait()
+		close(stopDist)
+	}()
+
+	go func() {
+		defer m.distWg.Done()
+
+		reqCount := 0
+		for _, s := range m.reqs {
+			reqCount += len(s)
+		}
+
+		storeReq := func(r distReq) {
+			queue := m.reqs[r.bitIdx]
+			i := 0
+			for i < len(queue) && r.sectionIdx > queue[i] {
+				i++
+			}
+			queue = append(queue, 0)
+			copy(queue[i+1:], queue[i:len(queue)-1])
+			queue[i] = r.sectionIdx
+
+			m.reqs[r.bitIdx] = queue
+			reqCount++
+		}
+
+		storeReqs := func(r distReq) {
+			storeReq(r)
+			timeout := time.After(time.Microsecond)
+			for {
+				select {
+				case <-timeout:
+					return
+				case r := <-m.distCh:
+					storeReq(r)
+				case <-stopDist:
+					return
+				}
+			}
+		}
+
+		for {
+			if reqCount == 0 {
+				select {
+				case r := <-m.distCh:
+					storeReqs(r)
+				case <-stopDist:
+					return
+				}
+			} else {
+				select {
+				case r := <-m.distCh:
+					storeReqs(r)
+				case <-stopDist:
+					return
+				case c := <-m.getNextReqCh:
+					var (
+						found       bool
+						bestBit     uint
+						bestSection uint64
+					)
+
+					for bitIdx, queue := range m.reqs {
+						if len(queue) > 0 && (!found || queue[0] < bestSection) {
+							found = true
+							bestBit = bitIdx
+							bestSection = queue[0]
+						}
+					}
+					if !found {
+						panic(nil)
+					}
+
+					bestQueue := m.reqs[bestBit]
+					cnt := len(bestQueue)
+					if cnt > maxRequestLength {
+						cnt = maxRequestLength
+					}
+					res := nextRequests{bestBit, bestQueue[:cnt]}
+					m.reqs[bestBit] = bestQueue[cnt:]
+					reqCount -= cnt
+
+					c <- res
+				}
+			}
+		}
+	}()
+}
+
+// NextRequest asks for a request from the request distributor, returns immediately if there
+// was a queued one, otherwise waits until the distributor forwards one of the requests sent
+// by one of the fetchers.
+func (m *Matcher) NextRequest(stop chan struct{}) (bitIdx uint, sectionIdxList []uint64) {
+	c := make(chan nextRequests)
+	select {
+	case m.getNextReqCh <- c:
+		r := <-c
+		return r.bitIdx, r.sectionIdxList
+	case <-stop:
+		return 0, nil
+	}
+}
+
+// Deliver delivers a bit vector to the appropriate fetcher.
+// It is possible to deliver data even after GetMatches has been stopped. Once a vector has been
+// requested, the next call to GetMatches will keep waiting for delivery.
+func (m *Matcher) Deliver(bitIdx uint, sectionIdxList []uint64, data [][]byte) {
+	m.fetchers[bitIdx].deliver(sectionIdxList, data)
+}

core/bloombits/matcher_test.go

@@ -0,0 +1,180 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+package bloombits
+
+import (
+	"math/rand"
+	"sync/atomic"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/core/types"
+)
+
+const testSectionSize = 4096
+
+func matcherTestVector(b uint, s uint64) []byte {
+	r := make([]byte, testSectionSize/8)
+	for i, _ := range r {
+		var bb byte
+		for bit := 0; bit < 8; bit++ {
+			blockIdx := s*testSectionSize + uint64(i*8+bit)
+			bb += bb
+			if (blockIdx % uint64(b)) == 0 {
+				bb++
+			}
+		}
+		r[i] = bb
+	}
+	return r
+}
+
+func expMatch1(idxs types.BloomIndexList, i uint64) bool {
+	for _, ii := range idxs {
+		if (i % uint64(ii)) != 0 {
+			return false
+		}
+	}
+	return true
+}
+
+func expMatch2(idxs []types.BloomIndexList, i uint64) bool {
+	for _, ii := range idxs {
+		if expMatch1(ii, i) {
+			return true
+		}
+	}
+	return false
+}
+
+func expMatch3(idxs [][]types.BloomIndexList, i uint64) bool {
+	for _, ii := range idxs {
+		if !expMatch2(ii, i) {
+			return false
+		}
+	}
+	return true
+}
+
+func testServeMatcher(m *Matcher, stop chan struct{}, cnt *uint32) {
+	// serve matcher with test vectors
+	for i := 0; i < 10; i++ {
+		go func() {
+			for {
+				select {
+				case <-stop:
+					return
+				default:
+				}
+				b, s := m.NextRequest(stop)
+				if s == nil {
+					return
+				}
+				res := make([][]byte, len(s))
+				for i, ss := range s {
+					res[i] = matcherTestVector(b, ss)
+					atomic.AddUint32(cnt, 1)
+				}
+				m.Deliver(b, s, res)
+			}
+		}()
+	}
+}
+
+func testMatcher(t *testing.T, idxs [][]types.BloomIndexList, cnt uint64, stopOnMatches bool, expCount uint32) uint32 {
+	m := NewMatcher(testSectionSize, nil, nil)
+
+	for _, idxss := range idxs {
+		for _, idxs := range idxss {
+			for _, idx := range idxs {
+				m.newFetcher(idx)
+			}
+		}
+	}
+
+	m.addresses = idxs[0]
+	m.topics = idxs[1:]
+	var reqCount uint32
+
+	stop := make(chan struct{})
+	chn := m.GetMatches(0, cnt-1, stop)
+	testServeMatcher(m, stop, &reqCount)
+
+	for i := uint64(0); i < cnt; i++ {
+		if expMatch3(idxs, i) {
+			match, ok := <-chn
+			if !ok {
+				t.Errorf("Error matching idxs = %v  count = %v  stopOnMatches = %v: expected #%v, results channel closed", idxs, cnt, stopOnMatches, i)
+				return 0
+			}
+			if match != i {
+				t.Errorf("Error matching idxs = %v  count = %v  stopOnMatches = %v: expected #%v, got #%v", idxs, cnt, stopOnMatches, i, match)
+			}
+			if stopOnMatches {
+				close(stop)
+				stop = make(chan struct{})
+				chn = m.GetMatches(i+1, cnt-1, stop)
+				testServeMatcher(m, stop, &reqCount)
+			}
+		}
+	}
+	match, ok := <-chn
+	if ok {
+		t.Errorf("Error matching idxs = %v  count = %v  stopOnMatches = %v: expected closed channel, got #%v", idxs, cnt, stopOnMatches, match)
+	}
+	close(stop)
+
+	if expCount != 0 && expCount != reqCount {
+		t.Errorf("Error matching idxs = %v  count = %v  stopOnMatches = %v: request count mismatch, expected #%v, got #%v", idxs, cnt, stopOnMatches, expCount, reqCount)
+	}
+
+	return reqCount
+}
+
+func testRandomIdxs(l []int, max int) [][]types.BloomIndexList {
+	res := make([][]types.BloomIndexList, len(l))
+	for i, ll := range l {
+		res[i] = make([]types.BloomIndexList, ll)
+		for j, _ := range res[i] {
+			for k, _ := range res[i][j] {
+				res[i][j][k] = uint(rand.Intn(max-1) + 2)
+			}
+		}
+	}
+	return res
+}
+
+func TestMatcher(t *testing.T) {
+	testMatcher(t, [][]types.BloomIndexList{{{10, 20, 30}}}, 1000000, false, 735)
+	testMatcher(t, [][]types.BloomIndexList{{{32, 3125, 100}}, {{40, 50, 10}}}, 1000000, false, 768)
+	testMatcher(t, [][]types.BloomIndexList{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 100000, false, 300)
+}
+
+func TestMatcherStopOnMatches(t *testing.T) {
+	testMatcher(t, [][]types.BloomIndexList{{{10, 20, 30}}}, 1000000, true, 735)
+	testMatcher(t, [][]types.BloomIndexList{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 100000, true, 300)
+}
+
+func TestMatcherRandom(t *testing.T) {
+	for i := 0; i < 20; i++ {
+		testMatcher(t, testRandomIdxs([]int{1}, 50), 1000000, false, 0)
+		testMatcher(t, testRandomIdxs([]int{3}, 50), 1000000, false, 0)
+		testMatcher(t, testRandomIdxs([]int{2, 2, 2}, 20), 1000000, false, 0)
+		testMatcher(t, testRandomIdxs([]int{5, 5, 5}, 50), 1000000, false, 0)
+		idxs := testRandomIdxs([]int{2, 2, 2}, 20)
+		reqCount := testMatcher(t, idxs, 100000, false, 0)
+		testMatcher(t, idxs, 100000, true, reqCount)
+	}
+}

core/bloombits/utils.go

@@ -0,0 +1,63 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+package bloombits
+
+import (
+	"github.com/ethereum/go-ethereum/core/types"
+)
+
+const BloomLength = 2048
+
+// BloomBitsCreator takes SectionSize number of header bloom filters and calculates the bloomBits vectors of the section
+type BloomBitsCreator struct {
+	blooms              [BloomLength][]byte
+	sectionSize, bitIdx uint64
+}
+
+func NewBloomBitsCreator(sectionSize uint64) *BloomBitsCreator {
+	b := &BloomBitsCreator{sectionSize: sectionSize}
+	for i, _ := range b.blooms {
+		b.blooms[i] = make([]byte, sectionSize/8)
+	}
+	return b
+}
+
+// AddHeaderBloom takes a single bloom filter and sets the corresponding bit column in memory accordingly
+func (b *BloomBitsCreator) AddHeaderBloom(bloom types.Bloom) {
+	if b.bitIdx >= b.sectionSize {
+		panic("too many header blooms added")
+	}
+
+	byteIdx := b.bitIdx / 8
+	bitMask := byte(1) << byte(7-b.bitIdx%8)
+	for bloomBitIdx, _ := range b.blooms {
+		bloomByteIdx := BloomLength/8 - 1 - bloomBitIdx/8
+		bloomBitMask := byte(1) << byte(bloomBitIdx%8)
+		if (bloom[bloomByteIdx] & bloomBitMask) != 0 {
+			b.blooms[bloomBitIdx][byteIdx] |= bitMask
+		}
+	}
+	b.bitIdx++
+}
+
+// GetBitVector returns the bit vector belonging to the given bit index after header blooms have been added
+func (b *BloomBitsCreator) GetBitVector(idx uint) []byte {
+	if b.bitIdx != b.sectionSize {
+		panic("not enough header blooms added")
+	}
+
+	return b.blooms[idx][:]
+}

core/chain_processor.go

@@ -0,0 +1,192 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package core implements the Ethereum consensus protocol.
+package core
+
+import (
+	"sync"
+	"time"
+)
+
+// ChainProcessor is an external process that creates auxiliary data structures
+// using the canonical chain as an input. Its callback function NewHead is called every
+// time a new head is added to the chain or it gets rolled back. The callback function
+// should never block because it is called under the chain's mutex lock so that the
+// structures can stay consistent with the canonical chain. It should also not call
+// any chain functions that use this mutex because that would cause a deadlock.
+// The same interface can be used for chaining processors (one using the output of the
+// other).
+type ChainProcessor interface {
+	NewHead(headNum uint64, rollBack bool)
+}
+
+// ChainSectionProcessor is a ChainProcessor that does a post-processing job for
+// equally sized sections of the canonical chain (like BlooomBits and CHT structures).
+// Further child ChainProcessors can be added which use the output of this section
+// processor. These child processors receive NewHead calls only after an entire section
+// has been finished or in case of rollbacks that might affect already finished sections.
+type ChainSectionProcessor struct {
+	backend                        ChainSectionProcessorBackend
+	sectionSize, confirmReq        uint64
+	stop                           chan struct{}
+	updateCh                       chan uint64
+	lock                           sync.Mutex
+	calcValid                      bool
+	procWait                       time.Duration
+	stored, calcIdx, lastForwarded uint64
+	childProcessors                []ChainProcessor
+}
+
+// ChainSectionProcessorBackend does the actual post-processing job.
+// GetStored and SetStored are called under the canonical chain lock and should
+// not block. Process is called without a chain lock and may take longer to
+// finish. If the chain gets rolled back while Process is running, the results
+// are considered invalid and SetStored is not called.
+type ChainSectionProcessorBackend interface {
+	Process(idx uint64) bool    // do the processing of section idx but do not mark it as valid yet; return false if processing failed
+	GetStored() uint64          // return the number of sections processed, stored and marked as valid
+	SetStored(count uint64)     // set the number of stored and valid processed sections (called after Process returned true if no reorg happened in the meantime)
+	UpdateMsg(done, all uint64) // print a progress update message if necessary (only called when multiple sections need to be processed)
+}
+
+// NewChainSectionProcessor creates a new  ChainSectionProcessor
+//  backend:		an implementation of ChainSectionProcessorBackend
+//  sectionSize:	the size of processable sections
+//  confirmReq:		required number of confirmation blocks before a new section is being processed
+//  procWait:		waiting time between processing sections (simple way of limiting the resource usage of a db upgrade)
+//  stop:		    quit channel
+func NewChainSectionProcessor(backend ChainSectionProcessorBackend, sectionSize, confirmReq uint64, procWait time.Duration, stop chan struct{}) *ChainSectionProcessor {
+	csp := &ChainSectionProcessor{
+		backend:     backend,
+		sectionSize: sectionSize,
+		confirmReq:  confirmReq,
+		stop:        stop,
+		procWait:    procWait,
+		updateCh:    make(chan uint64, 100),
+		stored:      backend.GetStored(),
+	}
+	go csp.updateLoop()
+	return csp
+}
+
+func (csp *ChainSectionProcessor) updateLoop() {
+	tryUpdate := make(chan struct{}, 1)
+	updating := false
+	var targetCount uint64
+	updateMsg := false
+
+	for {
+		select {
+		case <-csp.stop:
+			return
+		case targetCount = <-csp.updateCh:
+			if !updating {
+				updating = true
+				tryUpdate <- struct{}{}
+			}
+		case <-tryUpdate:
+			csp.lock.Lock()
+			if targetCount > csp.stored {
+				if !updateMsg && targetCount > csp.stored+1 {
+					updateMsg = true
+					csp.backend.UpdateMsg(csp.stored, targetCount)
+				}
+				csp.calcValid = true
+				csp.calcIdx = csp.stored
+
+				csp.lock.Unlock()
+				ok := csp.backend.Process(csp.calcIdx)
+				csp.lock.Lock()
+
+				if ok && csp.calcValid {
+					csp.stored = csp.calcIdx + 1
+					csp.backend.SetStored(csp.stored)
+					if updateMsg {
+						csp.backend.UpdateMsg(csp.stored, targetCount)
+						if csp.stored >= targetCount {
+							updateMsg = false
+						}
+					}
+					csp.lastForwarded = csp.stored*csp.sectionSize - 1
+					for _, cp := range csp.childProcessors {
+						cp.NewHead(csp.lastForwarded, false)
+					}
+				}
+				csp.calcValid = false
+			}
+			stored := csp.stored
+			csp.lock.Unlock()
+
+			if targetCount > stored {
+				go func() {
+					time.Sleep(csp.procWait)
+					tryUpdate <- struct{}{}
+				}()
+			} else {
+				updating = false
+			}
+		}
+	}
+}
+
+// NewHead implements the ChainProcessor interface
+func (csp *ChainSectionProcessor) NewHead(headNum uint64, rollback bool) {
+	csp.lock.Lock()
+	defer csp.lock.Unlock()
+
+	if rollback {
+		firstChanged := headNum / csp.sectionSize
+		if firstChanged <= csp.calcIdx {
+			csp.calcValid = false
+		}
+		if firstChanged < csp.stored {
+			csp.stored = firstChanged
+			csp.backend.SetStored(csp.stored)
+			select {
+			case <-csp.stop:
+			case csp.updateCh <- firstChanged:
+			}
+		}
+
+		if headNum < csp.lastForwarded {
+			csp.lastForwarded = headNum
+			for _, cp := range csp.childProcessors {
+				cp.NewHead(csp.lastForwarded, true)
+			}
+		}
+
+	} else {
+		var newCount uint64
+		if headNum >= csp.confirmReq {
+			newCount = (headNum + 1 - csp.confirmReq) / csp.sectionSize
+			if newCount > csp.stored {
+				go func() {
+					select {
+					case <-csp.stop:
+					case csp.updateCh <- newCount:
+					}
+				}()
+			}
+		}
+	}
+}
+
+// AddChildProcessor adds a child ChainProcessor that can use the output of this
+// section processor.
+func (csp *ChainSectionProcessor) AddChildProcessor(cp ChainProcessor) {
+	csp.childProcessors = append(csp.childProcessors, cp)
+}

core/database_util.go

@@ -23,7 +23,6 @@ import (
 	"errors"
 	"fmt"
 	"math/big"
-	"sync"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -50,9 +49,6 @@ var (
 	txMetaSuffix   = []byte{0x01}
 	receiptsPrefix = []byte("receipts-")
 
-	mipmapPre    = []byte("mipmap-log-bloom-")
-	MIPMapLevels = []uint64{1000000, 500000, 100000, 50000, 1000}
-
 	configPrefix = []byte("ethereum-config-") // config prefix for the db
 
 	// used by old (non-sequential keys) db, now only used for conversion
@@ -66,10 +62,11 @@ var (
 
 	ChainConfigNotFoundErr = errors.New("ChainConfig not found") // general config not found error
 
-	mipmapBloomMu sync.Mutex // protect against race condition when updating mipmap blooms
-
 	preimageCounter    = metrics.NewCounter("db/preimage/total")
 	preimageHitCounter = metrics.NewCounter("db/preimage/hits")
+
+	bloomBitsPrefix   = []byte("bloomBits-")
+	bloomBitsAvailKey = []byte("bloomBitsAvailable")
 )
 
 // encodeBlockNumber encodes a block number as big endian uint64
@@ -535,48 +532,6 @@ func DeleteReceipt(db ethdb.Database, hash common.Hash) {
 	db.Delete(append(receiptsPrefix, hash.Bytes()...))
 }
 
-// returns a formatted MIP mapped key by adding prefix, canonical number and level
-//
-// ex. fn(98, 1000) = (prefix || 1000 || 0)
-func mipmapKey(num, level uint64) []byte {
-	lkey := make([]byte, 8)
-	binary.BigEndian.PutUint64(lkey, level)
-	key := new(big.Int).SetUint64(num / level * level)
-
-	return append(mipmapPre, append(lkey, key.Bytes()...)...)
-}
-
-// WriteMapmapBloom writes each address included in the receipts' logs to the
-// MIP bloom bin.
-func WriteMipmapBloom(db ethdb.Database, number uint64, receipts types.Receipts) error {
-	mipmapBloomMu.Lock()
-	defer mipmapBloomMu.Unlock()
-
-	batch := db.NewBatch()
-	for _, level := range MIPMapLevels {
-		key := mipmapKey(number, level)
-		bloomDat, _ := db.Get(key)
-		bloom := types.BytesToBloom(bloomDat)
-		for _, receipt := range receipts {
-			for _, log := range receipt.Logs {
-				bloom.Add(log.Address.Big())
-			}
-		}
-		batch.Put(key, bloom.Bytes())
-	}
-	if err := batch.Write(); err != nil {
-		return fmt.Errorf("mipmap write fail for: %d: %v", number, err)
-	}
-	return nil
-}
-
-// GetMipmapBloom returns a bloom filter using the number and level as input
-// parameters. For available levels see MIPMapLevels.
-func GetMipmapBloom(db ethdb.Database, number, level uint64) types.Bloom {
-	bloomDat, _ := db.Get(mipmapKey(number, level))
-	return types.BytesToBloom(bloomDat)
-}
-
 // PreimageTable returns a Database instance with the key prefix for preimage entries.
 func PreimageTable(db ethdb.Database) ethdb.Database {
 	return ethdb.NewTable(db, preimagePrefix)
@@ -675,3 +630,38 @@ func FindCommonAncestor(db ethdb.Database, a, b *types.Header) *types.Header {
 	}
 	return a
 }
+
+// GetBloomBits reads the compressed bloomBits vector belonging to the given section and bit index from the db
+func GetBloomBits(db ethdb.Database, bitIdx, sectionIdx uint64) ([]byte, error) {
+	var encKey [10]byte
+	binary.BigEndian.PutUint16(encKey[0:2], uint16(bitIdx))
+	binary.BigEndian.PutUint64(encKey[2:10], sectionIdx)
+	key := append(bloomBitsPrefix, encKey[:]...)
+	bloomBits, err := db.Get(key)
+	return bloomBits, err
+}
+
+// StoreBloomBits writes the compressed bloomBits vector belonging to the given section and bit index to the db
+func StoreBloomBits(db ethdb.Database, bitIdx, sectionIdx uint64, bloomBits []byte) {
+	var encKey [10]byte
+	binary.BigEndian.PutUint16(encKey[0:2], uint16(bitIdx))
+	binary.BigEndian.PutUint64(encKey[2:10], sectionIdx)
+	key := append(bloomBitsPrefix, encKey[:]...)
+	db.Put(key, bloomBits)
+}
+
+// GetBloomBitsAvailable reads the number of available bloomBits sections from the db
+func GetBloomBitsAvailable(db ethdb.Database) uint64 {
+	data, _ := db.Get(bloomBitsAvailKey)
+	if len(data) == 8 {
+		return binary.BigEndian.Uint64(data[:])
+	}
+	return 0
+}
+
+// StoreBloomBitsAvailable writes the number of available bloomBits sections to the db
+func StoreBloomBitsAvailable(db ethdb.Database, cnt uint64) {
+	var data [8]byte
+	binary.BigEndian.PutUint64(data[:], cnt)
+	db.Put(bloomBitsAvailKey, data[:])
+}

core/database_util_test.go

@@ -18,17 +18,13 @@ package core
 
 import (
 	"bytes"
-	"io/ioutil"
 	"math/big"
-	"os"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/crypto/sha3"
 	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 )
 
@@ -446,111 +442,3 @@ func TestBlockReceiptStorage(t *testing.T) {
 		t.Fatalf("deleted receipts returned: %v", rs)
 	}
 }
-
-func TestMipmapBloom(t *testing.T) {
-	db, _ := ethdb.NewMemDatabase()
-
-	receipt1 := new(types.Receipt)
-	receipt1.Logs = []*types.Log{
-		{Address: common.BytesToAddress([]byte("test"))},
-		{Address: common.BytesToAddress([]byte("address"))},
-	}
-	receipt2 := new(types.Receipt)
-	receipt2.Logs = []*types.Log{
-		{Address: common.BytesToAddress([]byte("test"))},
-		{Address: common.BytesToAddress([]byte("address1"))},
-	}
-
-	WriteMipmapBloom(db, 1, types.Receipts{receipt1})
-	WriteMipmapBloom(db, 2, types.Receipts{receipt2})
-
-	for _, level := range MIPMapLevels {
-		bloom := GetMipmapBloom(db, 2, level)
-		if !bloom.Test(new(big.Int).SetBytes([]byte("address1"))) {
-			t.Error("expected test to be included on level:", level)
-		}
-	}
-
-	// reset
-	db, _ = ethdb.NewMemDatabase()
-	receipt := new(types.Receipt)
-	receipt.Logs = []*types.Log{
-		{Address: common.BytesToAddress([]byte("test"))},
-	}
-	WriteMipmapBloom(db, 999, types.Receipts{receipt1})
-
-	receipt = new(types.Receipt)
-	receipt.Logs = []*types.Log{
-		{Address: common.BytesToAddress([]byte("test 1"))},
-	}
-	WriteMipmapBloom(db, 1000, types.Receipts{receipt})
-
-	bloom := GetMipmapBloom(db, 1000, 1000)
-	if bloom.TestBytes([]byte("test")) {
-		t.Error("test should not have been included")
-	}
-}
-
-func TestMipmapChain(t *testing.T) {
-	dir, err := ioutil.TempDir("", "mipmap")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(dir)
-
-	var (
-		db, _   = ethdb.NewLDBDatabase(dir, 0, 0)
-		key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
-		addr    = crypto.PubkeyToAddress(key1.PublicKey)
-		addr2   = common.BytesToAddress([]byte("jeff"))
-
-		hash1 = common.BytesToHash([]byte("topic1"))
-	)
-	defer db.Close()
-
-	gspec := &Genesis{
-		Config: params.TestChainConfig,
-		Alloc:  GenesisAlloc{addr: {Balance: big.NewInt(1000000)}},
-	}
-	genesis := gspec.MustCommit(db)
-	chain, receipts := GenerateChain(params.TestChainConfig, genesis, db, 1010, func(i int, gen *BlockGen) {
-		var receipts types.Receipts
-		switch i {
-		case 1:
-			receipt := types.NewReceipt(nil, new(big.Int))
-			receipt.Logs = []*types.Log{{Address: addr, Topics: []common.Hash{hash1}}}
-			gen.AddUncheckedReceipt(receipt)
-			receipts = types.Receipts{receipt}
-		case 1000:
-			receipt := types.NewReceipt(nil, new(big.Int))
-			receipt.Logs = []*types.Log{{Address: addr2}}
-			gen.AddUncheckedReceipt(receipt)
-			receipts = types.Receipts{receipt}
-
-		}
-
-		// store the receipts
-		err := WriteReceipts(db, receipts)
-		if err != nil {
-			t.Fatal(err)
-		}
-		WriteMipmapBloom(db, uint64(i+1), receipts)
-	})
-	for i, block := range chain {
-		WriteBlock(db, block)
-		if err := WriteCanonicalHash(db, block.Hash(), block.NumberU64()); err != nil {
-			t.Fatalf("failed to insert block number: %v", err)
-		}
-		if err := WriteHeadBlockHash(db, block.Hash()); err != nil {
-			t.Fatalf("failed to insert block number: %v", err)
-		}
-		if err := WriteBlockReceipts(db, block.Hash(), block.NumberU64(), receipts[i]); err != nil {
-			t.Fatal("error writing block receipts:", err)
-		}
-	}
-
-	bloom := GetMipmapBloom(db, 0, 1000)
-	if bloom.TestBytes(addr2[:]) {
-		t.Error("address was included in bloom and should not have")
-	}
-}

core/headerchain.go

@@ -119,6 +119,10 @@ func (hc *HeaderChain) GetBlockNumber(hash common.Hash) uint64 {
 	return number
 }
 
+// ReorgCallback is a callback function to be called in case of a chain reorg.
+// The parameter is the header of the last unchanged canonical block.
+type ReorgCallback func(*types.Header)
+
 // WriteHeader writes a header into the local chain, given that its parent is
 // already known. If the total difficulty of the newly inserted header becomes
 // greater than the current known TD, the canonical chain is re-routed.
@@ -128,7 +132,7 @@ func (hc *HeaderChain) GetBlockNumber(hash common.Hash) uint64 {
 // without the real blocks. Hence, writing headers directly should only be done
 // in two scenarios: pure-header mode of operation (light clients), or properly
 // separated header/block phases (non-archive clients).
-func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, err error) {
+func (hc *HeaderChain) WriteHeader(header *types.Header, reorgCallback ReorgCallback) (status WriteStatus, err error) {
 	// Cache some values to prevent constant recalculation
 	var (
 		hash   = header.Hash()
@@ -174,6 +178,11 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er
 			headNumber = headHeader.Number.Uint64() - 1
 			headHeader = hc.GetHeader(headHash, headNumber)
 		}
+
+		if reorgCallback != nil && headNumber < hc.currentHeader.Number.Uint64() {
+			reorgCallback(headHeader)
+		}
+
 		// Extend the canonical chain with the new header
 		if err := WriteCanonicalHash(hc.chainDb, hash, number); err != nil {
 			log.Crit("Failed to insert header number", "err", err)

core/types/bloom9.go

@@ -106,6 +106,20 @@ func LogsBloom(logs []*Log) *big.Int {
 	return bin
 }
 
+type BloomIndexList [3]uint
+
+// BloomIndexes returns the bloom filter bit indexes belonging to the given key
+func BloomIndexes(b []byte) BloomIndexList {
+	b = crypto.Keccak256(b[:])
+
+	var r [3]uint
+	for i, _ := range r {
+		r[i] = (uint(b[i+i+1]) + (uint(b[i+i]) << 8)) & 2047
+	}
+
+	return r
+}
+
 func bloom9(b []byte) *big.Int {
 	b = crypto.Keccak256(b[:])
 

eth/api_backend.go

@@ -201,6 +201,22 @@ func (b *EthApiBackend) AccountManager() *accounts.Manager {
 	return b.eth.AccountManager()
 }
 
+func (b *EthApiBackend) GetBloomBits(ctx context.Context, bitIdx uint64, sectionIdxList []uint64) ([][]byte, error) {
+	results := make([][]byte, len(sectionIdxList))
+	var err error
+	for i, idx := range sectionIdxList {
+		results[i], err = core.GetBloomBits(b.eth.chainDb, bitIdx, idx)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return results, nil
+}
+
+func (b *EthApiBackend) BloomBitsSectionSize() uint64 {
+	return bloomBitsSection
+}
+
 type EthApiState struct {
 	state *state.StateDB
 }

eth/backend.go

@@ -125,9 +125,6 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
 		MinerThreads:   config.MinerThreads,
 	}
 
-	if err := addMipmapBloomBins(chainDb); err != nil {
-		return nil, err
-	}
 	log.Info("Initialising Ethereum protocol", "versions", ProtocolVersions, "network", config.NetworkId)
 
 	if !config.SkipBcVersionCheck {
@@ -266,7 +263,7 @@ func (s *Ethereum) APIs() []rpc.API {
 		}, {
 			Namespace: "eth",
 			Version:   "1.0",
-			Service:   filters.NewPublicFilterAPI(s.ApiBackend, false),
+			Service:   filters.NewPublicFilterAPI(s.ApiBackend, false, bloomBitsSection),
 			Public:    true,
 		}, {
 			Namespace: "admin",

eth/backend_test.go

@@ -1,83 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package eth
-
-import (
-	"math/big"
-	"testing"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/ethereum/go-ethereum/params"
-)
-
-func TestMipmapUpgrade(t *testing.T) {
-	db, _ := ethdb.NewMemDatabase()
-	addr := common.BytesToAddress([]byte("jeff"))
-	genesis := new(core.Genesis).MustCommit(db)
-
-	chain, receipts := core.GenerateChain(params.TestChainConfig, genesis, db, 10, func(i int, gen *core.BlockGen) {
-		var receipts types.Receipts
-		switch i {
-		case 1:
-			receipt := types.NewReceipt(nil, new(big.Int))
-			receipt.Logs = []*types.Log{{Address: addr}}
-			gen.AddUncheckedReceipt(receipt)
-			receipts = types.Receipts{receipt}
-		case 2:
-			receipt := types.NewReceipt(nil, new(big.Int))
-			receipt.Logs = []*types.Log{{Address: addr}}
-			gen.AddUncheckedReceipt(receipt)
-			receipts = types.Receipts{receipt}
-		}
-
-		// store the receipts
-		err := core.WriteReceipts(db, receipts)
-		if err != nil {
-			t.Fatal(err)
-		}
-	})
-	for i, block := range chain {
-		core.WriteBlock(db, block)
-		if err := core.WriteCanonicalHash(db, block.Hash(), block.NumberU64()); err != nil {
-			t.Fatalf("failed to insert block number: %v", err)
-		}
-		if err := core.WriteHeadBlockHash(db, block.Hash()); err != nil {
-			t.Fatalf("failed to insert block number: %v", err)
-		}
-		if err := core.WriteBlockReceipts(db, block.Hash(), block.NumberU64(), receipts[i]); err != nil {
-			t.Fatal("error writing block receipts:", err)
-		}
-	}
-
-	err := addMipmapBloomBins(db)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	bloom := core.GetMipmapBloom(db, 1, core.MIPMapLevels[0])
-	if (bloom == types.Bloom{}) {
-		t.Error("got empty bloom filter")
-	}
-
-	data, _ := db.Get([]byte("setting-mipmap-version"))
-	if len(data) == 0 {
-		t.Error("setting-mipmap-version not written to database")
-	}
-}

eth/db_upgrade.go

@@ -20,12 +20,12 @@ package eth
 import (
 	"bytes"
 	"encoding/binary"
-	"fmt"
 	"math/big"
 	"time"
 
-	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/bitutil"
 	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/bloombits"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
@@ -252,45 +252,64 @@ func upgradeSequentialBlockData(db ethdb.Database, hash []byte) error {
 	return nil
 }
 
-func addMipmapBloomBins(db ethdb.Database) (err error) {
+// BloomBitsProcessorBackend implements ChainSectionProcessorBackend
-	const mipmapVersion uint = 2
+type BloomBitsProcessorBackend struct {
-
+	db                  ethdb.Database
-	// check if the version is set. We ignore data for now since there's
+	lastCount, lastProg uint64
-	// only one version so we can easily ignore it for now
+}
-	var data []byte
+
-	data, _ = db.Get([]byte("setting-mipmap-version"))
+// number of confirmation blocks before a section is considered probably final and its bloom bits are calculated
-	if len(data) > 0 {
+const bloomBitsConfirmations = 512
-		var version uint
+
-		if err := rlp.DecodeBytes(data, &version); err == nil && version == mipmapVersion {
+// NewBloomBitsProcessor returns a chain processor that generates bloom bits data for the canonical chain
-			return nil
+func NewBloomBitsProcessor(db ethdb.Database, stop chan struct{}) *core.ChainSectionProcessor {
-		}
+	backend := &BloomBitsProcessorBackend{db: db, lastCount: core.GetBloomBitsAvailable(db)}
-	}
+	return core.NewChainSectionProcessor(backend, bloomBitsSection, bloomBitsConfirmations, time.Millisecond*100, stop)
-
+}
-	defer func() {
+
-		if err == nil {
+// Process calculates bloom bits for a given section
-			var val []byte
+func (b *BloomBitsProcessorBackend) Process(sectionIdx uint64) bool {
-			val, err = rlp.EncodeToBytes(mipmapVersion)
+	bc := bloombits.NewBloomBitsCreator(bloomBitsSection)
-			if err == nil {
+	var header *types.Header
-				err = db.Put([]byte("setting-mipmap-version"), val)
+	for i := sectionIdx * bloomBitsSection; i < (sectionIdx+1)*bloomBitsSection; i++ {
-			}
+		hash := core.GetCanonicalHash(b.db, i)
-			return
+		header = core.GetHeader(b.db, hash, i)
-		}
+		if header == nil {
-	}()
+			log.Error("Error creating bloomBits data", "section", sectionIdx, "missing header", i)
-	latestHash := core.GetHeadBlockHash(db)
+			return false
-	latestBlock := core.GetBlock(db, latestHash, core.GetBlockNumber(db, latestHash))
+		}
-	if latestBlock == nil { // clean database
+		bc.AddHeaderBloom(header.Bloom)
-		return
+	}
-	}
+
-
+	for i := 0; i < bloombits.BloomLength; i++ {
-	tstart := time.Now()
+		compVector := bitutil.CompressBytes(bc.GetBitVector(uint(i)))
-	log.Warn("Upgrading db log bloom bins")
+		core.StoreBloomBits(b.db, uint64(i), sectionIdx, compVector)
-	for i := uint64(0); i <= latestBlock.NumberU64(); i++ {
+	}
-		hash := core.GetCanonicalHash(db, i)
+
-		if (hash == common.Hash{}) {
+	return true
-			return fmt.Errorf("chain db corrupted. Could not find block %d.", i)
+}
-		}
+
-		core.WriteMipmapBloom(db, i, core.GetBlockReceipts(db, hash, i))
+// SetStored sets the number of stored sections (last valid section is count-1)
-	}
+func (b *BloomBitsProcessorBackend) SetStored(count uint64) {
-	log.Info("Bloom-bin upgrade completed", "elapsed", common.PrettyDuration(time.Since(tstart)))
+	if count > b.lastCount {
-	return nil
+		log.Debug("Stored bloomBits data", "count", count)
+	} else {
+		log.Debug("Rolled back bloomBits data", "count", count)
+	}
+	b.lastCount = count
+	core.StoreBloomBitsAvailable(b.db, count)
+}
+
+// GetStored retrieves the number of stored sections (last valid section is count-1)
+func (b *BloomBitsProcessorBackend) GetStored() uint64 {
+	return core.GetBloomBitsAvailable(b.db)
+}
+
+// UpdateMsg prints an update message after each 5% of progress
+func (b *BloomBitsProcessorBackend) UpdateMsg(done, all uint64) {
+	prog := done * 20 / all
+	if done == 0 || prog > b.lastProg {
+		b.lastProg = prog
+		log.Info("Updating quick bloom filter database", "%", prog*5)
+	}
 }

eth/filters/api.go

@@ -51,25 +51,25 @@ type filter struct {
 // PublicFilterAPI offers support to create and manage filters. This will allow external clients to retrieve various
 // information related to the Ethereum protocol such als blocks, transactions and logs.
 type PublicFilterAPI struct {
-	backend   Backend
+	backend          Backend
-	useMipMap bool
+	bloomBitsSection uint64
-	mux       *event.TypeMux
+	mux              *event.TypeMux
-	quit      chan struct{}
+	quit             chan struct{}
-	chainDb   ethdb.Database
+	chainDb          ethdb.Database
-	events    *EventSystem
+	events           *EventSystem
-	filtersMu sync.Mutex
+	filtersMu        sync.Mutex
-	filters   map[rpc.ID]*filter
+	filters          map[rpc.ID]*filter
 }
 
 // NewPublicFilterAPI returns a new PublicFilterAPI instance.
-func NewPublicFilterAPI(backend Backend, lightMode bool) *PublicFilterAPI {
+func NewPublicFilterAPI(backend Backend, lightMode bool, bloomBitsSection uint64) *PublicFilterAPI {
 	api := &PublicFilterAPI{
-		backend:   backend,
+		backend:          backend,
-		useMipMap: !lightMode,
+		bloomBitsSection: bloomBitsSection,
-		mux:       backend.EventMux(),
+		mux:              backend.EventMux(),
-		chainDb:   backend.ChainDb(),
+		chainDb:          backend.ChainDb(),
-		events:    NewEventSystem(backend.EventMux(), backend, lightMode),
+		events:           NewEventSystem(backend.EventMux(), backend, lightMode),
-		filters:   make(map[rpc.ID]*filter),
+		filters:          make(map[rpc.ID]*filter),
 	}
 
 	go api.timeoutLoop()
@@ -333,11 +333,7 @@ func (api *PublicFilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([
 		crit.ToBlock = big.NewInt(rpc.LatestBlockNumber.Int64())
 	}
 
-	filter := New(api.backend, api.useMipMap)
+	filter := New(api.backend, crit.FromBlock.Int64(), crit.ToBlock.Int64(), crit.Addresses, crit.Topics)
-	filter.SetBeginBlock(crit.FromBlock.Int64())
-	filter.SetEndBlock(crit.ToBlock.Int64())
-	filter.SetAddresses(crit.Addresses)
-	filter.SetTopics(crit.Topics)
 
 	logs, err := filter.Find(ctx)
 	return returnLogs(logs), err
@@ -373,19 +369,18 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*ty
 		return nil, fmt.Errorf("filter not found")
 	}
 
-	filter := New(api.backend, api.useMipMap)
+	var begin, end int64
 	if f.crit.FromBlock != nil {
-		filter.SetBeginBlock(f.crit.FromBlock.Int64())
+		begin = f.crit.FromBlock.Int64()
 	} else {
-		filter.SetBeginBlock(rpc.LatestBlockNumber.Int64())
+		begin = rpc.LatestBlockNumber.Int64()
 	}
 	if f.crit.ToBlock != nil {
-		filter.SetEndBlock(f.crit.ToBlock.Int64())
+		end = f.crit.ToBlock.Int64()
 	} else {
-		filter.SetEndBlock(rpc.LatestBlockNumber.Int64())
+		end = rpc.LatestBlockNumber.Int64()
 	}
-	filter.SetAddresses(f.crit.Addresses)
+	filter := New(api.backend, begin, end, f.crit.Addresses, f.crit.Topics)
-	filter.SetTopics(f.crit.Topics)
 
 	logs, err := filter.Find(ctx)
 	if err != nil {

eth/filters/bench_test.go

@@ -0,0 +1,239 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package filters
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/bitutil"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/bloombits"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/event"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/golang/snappy"
+)
+
+func BenchmarkBloomBits512(b *testing.B) {
+	benchmarkBloomBitsForSize(b, 512)
+}
+
+func BenchmarkBloomBits1k(b *testing.B) {
+	benchmarkBloomBitsForSize(b, 1024)
+}
+
+func BenchmarkBloomBits2k(b *testing.B) {
+	benchmarkBloomBitsForSize(b, 2048)
+}
+
+func BenchmarkBloomBits4k(b *testing.B) {
+	benchmarkBloomBitsForSize(b, 4096)
+}
+
+func BenchmarkBloomBits8k(b *testing.B) {
+	benchmarkBloomBitsForSize(b, 8192)
+}
+
+func BenchmarkBloomBits16k(b *testing.B) {
+	benchmarkBloomBitsForSize(b, 16384)
+}
+
+func BenchmarkBloomBits32k(b *testing.B) {
+	benchmarkBloomBitsForSize(b, 32768)
+}
+
+func benchmarkBloomBitsForSize(b *testing.B, sectionSize uint64) {
+	benchmarkBloomBits(b, sectionSize, 0)
+	benchmarkBloomBits(b, sectionSize, 1)
+	benchmarkBloomBits(b, sectionSize, 2)
+}
+
+const benchFilterCnt = 2000
+
+func benchmarkBloomBits(b *testing.B, sectionSize uint64, comp int) {
+	benchDataDir := node.DefaultDataDir() + "/geth/chaindata"
+	fmt.Println("Running bloombits benchmark   section size:", sectionSize, "  compression method:", comp)
+
+	var (
+		compressFn   func([]byte) []byte
+		decompressFn func([]byte, int) ([]byte, error)
+	)
+	switch comp {
+	case 0:
+		// no compression
+		compressFn = func(data []byte) []byte {
+			return data
+		}
+		decompressFn = func(data []byte, target int) ([]byte, error) {
+			if len(data) != target {
+				panic(nil)
+			}
+			return data, nil
+		}
+	case 1:
+		// bitutil/compress.go
+		compressFn = bitutil.CompressBytes
+		decompressFn = bitutil.DecompressBytes
+	case 2:
+		// go snappy
+		compressFn = func(data []byte) []byte {
+			return snappy.Encode(nil, data)
+		}
+		decompressFn = func(data []byte, target int) ([]byte, error) {
+			decomp, err := snappy.Decode(nil, data)
+			if err != nil || len(decomp) != target {
+				panic(err)
+			}
+			return decomp, nil
+		}
+	}
+
+	db, err := ethdb.NewLDBDatabase(benchDataDir, 128, 1024)
+	if err != nil {
+		b.Fatalf("error opening database at %v: %v", benchDataDir, err)
+	}
+	head := core.GetHeadBlockHash(db)
+	if head == (common.Hash{}) {
+		b.Fatalf("chain data not found at %v", benchDataDir)
+	}
+
+	clearBloomBits(db)
+	fmt.Println("Generating bloombits data...")
+	headNum := core.GetBlockNumber(db, head)
+	if headNum < sectionSize+512 {
+		b.Fatalf("not enough blocks for running a benchmark")
+	}
+
+	start := time.Now()
+	cnt := (headNum - 512) / sectionSize
+	var dataSize, compSize uint64
+	for sectionIdx := uint64(0); sectionIdx < cnt; sectionIdx++ {
+		bc := bloombits.NewBloomBitsCreator(sectionSize)
+		var header *types.Header
+		for i := sectionIdx * sectionSize; i < (sectionIdx+1)*sectionSize; i++ {
+			hash := core.GetCanonicalHash(db, i)
+			header = core.GetHeader(db, hash, i)
+			if header == nil {
+				b.Fatalf("Error creating bloomBits data")
+			}
+			bc.AddHeaderBloom(header.Bloom)
+		}
+		for i := 0; i < bloombits.BloomLength; i++ {
+			data := bc.GetBitVector(uint(i))
+			comp := compressFn(data)
+			dataSize += uint64(len(data))
+			compSize += uint64(len(comp))
+			core.StoreBloomBits(db, uint64(i), sectionIdx, comp)
+		}
+		//if sectionIdx%50 == 0 {
+		//	fmt.Println(" section", sectionIdx, "/", cnt)
+		//}
+	}
+	core.StoreBloomBitsAvailable(db, cnt)
+
+	d := time.Since(start)
+	fmt.Println("Finished generating bloombits data")
+	fmt.Println(" ", d, "total  ", d/time.Duration(cnt*sectionSize), "per block")
+	fmt.Println(" data size:", dataSize, "  compressed size:", compSize, "  compression ratio:", float64(compSize)/float64(dataSize))
+
+	fmt.Println("Running filter benchmarks...")
+	start = time.Now()
+	mux := new(event.TypeMux)
+	var backend *testBackend
+
+	for i := 0; i < benchFilterCnt; i++ {
+		if i%20 == 0 {
+			db.Close()
+			db, _ = ethdb.NewLDBDatabase(benchDataDir, 128, 1024)
+			backend = &testBackend{mux, db}
+		}
+		var addr common.Address
+		addr[0] = byte(i)
+		addr[1] = byte(i / 256)
+		filter := New(backend, 0, int64(cnt*sectionSize-1), []common.Address{addr}, nil)
+		filter.decompress = decompressFn
+		if _, err := filter.Find(context.Background()); err != nil {
+			b.Error("filter.Find error:", err)
+		}
+	}
+	d = time.Since(start)
+	fmt.Println("Finished running filter benchmarks")
+	fmt.Println(" ", d, "total  ", d/time.Duration(benchFilterCnt), "per address", d*time.Duration(1000000)/time.Duration(benchFilterCnt*cnt*sectionSize), "per million blocks")
+	db.Close()
+}
+
+func forEachKey(db ethdb.Database, startPrefix, endPrefix []byte, fn func(key []byte)) {
+	it := db.(*ethdb.LDBDatabase).NewIterator()
+	it.Seek(startPrefix)
+	for it.Valid() {
+		key := it.Key()
+		cmpLen := len(key)
+		if len(endPrefix) < cmpLen {
+			cmpLen = len(endPrefix)
+		}
+		if bytes.Compare(key[:cmpLen], endPrefix) == 1 {
+			break
+		}
+		fn(common.CopyBytes(key))
+		it.Next()
+	}
+	it.Release()
+}
+
+var bloomBitsPrefix = []byte("bloomBits-")
+
+func clearBloomBits(db ethdb.Database) {
+	fmt.Println("Clearing bloombits data...")
+	forEachKey(db, bloomBitsPrefix, bloomBitsPrefix, func(key []byte) {
+		db.Delete(key)
+	})
+	core.StoreBloomBitsAvailable(db, 0)
+	fmt.Println("Cleared bloombits data")
+}
+
+func BenchmarkNoBloomBits(b *testing.B) {
+	benchDataDir := node.DefaultDataDir() + "/geth/chaindata"
+	fmt.Println("Running benchmark without bloombits")
+	db, err := ethdb.NewLDBDatabase(benchDataDir, 128, 1024)
+	if err != nil {
+		b.Fatalf("error opening database at %v: %v", benchDataDir, err)
+	}
+	head := core.GetHeadBlockHash(db)
+	if head == (common.Hash{}) {
+		b.Fatalf("chain data not found at %v", benchDataDir)
+	}
+	headNum := core.GetBlockNumber(db, head)
+
+	clearBloomBits(db)
+
+	fmt.Println("Running filter benchmarks...")
+	start := time.Now()
+	mux := new(event.TypeMux)
+	backend := &testBackend{mux, db}
+	filter := New(backend, 0, int64(headNum), []common.Address{common.Address{}}, nil)
+	filter.Find(context.Background())
+	d := time.Since(start)
+	fmt.Println("Finished running filter benchmarks")
+	fmt.Println(" ", d, "total  ", d*time.Duration(1000000)/time.Duration(headNum+1), "per million blocks")
+	db.Close()
+}

eth/filters/filter.go

@@ -18,12 +18,14 @@ package filters
 
 import (
 	"context"
-	"math"
 	"math/big"
+	"sync"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/bitutil"
 	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/bloombits"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
@@ -35,12 +37,14 @@ type Backend interface {
 	EventMux() *event.TypeMux
 	HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error)
 	GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error)
+	GetBloomBits(ctx context.Context, bitIdx uint64, sectionIdxList []uint64) ([][]byte, error)
+	BloomBitsSectionSize() uint64
 }
 
 // Filter can be used to retrieve and filter logs.
 type Filter struct {
-	backend   Backend
+	backend          Backend
-	useMipMap bool
+	bloomBitsSection uint64
 
 	created time.Time
 
@@ -48,43 +52,27 @@ type Filter struct {
 	begin, end int64
 	addresses  []common.Address
 	topics     [][]common.Hash
+
+	decompress func([]byte, int) ([]byte, error)
+	matcher    *bloombits.Matcher
 }
 
 // New creates a new filter which uses a bloom filter on blocks to figure out whether
 // a particular block is interesting or not.
-// MipMaps allow past blocks to be searched much more efficiently, but are not available
+func New(backend Backend, begin, end int64, addresses []common.Address, topics [][]common.Hash) *Filter {
-// to light clients.
-func New(backend Backend, useMipMap bool) *Filter {
 	return &Filter{
-		backend:   backend,
+		backend:          backend,
-		useMipMap: useMipMap,
+		begin:            begin,
-		db:        backend.ChainDb(),
+		end:              end,
+		addresses:        addresses,
+		topics:           topics,
+		bloomBitsSection: backend.BloomBitsSectionSize(),
+		db:               backend.ChainDb(),
+		matcher:          bloombits.NewMatcher(backend.BloomBitsSectionSize(), addresses, topics),
+		decompress:       bitutil.DecompressBytes,
 	}
 }
 
-// SetBeginBlock sets the earliest block for filtering.
-// -1 = latest block (i.e., the current block)
-// hash = particular hash from-to
-func (f *Filter) SetBeginBlock(begin int64) {
-	f.begin = begin
-}
-
-// SetEndBlock sets the latest block for filtering.
-func (f *Filter) SetEndBlock(end int64) {
-	f.end = end
-}
-
-// SetAddresses matches only logs that are generated from addresses that are included
-// in the given addresses.
-func (f *Filter) SetAddresses(addr []common.Address) {
-	f.addresses = addr
-}
-
-// SetTopics matches only logs that have topics matching the given topics.
-func (f *Filter) SetTopics(topics [][]common.Hash) {
-	f.topics = topics
-}
-
 // FindOnce searches the blockchain for matching log entries, returning
 // all matching entries from the first block that contains matches,
 // updating the start point of the filter accordingly. If no results are
@@ -105,18 +93,9 @@ func (f *Filter) FindOnce(ctx context.Context) ([]*types.Log, error) {
 		endBlockNo = headBlockNumber
 	}
 
-	// if no addresses are present we can't make use of fast search which
+	logs, blockNumber, err := f.getLogs(ctx, beginBlockNo, endBlockNo)
-	// uses the mipmap bloom filters to check for fast inclusion and uses
-	// higher range probability in order to ensure at least a false positive
-	if !f.useMipMap || len(f.addresses) == 0 {
-		logs, blockNumber, err := f.getLogs(ctx, beginBlockNo, endBlockNo)
-		f.begin = int64(blockNumber + 1)
-		return logs, err
-	}
-
-	logs, blockNumber := f.mipFind(beginBlockNo, endBlockNo, 0)
 	f.begin = int64(blockNumber + 1)
-	return logs, nil
+	return logs, err
 }
 
 // Run filters logs with the current parameters set
@@ -130,43 +109,118 @@ func (f *Filter) Find(ctx context.Context) (logs []*types.Log, err error) {
 	}
 }
 
-func (f *Filter) mipFind(start, end uint64, depth int) (logs []*types.Log, blockNumber uint64) {
+// serveMatcher serves the bloomBits matcher by fetching the requested vectors
-	level := core.MIPMapLevels[depth]
+// through the filter backend
-	// normalise numerator so we can work in level specific batches and
+func (f *Filter) serveMatcher(ctx context.Context, stop chan struct{}, wg *sync.WaitGroup) chan error {
-	// work with the proper range checks
+	errChn := make(chan error, 1)
-	for num := start / level * level; num <= end; num += level {
+	wg.Add(10)
-		// find addresses in bloom filters
+	for i := 0; i < 10; i++ {
-		bloom := core.GetMipmapBloom(f.db, num, level)
+		go func(i int) {
-		// Don't bother checking the first time through the loop - we're probably picking
+			defer wg.Done()
-		// up where a previous run left off.
+
-		first := true
+			for {
-		for _, addr := range f.addresses {
+				b, s := f.matcher.NextRequest(stop)
-			if first || bloom.TestBytes(addr[:]) {
+				if s == nil {
-				first = false
+					return
-				// range check normalised values and make sure that
+				}
-				// we're resolving the correct range instead of the
+				data, err := f.backend.GetBloomBits(ctx, uint64(b), s)
-				// normalised values.
+				if err != nil {
-				start := uint64(math.Max(float64(num), float64(start)))
+					select {
-				end := uint64(math.Min(float64(num+level-1), float64(end)))
+					case errChn <- err:
-				if depth+1 == len(core.MIPMapLevels) {
+					case <-stop:
-					l, blockNumber, _ := f.getLogs(context.Background(), start, end)
-					if len(l) > 0 {
-						return l, blockNumber
 					}
-				} else {
+					return
-					l, blockNumber := f.mipFind(start, end, depth+1)
+				}
-					if len(l) > 0 {
+				decomp := make([][]byte, len(data))
-						return l, blockNumber
+				for i, d := range data {
+					var err error
+					if decomp[i], err = f.decompress(d, int(f.bloomBitsSection/8)); err != nil {
+						select {
+						case errChn <- err:
+						case <-stop:
+						}
+						return
 					}
 				}
+				f.matcher.Deliver(b, s, decomp)
 			}
-		}
+		}(i)
 	}
 
-	return nil, end
+	return errChn
+}
+
+// checkMatches checks if the receipts belonging to the given header contain any log events that
+// match the filter criteria. This function is called when the bloom filter signals a potential match.
+func (f *Filter) checkMatches(ctx context.Context, header *types.Header) (logs []*types.Log, err error) {
+	// Get the logs of the block
+	receipts, err := f.backend.GetReceipts(ctx, header.Hash())
+	if err != nil {
+		return nil, err
+	}
+	var unfiltered []*types.Log
+	for _, receipt := range receipts {
+		unfiltered = append(unfiltered, ([]*types.Log)(receipt.Logs)...)
+	}
+	logs = filterLogs(unfiltered, nil, nil, f.addresses, f.topics)
+	if len(logs) > 0 {
+		return logs, nil
+	}
+	return nil, nil
 }
 
 func (f *Filter) getLogs(ctx context.Context, start, end uint64) (logs []*types.Log, blockNumber uint64, err error) {
+
+	haveBloomBitsBefore := core.GetBloomBitsAvailable(f.db) * f.bloomBitsSection
+	if haveBloomBitsBefore > start {
+		e := end
+		if haveBloomBitsBefore <= e {
+			e = haveBloomBitsBefore - 1
+		}
+
+		stop := make(chan struct{})
+		var wg sync.WaitGroup
+		matches := f.matcher.GetMatches(start, e, stop)
+		errChn := f.serveMatcher(ctx, stop, &wg)
+
+		defer wg.Wait()
+		defer close(stop)
+
+	loop:
+		for {
+			select {
+			case i, ok := <-matches:
+				if !ok {
+					break loop
+				}
+
+				blockNumber := rpc.BlockNumber(i)
+				header, err := f.backend.HeaderByNumber(ctx, blockNumber)
+				if header == nil || err != nil {
+					return logs, end, err
+				}
+
+				logs, err := f.checkMatches(ctx, header)
+				if err != nil {
+					return nil, end, err
+				}
+				if logs != nil {
+					return logs, i, nil
+				}
+			case err := <-errChn:
+				return logs, end, err
+			case <-ctx.Done():
+				return nil, end, ctx.Err()
+			}
+		}
+
+		if end < haveBloomBitsBefore {
+			return logs, end, nil
+		}
+		start = haveBloomBitsBefore
+	}
+
+	// search the rest with regular block-by-block bloom filtering
 	for i := start; i <= end; i++ {
 		blockNumber := rpc.BlockNumber(i)
 		header, err := f.backend.HeaderByNumber(ctx, blockNumber)
@@ -177,18 +231,12 @@ func (f *Filter) getLogs(ctx context.Context, start, end uint64) (logs []*types.
 		// Use bloom filtering to see if this block is interesting given the
 		// current parameters
 		if f.bloomFilter(header.Bloom) {
-			// Get the logs of the block
+			logs, err := f.checkMatches(ctx, header)
-			receipts, err := f.backend.GetReceipts(ctx, header.Hash())
 			if err != nil {
 				return nil, end, err
 			}
-			var unfiltered []*types.Log
+			if logs != nil {
-			for _, receipt := range receipts {
+				return logs, i, nil
-				unfiltered = append(unfiltered, ([]*types.Log)(receipt.Logs)...)
-			}
-			logs = filterLogs(unfiltered, nil, nil, f.addresses, f.topics)
-			if len(logs) > 0 {
-				return logs, uint64(blockNumber), nil
 			}
 		}
 	}

eth/filters/filter_system_test.go

@@ -63,6 +63,22 @@ func (b *testBackend) GetReceipts(ctx context.Context, blockHash common.Hash) (t
 	return core.GetBlockReceipts(b.db, blockHash, num), nil
 }
 
+func (b *testBackend) GetBloomBits(ctx context.Context, bitIdx uint64, sectionIdxList []uint64) ([][]byte, error) {
+	results := make([][]byte, len(sectionIdxList))
+	var err error
+	for i, idx := range sectionIdxList {
+		results[i], err = core.GetBloomBits(b.db, bitIdx, idx)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return results, nil
+}
+
+func (b *testBackend) BloomBitsSectionSize() uint64 {
+	return testBloomBitsSection
+}
+
 // TestBlockSubscription tests if a block subscription returns block hashes for posted chain events.
 // It creates multiple subscriptions:
 // - one at the start and should receive all posted chain events and a second (blockHashes)
@@ -75,7 +91,7 @@ func TestBlockSubscription(t *testing.T) {
 		mux         = new(event.TypeMux)
 		db, _       = ethdb.NewMemDatabase()
 		backend     = &testBackend{mux, db}
-		api         = NewPublicFilterAPI(backend, false)
+		api         = NewPublicFilterAPI(backend, false, 0)
 		genesis     = new(core.Genesis).MustCommit(db)
 		chain, _    = core.GenerateChain(params.TestChainConfig, genesis, db, 10, func(i int, gen *core.BlockGen) {})
 		chainEvents = []core.ChainEvent{}
@@ -128,7 +144,7 @@ func TestPendingTxFilter(t *testing.T) {
 		mux     = new(event.TypeMux)
 		db, _   = ethdb.NewMemDatabase()
 		backend = &testBackend{mux, db}
-		api     = NewPublicFilterAPI(backend, false)
+		api     = NewPublicFilterAPI(backend, false, 0)
 
 		transactions = []*types.Transaction{
 			types.NewTransaction(0, common.HexToAddress("0xb794f5ea0ba39494ce83a213fffba74279579268"), new(big.Int), new(big.Int), new(big.Int), nil),
@@ -178,7 +194,7 @@ func TestLogFilterCreation(t *testing.T) {
 		mux     = new(event.TypeMux)
 		db, _   = ethdb.NewMemDatabase()
 		backend = &testBackend{mux, db}
-		api     = NewPublicFilterAPI(backend, false)
+		api     = NewPublicFilterAPI(backend, false, 0)
 
 		testCases = []struct {
 			crit    FilterCriteria
@@ -223,7 +239,7 @@ func TestInvalidLogFilterCreation(t *testing.T) {
 		mux     = new(event.TypeMux)
 		db, _   = ethdb.NewMemDatabase()
 		backend = &testBackend{mux, db}
-		api     = NewPublicFilterAPI(backend, false)
+		api     = NewPublicFilterAPI(backend, false, 0)
 	)
 
 	// different situations where log filter creation should fail.
@@ -249,7 +265,7 @@ func TestLogFilter(t *testing.T) {
 		mux     = new(event.TypeMux)
 		db, _   = ethdb.NewMemDatabase()
 		backend = &testBackend{mux, db}
-		api     = NewPublicFilterAPI(backend, false)
+		api     = NewPublicFilterAPI(backend, false, 0)
 
 		firstAddr      = common.HexToAddress("0x1111111111111111111111111111111111111111")
 		secondAddr     = common.HexToAddress("0x2222222222222222222222222222222222222222")
@@ -357,7 +373,7 @@ func TestPendingLogsSubscription(t *testing.T) {
 		mux     = new(event.TypeMux)
 		db, _   = ethdb.NewMemDatabase()
 		backend = &testBackend{mux, db}
-		api     = NewPublicFilterAPI(backend, false)
+		api     = NewPublicFilterAPI(backend, false, 0)
 
 		firstAddr      = common.HexToAddress("0x1111111111111111111111111111111111111111")
 		secondAddr     = common.HexToAddress("0x2222222222222222222222222222222222222222")

eth/filters/filter_test.go

@@ -32,6 +32,8 @@ import (
 	"github.com/ethereum/go-ethereum/params"
 )
 
+const testBloomBitsSection = 4096
+
 func makeReceipt(addr common.Address) *types.Receipt {
 	receipt := types.NewReceipt(nil, new(big.Int))
 	receipt.Logs = []*types.Log{
@@ -41,8 +43,8 @@ func makeReceipt(addr common.Address) *types.Receipt {
 	return receipt
 }
 
-func BenchmarkMipmaps(b *testing.B) {
+func BenchmarkFilters(b *testing.B) {
-	dir, err := ioutil.TempDir("", "mipmap")
+	dir, err := ioutil.TempDir("", "filtertest")
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -88,7 +90,6 @@ func BenchmarkMipmaps(b *testing.B) {
 		if err != nil {
 			b.Fatal(err)
 		}
-		core.WriteMipmapBloom(db, uint64(i+1), receipts)
 	})
 	for i, block := range chain {
 		core.WriteBlock(db, block)
@@ -104,10 +105,7 @@ func BenchmarkMipmaps(b *testing.B) {
 	}
 	b.ResetTimer()
 
-	filter := New(backend, true)
+	filter := New(backend, 0, -1, []common.Address{addr1, addr2, addr3, addr4}, nil)
-	filter.SetAddresses([]common.Address{addr1, addr2, addr3, addr4})
-	filter.SetBeginBlock(0)
-	filter.SetEndBlock(-1)
 
 	for i := 0; i < b.N; i++ {
 		logs, _ := filter.Find(context.Background())
@@ -118,7 +116,7 @@ func BenchmarkMipmaps(b *testing.B) {
 }
 
 func TestFilters(t *testing.T) {
-	dir, err := ioutil.TempDir("", "mipmap")
+	dir, err := ioutil.TempDir("", "filtertest")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -189,10 +187,6 @@ func TestFilters(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		// i is used as block number for the writes but since the i
-		// starts at 0 and block 0 (genesis) is already present increment
-		// by one
-		core.WriteMipmapBloom(db, uint64(i+1), receipts)
 	})
 	for i, block := range chain {
 		core.WriteBlock(db, block)
@@ -207,22 +201,14 @@ func TestFilters(t *testing.T) {
 		}
 	}
 
-	filter := New(backend, true)
+	filter := New(backend, 0, -1, []common.Address{addr}, [][]common.Hash{{hash1, hash2, hash3, hash4}})
-	filter.SetAddresses([]common.Address{addr})
-	filter.SetTopics([][]common.Hash{{hash1, hash2, hash3, hash4}})
-	filter.SetBeginBlock(0)
-	filter.SetEndBlock(-1)
 
 	logs, _ := filter.Find(context.Background())
 	if len(logs) != 4 {
 		t.Error("expected 4 log, got", len(logs))
 	}
 
-	filter = New(backend, true)
+	filter = New(backend, 900, 999, []common.Address{addr}, [][]common.Hash{{hash3}})
-	filter.SetAddresses([]common.Address{addr})
-	filter.SetTopics([][]common.Hash{{hash3}})
-	filter.SetBeginBlock(900)
-	filter.SetEndBlock(999)
 	logs, _ = filter.Find(context.Background())
 	if len(logs) != 1 {
 		t.Error("expected 1 log, got", len(logs))
@@ -231,11 +217,7 @@ func TestFilters(t *testing.T) {
 		t.Errorf("expected log[0].Topics[0] to be %x, got %x", hash3, logs[0].Topics[0])
 	}
 
-	filter = New(backend, true)
+	filter = New(backend, 990, -1, []common.Address{addr}, [][]common.Hash{{hash3}})
-	filter.SetAddresses([]common.Address{addr})
-	filter.SetTopics([][]common.Hash{{hash3}})
-	filter.SetBeginBlock(990)
-	filter.SetEndBlock(-1)
 	logs, _ = filter.Find(context.Background())
 	if len(logs) != 1 {
 		t.Error("expected 1 log, got", len(logs))
@@ -244,10 +226,7 @@ func TestFilters(t *testing.T) {
 		t.Errorf("expected log[0].Topics[0] to be %x, got %x", hash3, logs[0].Topics[0])
 	}
 
-	filter = New(backend, true)
+	filter = New(backend, 1, 10, nil, [][]common.Hash{{hash1, hash2}})
-	filter.SetTopics([][]common.Hash{{hash1, hash2}})
-	filter.SetBeginBlock(1)
-	filter.SetEndBlock(10)
 
 	logs, _ = filter.Find(context.Background())
 	if len(logs) != 2 {
@@ -255,10 +234,7 @@ func TestFilters(t *testing.T) {
 	}
 
 	failHash := common.BytesToHash([]byte("fail"))
-	filter = New(backend, true)
+	filter = New(backend, 0, -1, nil, [][]common.Hash{{failHash}})
-	filter.SetTopics([][]common.Hash{{failHash}})
-	filter.SetBeginBlock(0)
-	filter.SetEndBlock(-1)
 
 	logs, _ = filter.Find(context.Background())
 	if len(logs) != 0 {
@@ -266,20 +242,14 @@ func TestFilters(t *testing.T) {
 	}
 
 	failAddr := common.BytesToAddress([]byte("failmenow"))
-	filter = New(backend, true)
+	filter = New(backend, 0, -1, []common.Address{failAddr}, nil)
-	filter.SetAddresses([]common.Address{failAddr})
-	filter.SetBeginBlock(0)
-	filter.SetEndBlock(-1)
 
 	logs, _ = filter.Find(context.Background())
 	if len(logs) != 0 {
 		t.Error("expected 0 log, got", len(logs))
 	}
 
-	filter = New(backend, true)
+	filter = New(backend, 0, -1, nil, [][]common.Hash{{failHash}, {hash1}})
-	filter.SetTopics([][]common.Hash{{failHash}, {hash1}})
-	filter.SetBeginBlock(0)
-	filter.SetEndBlock(-1)
 
 	logs, _ = filter.Find(context.Background())
 	if len(logs) != 0 {

eth/handler.go

@@ -45,6 +45,8 @@ import (
 const (
 	softResponseLimit = 2 * 1024 * 1024 // Target maximum size of returned blocks, headers or node data.
 	estHeaderRlpSize  = 500             // Approximate size of an RLP encoded block header
+
+	bloomBitsSection = 4096
 )
 
 var (
@@ -87,7 +89,8 @@ type ProtocolManager struct {
 	quitSync    chan struct{}
 	noMorePeers chan struct{}
 
-	lesServer LesServer
+	lesServer          LesServer
+	bloomBitsProcessor *core.ChainSectionProcessor
 
 	// wait group is used for graceful shutdowns during downloading
 	// and processing
@@ -112,6 +115,10 @@ func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, ne
 		txsyncCh:    make(chan *txsync),
 		quitSync:    make(chan struct{}),
 	}
+
+	manager.bloomBitsProcessor = NewBloomBitsProcessor(manager.chaindb, manager.quitSync)
+	blockchain.AddChainProcessor(manager.bloomBitsProcessor)
+
 	// Figure out whether to allow fast sync or not
 	if mode == downloader.FastSync && blockchain.CurrentBlock().NumberU64() > 0 {
 		log.Warn("Blockchain not empty, fast sync disabled")

les/api_backend.go

@@ -155,3 +155,11 @@ func (b *LesApiBackend) EventMux() *event.TypeMux {
 func (b *LesApiBackend) AccountManager() *accounts.Manager {
 	return b.eth.accountManager
 }
+
+func (b *LesApiBackend) GetBloomBits(ctx context.Context, bitIdx uint64, sectionIdxList []uint64) ([][]byte, error) {
+	return nil, nil // implemented in a subsequent PR
+}
+
+func (b *LesApiBackend) BloomBitsSectionSize() uint64 {
+	return 0
+}

les/backend.go

@@ -155,7 +155,7 @@ func (s *LightEthereum) APIs() []rpc.API {
 		}, {
 			Namespace: "eth",
 			Version:   "1.0",
-			Service:   filters.NewPublicFilterAPI(s.ApiBackend, true),
+			Service:   filters.NewPublicFilterAPI(s.ApiBackend, true, 0),
 			Public:    true,
 		}, {
 			Namespace: "net",

light/lightchain.go

@@ -64,6 +64,8 @@ type LightChain struct {
 	procInterrupt int32 // interrupt signaler for block processing
 	wg            sync.WaitGroup
 
+	chainProcessors []core.ChainProcessor // NewHead function called under chain lock after a new head has been written to the db
+
 	engine consensus.Engine
 }
 
@@ -114,6 +116,15 @@ func NewLightChain(odr OdrBackend, config *params.ChainConfig, engine consensus.
 	return bc, nil
 }
 
+// AddChainProcessor adds a new head callback function that will be
+// called under chain lock after a new head has been written to the db
+func (self *LightChain) AddChainProcessor(cp core.ChainProcessor) {
+	self.chainmu.Lock()
+	self.chainProcessors = append(self.chainProcessors, cp)
+	cp.NewHead(self.hc.CurrentHeader().Number.Uint64(), false)
+	self.chainmu.Unlock()
+}
+
 func (self *LightChain) getProcInterrupt() bool {
 	return atomic.LoadInt32(&self.procInterrupt) == 1
 }
@@ -149,7 +160,13 @@ func (bc *LightChain) SetHead(head uint64) {
 	bc.mu.Lock()
 	defer bc.mu.Unlock()
 
+	oldHead := bc.hc.CurrentHeader().Number.Uint64()
 	bc.hc.SetHead(head, nil)
+	if head < oldHead {
+		for _, cp := range bc.chainProcessors {
+			cp.NewHead(head, true)
+		}
+	}
 	bc.loadLastState()
 }
 
@@ -312,11 +329,20 @@ func (self *LightChain) Rollback(chain []common.Hash) {
 	self.mu.Lock()
 	defer self.mu.Unlock()
 
+	var rollbackHead *types.Header
+
 	for i := len(chain) - 1; i >= 0; i-- {
 		hash := chain[i]
 
 		if head := self.hc.CurrentHeader(); head.Hash() == hash {
-			self.hc.SetCurrentHeader(self.GetHeader(head.ParentHash, head.Number.Uint64()-1))
+			rollbackHead = self.GetHeader(head.ParentHash, head.Number.Uint64()-1)
+			self.hc.SetCurrentHeader(rollbackHead)
+		}
+	}
+
+	if rollbackHead != nil {
+		for _, cp := range self.chainProcessors {
+			cp.NewHead(rollbackHead.Number.Uint64(), true)
 		}
 	}
 }
@@ -367,7 +393,15 @@ func (self *LightChain) InsertHeaderChain(chain []*types.Header, checkFreq int)
 		self.mu.Lock()
 		defer self.mu.Unlock()
 
-		status, err := self.hc.WriteHeader(header)
+		status, err := self.hc.WriteHeader(header, func(head *types.Header) {
+			for _, cp := range self.chainProcessors {
+				cp.NewHead(head.Number.Uint64(), true)
+			}
+		})
+
+		for _, cp := range self.chainProcessors {
+			cp.NewHead(header.Number.Uint64(), false)
+		}
 
 		switch status {
 		case core.CanonStatTy:

miner/worker.go

@@ -296,8 +296,6 @@ func (self *worker) wait() {
 					core.WriteTransactions(self.chainDb, block)
 					// store the receipts
 					core.WriteReceipts(self.chainDb, work.receipts)
-					// Write map map bloom filters
-					core.WriteMipmapBloom(self.chainDb, block.NumberU64(), work.receipts)
 					// implicit by posting ChainHeadEvent
 					mustCommitNewWork = false
 				}