core, eth/filter: optimize log retrieval

This PR includes:

- optimizes (re)generation of mipmap bloom pre-images
- optimizes receipts integration on fast sync
- optimizes mipmap bloom levels for logs searching
- replaces custom bloom filter implementation with package github.com/tylertreat/BoomFilters

use db batches for inserting receipts during fast sync

core/blockchain.go

@@ -23,7 +23,6 @@ import (
 	"io"
 	"math/big"
 	mrand "math/rand"
-	"runtime"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -44,6 +43,7 @@ import (
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/hashicorp/golang-lru"
+	boom "github.com/tylertreat/BoomFilters"
 )
 
 var (
@@ -691,102 +691,79 @@ func (self *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain
 	self.wg.Add(1)
 	defer self.wg.Done()
 
-	// Collect some import statistics to report on
+	var (
-	stats := struct{ processed, ignored int32 }{}
+		chainLength      = len(blockChain)
-	start := time.Now()
+		stats            = struct{ processed, ignored int }{}
+		start            = time.Now()
+		insertedReceipts = make(map[uint64]types.Receipts)
+	)
 
-	// Create the block importing task queue and worker functions
+	if len(receiptChain) < chainLength {
-	tasks := make(chan int, len(blockChain))
+		chainLength = len(receiptChain)
-	for i := 0; i < len(blockChain) && i < len(receiptChain); i++ {
-		tasks <- i
 	}
-	close(tasks)
 
-	errs, failed := make([]error, len(tasks)), int32(0)
+	for i := 0; i < chainLength; i++ {
-	process := func(worker int) {
+		batch := self.chainDb.NewBatch()
-		for index := range tasks {
+		block, receipts := blockChain[i], receiptChain[i]
-			block, receipts := blockChain[index], receiptChain[index]
 
 		// Short circuit insertion if shutting down or processing failed
 		if atomic.LoadInt32(&self.procInterrupt) == 1 {
-				return
+			return 0, nil
-			}
-			if atomic.LoadInt32(&failed) > 0 {
-				return
 		}
+
 		// Short circuit if the owner header is unknown
 		if !self.HasHeader(block.Hash()) {
-				errs[index] = fmt.Errorf("containing header #%d [%x…] unknown", block.Number(), block.Hash().Bytes()[:4])
+			return i, fmt.Errorf("containing header #%d [%x…] unknown", block.Number(), block.Hash().Bytes()[:4])
-				atomic.AddInt32(&failed, 1)
-				return
 		}
+
 		// Skip if the entire data is already known
 		if self.HasBlock(block.Hash()) {
-				atomic.AddInt32(&stats.ignored, 1)
+			stats.ignored += len(receipts)
 			continue
 		}
+
 		// Compute all the non-consensus fields of the receipts
 		SetReceiptsData(self.config, block, receipts)
-			// Write all the data out into the database
-			if err := WriteBody(self.chainDb, block.Hash(), block.NumberU64(), block.Body()); err != nil {
-				errs[index] = fmt.Errorf("failed to write block body: %v", err)
-				atomic.AddInt32(&failed, 1)
-				glog.Fatal(errs[index])
-				return
-			}
-			if err := WriteBlockReceipts(self.chainDb, block.Hash(), block.NumberU64(), receipts); err != nil {
-				errs[index] = fmt.Errorf("failed to write block receipts: %v", err)
-				atomic.AddInt32(&failed, 1)
-				glog.Fatal(errs[index])
-				return
-			}
-			if err := WriteMipmapBloom(self.chainDb, block.NumberU64(), receipts); err != nil {
-				errs[index] = fmt.Errorf("failed to write log blooms: %v", err)
-				atomic.AddInt32(&failed, 1)
-				glog.Fatal(errs[index])
-				return
-			}
-			if err := WriteTransactions(self.chainDb, block); err != nil {
-				errs[index] = fmt.Errorf("failed to write individual transactions: %v", err)
-				atomic.AddInt32(&failed, 1)
-				glog.Fatal(errs[index])
-				return
-			}
-			if err := WriteReceipts(self.chainDb, receipts); err != nil {
-				errs[index] = fmt.Errorf("failed to write individual receipts: %v", err)
-				atomic.AddInt32(&failed, 1)
-				glog.Fatal(errs[index])
-				return
-			}
-			atomic.AddInt32(&stats.processed, 1)
-		}
-	}
-	// Start as many worker threads as goroutines allowed
-	pending := new(sync.WaitGroup)
-	for i := 0; i < runtime.GOMAXPROCS(0); i++ {
-		pending.Add(1)
-		go func(id int) {
-			defer pending.Done()
-			process(id)
-		}(i)
-	}
-	pending.Wait()
 
-	// If anything failed, report
+		// Write all the data out into a batch
-	if failed > 0 {
+		if err := WriteBodyBatch(batch, block.Hash(), block.NumberU64(), block.Body()); err != nil {
-		for i, err := range errs {
+			err := fmt.Errorf("failed to write block body: %v", err)
-			if err != nil {
+			glog.Fatal(err)
-				return i, err
 		}
+		if err := WriteBlockReceiptsBatch(batch, block.Hash(), block.NumberU64(), receipts); err != nil {
+			err := fmt.Errorf("failed to write block receipts: %v", err)
+			glog.Fatal(err)
 		}
+		if err := WriteTransactionsBatch(batch, block); err != nil {
+			err := fmt.Errorf("failed to write individual transactions: %v", err)
+			glog.Fatal(err)
 		}
+		if err := WriteReceiptsBatch(batch, receipts); err != nil {
+			err := fmt.Errorf("failed to write individual receipts: %v", err)
+			glog.Fatal(err)
+		}
+
+		// commit batch to db.
+		if err := batch.Write(); err != nil {
+			glog.Fatalf("failed to write receipts chain into database: %v", err)
+		}
+
+		insertedReceipts[block.NumberU64()] = receipts
+		stats.processed += len(receipts)
+	}
+
+	if err := updateReceiptAddressBlooms(self.chainDb, insertedReceipts); err != nil {
+		glog.Fatalf("failed to write log address index to database: %v", err)
+	}
+
 	if atomic.LoadInt32(&self.procInterrupt) == 1 {
 		glog.V(logger.Debug).Infoln("premature abort during receipt chain processing")
 		return 0, nil
 	}
+
 	// Update the head fast sync block if better
 	self.mu.Lock()
-	head := blockChain[len(errs)-1]
+	head := blockChain[chainLength-1]
 	if self.GetTd(self.currentFastBlock.Hash(), self.currentFastBlock.NumberU64()).Cmp(self.GetTd(head.Hash(), head.NumberU64())) < 0 {
 		if err := WriteHeadFastBlockHash(self.chainDb, head.Hash()); err != nil {
 			glog.Fatalf("failed to update head fast block hash: %v", err)
@@ -796,17 +773,54 @@ func (self *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain
 	self.mu.Unlock()
 
 	// Report some public statistics so the user has a clue what's going on
-	first, last := blockChain[0], blockChain[len(blockChain)-1]
+	first, last := blockChain[0], blockChain[chainLength-1]
 
-	ignored := ""
+	ignoredStr := ""
 	if stats.ignored > 0 {
-		ignored = fmt.Sprintf(" (%d ignored)", stats.ignored)
+		ignoredStr = fmt.Sprintf(" (%d ignored)", stats.ignored)
 	}
-	glog.V(logger.Info).Infof("imported %4d receipts in %9v. #%d [%x… / %x…]%s", stats.processed, common.PrettyDuration(time.Since(start)), last.Number(), first.Hash().Bytes()[:4], last.Hash().Bytes()[:4], ignored)
+	glog.V(logger.Info).Infof("imported %5d receipts in %9v. #%d [%x… / %x…]%s", stats.processed, common.PrettyDuration(time.Since(start)), last.Number(), first.Hash().Bytes()[:4], last.Hash().Bytes()[:4], ignoredStr)
 
 	return 0, nil
 }
 
+// updateReceiptAddressBlooms integrates the addresses of all logs in the given set of receipts
+// into the given database.
+func updateReceiptAddressBlooms(chainDb ethdb.Database, receipts map[uint64]types.Receipts) error {
+	blooms := make(map[string]*boom.BloomFilter)
+	for blockNum, receipts := range receipts {
+		for _, receipt := range receipts {
+			for depth := 0; depth < len(AddrBloomMapLevels); depth++ {
+				key := BloomLogsKey(depth, blockNum)
+				f, found := blooms[string(key)]
+				if !found {
+					count, ratio := AddrBloomMapCount[depth], AddrBloomMapRatio[depth]
+					f = boom.NewBloomFilter(count, ratio)
+					if data, err := chainDb.Get(key); err == nil {
+						if err := f.GobDecode(data); err != nil {
+							return err
+						}
+					}
+					blooms[string(key)] = f
+				}
+				for _, log := range receipt.Logs {
+					f.Add(log.Address.Bytes())
+				}
+			}
+		}
+	}
+
+	batch := chainDb.NewBatch()
+	for key, filter := range blooms {
+		data, err := filter.GobEncode()
+		if err != nil {
+			return err
+		}
+		batch.Put([]byte(key), data)
+	}
+	return batch.Write()
+}
+
 // WriteBlock writes the block to the chain.
 func (self *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err error) {
 	self.wg.Add(1)

core/database_util.go

@@ -23,6 +23,7 @@ import (
 	"errors"
 	"fmt"
 	"math/big"
+
 	"sync"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -33,6 +34,7 @@ import (
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/tylertreat/BoomFilters"
 )
 
 var (
@@ -51,8 +53,10 @@ var (
 	txMetaSuffix   = []byte{0x01}
 	receiptsPrefix = []byte("receipts-")
 
-	mipmapPre    = []byte("mipmap-log-bloom-")
+	AddrBloomMapPre    = []byte("blm-log-addr-")
-	MIPMapLevels = []uint64{1000000, 500000, 100000, 50000, 1000}
+	AddrBloomMapLevels = []uint64{500000, 100000, 25000, 2500, 50}
+	AddrBloomMapCount  = []uint{500000, 250000, 75000, 15000, 2000}
+	AddrBloomMapRatio  = []float64{0.5, 0.5, 0.25, 0.1, 0.01}
 
 	configPrefix = []byte("ethereum-config-") // config prefix for the db
 
@@ -66,9 +70,6 @@ var (
 	oldBlockHashPrefix     = []byte("block-hash-") // [deprecated by the header/block split, remove eventually]
 
 	ChainConfigNotFoundErr = errors.New("ChainConfig not found") // general config not found error
-
-	mipmapBloomMu sync.Mutex // protect against race condition when updating mipmap blooms
-
 	preimageCounter        = metrics.NewCounter("db/preimage/total")
 	preimageHitCounter     = metrics.NewCounter("db/preimage/hits")
 )
@@ -353,22 +354,47 @@ func WriteHeader(db ethdb.Database, header *types.Header) error {
 	return nil
 }
 
-// WriteBody serializes the body of a block into the database.
+// WriteBodyBatch serializes the body of a block into the batch.
-func WriteBody(db ethdb.Database, hash common.Hash, number uint64, body *types.Body) error {
+func WriteBodyBatch(batch ethdb.Batch, hash common.Hash, number uint64, body *types.Body) error {
 	data, err := rlp.EncodeToBytes(body)
 	if err != nil {
 		return err
 	}
-	return WriteBodyRLP(db, hash, number, data)
+	return WriteBodyRLPBatch(batch, hash, number, data)
+}
+
+// WriteBody serializes the body of a block into the database.
+func WriteBody(db ethdb.Database, hash common.Hash, number uint64, body *types.Body) error {
+	batch := db.NewBatch()
+	err := WriteBodyBatch(batch, hash, number, body)
+	if err != nil {
+		return err
+	}
+	if err := batch.Write(); err != nil {
+		glog.Fatalf("failed to store block body into database: %v", err)
+	}
+	return nil
+}
+
+// WriteBodyRLPBatch writes a serialized body of a block into the batch.
+func WriteBodyRLPBatch(batch ethdb.Batch, hash common.Hash, number uint64, rlp rlp.RawValue) error {
+	key := append(append(bodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...)
+	if err := batch.Put(key, rlp); err != nil {
+		return fmt.Errorf("failed to store block body into batch: %v", err)
+	}
+	glog.V(logger.Debug).Infof("stored block body [%x…]", hash.Bytes()[:4])
+	return nil
 }
 
 // WriteBodyRLP writes a serialized body of a block into the database.
 func WriteBodyRLP(db ethdb.Database, hash common.Hash, number uint64, rlp rlp.RawValue) error {
-	key := append(append(bodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...)
+	batch := db.NewBatch()
-	if err := db.Put(key, rlp); err != nil {
+	if err := WriteBodyRLPBatch(batch, hash, number, rlp); err != nil {
+		return err
+	}
+	if err := batch.Write(); err != nil {
 		glog.Fatalf("failed to store block body into database: %v", err)
 	}
-	glog.V(logger.Debug).Infof("stored block body [%x…]", hash.Bytes()[:4])
 	return nil
 }
 
@@ -399,10 +425,10 @@ func WriteBlock(db ethdb.Database, block *types.Block) error {
 	return nil
 }
 
-// WriteBlockReceipts stores all the transaction receipts belonging to a block
+// WriteBlockReceiptsBatch stores all the transaction receipts belonging to a block
 // as a single receipt slice. This is used during chain reorganisations for
 // rescheduling dropped transactions.
-func WriteBlockReceipts(db ethdb.Database, hash common.Hash, number uint64, receipts types.Receipts) error {
+func WriteBlockReceiptsBatch(batch ethdb.Batch, hash common.Hash, number uint64, receipts types.Receipts) error {
 	// Convert the receipts into their storage form and serialize them
 	storageReceipts := make([]*types.ReceiptForStorage, len(receipts))
 	for i, receipt := range receipts {
@@ -414,20 +440,30 @@ func WriteBlockReceipts(db ethdb.Database, hash common.Hash, number uint64, rece
 	}
 	// Store the flattened receipt slice
 	key := append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash.Bytes()...)
-	if err := db.Put(key, bytes); err != nil {
+	if err := batch.Put(key, bytes); err != nil {
-		glog.Fatalf("failed to store block receipts into database: %v", err)
+		return err
 	}
 	glog.V(logger.Debug).Infof("stored block receipts [%x…]", hash.Bytes()[:4])
 	return nil
 }
 
-// WriteTransactions stores the transactions associated with a specific block
+// WriteBlockReceipts stores all the transaction receipts belonging to a block
-// into the given database. Beside writing the transaction, the function also
+// as a single receipt slice. This is used during chain reorganisations for
-// stores a metadata entry along with the transaction, detailing the position
+// rescheduling dropped transactions.
-// of this within the blockchain.
+func WriteBlockReceipts(db ethdb.Database, hash common.Hash, number uint64, receipts types.Receipts) error {
-func WriteTransactions(db ethdb.Database, block *types.Block) error {
 	batch := db.NewBatch()
+	err := WriteBlockReceiptsBatch(batch, hash, number, receipts)
+	if err != nil {
+		glog.Fatalf("failed to store block receipts into database: %v", err)
+	}
+	return batch.Write()
+}
 
+// WriteTransactionsBatch stores the transactions associated with a specific
+// block into the given batch. Beside writing the transaction, the function
+// also stores a metadata entry along with the transaction, detailing the
+// position of this within the blockchain.
+func WriteTransactionsBatch(batch ethdb.Batch, block *types.Block) error {
 	// Iterate over each transaction and encode it with its metadata
 	for i, tx := range block.Transactions() {
 		// Encode and queue up the transaction for storage
@@ -456,10 +492,24 @@ func WriteTransactions(db ethdb.Database, block *types.Block) error {
 			return err
 		}
 	}
+	return nil
+}
+
+// WriteTransactions stores the transactions associated with a specific block
+// into the given database. Beside writing the transaction, the function also
+// stores a metadata entry along with the transaction, detailing the position
+// of this within the blockchain.
+func WriteTransactions(db ethdb.Database, block *types.Block) error {
+	batch := db.NewBatch()
+	if err := WriteTransactionsBatch(batch, block); err != nil {
+		return err
+	}
+
 	// Write the scheduled data into the database
 	if err := batch.Write(); err != nil {
 		glog.Fatalf("failed to store transactions into database: %v", err)
 	}
+
 	return nil
 }
 
@@ -473,10 +523,8 @@ func WriteReceipt(db ethdb.Database, receipt *types.Receipt) error {
 	return db.Put(append(receiptsPrefix, receipt.TxHash.Bytes()...), data)
 }
 
-// WriteReceipts stores a batch of transaction receipts into the database.
+// WriteReceipts stores a batch of transaction receipts into the db batch.
-func WriteReceipts(db ethdb.Database, receipts types.Receipts) error {
+func WriteReceiptsBatch(batch ethdb.Batch, receipts types.Receipts) error {
-	batch := db.NewBatch()
-
 	// Iterate over all the receipts and queue them for database injection
 	for _, receipt := range receipts {
 		storageReceipt := (*types.ReceiptForStorage)(receipt)
@@ -488,6 +536,17 @@ func WriteReceipts(db ethdb.Database, receipts types.Receipts) error {
 			return err
 		}
 	}
+	return nil
+}
+
+// WriteReceipts stores a batch of transaction receipts into the database.
+func WriteReceipts(db ethdb.Database, receipts types.Receipts) error {
+	batch := db.NewBatch()
+
+	err := WriteReceiptsBatch(batch, receipts)
+	if err != nil {
+		return err
+	}
 	// Write the scheduled data into the database
 	if err := batch.Write(); err != nil {
 		glog.Fatalf("failed to store receipts into database: %v", err)
@@ -558,48 +617,87 @@ func GetBlockByHashOld(db ethdb.Database, hash common.Hash) *types.Block {
 	return (*types.Block)(&block)
 }
 
-// returns a formatted MIP mapped key by adding prefix, canonical number and level
+// BloomLogsKey returns the key that is used to store the address bloom filter
-//
+// on the given depth and block height.
-// ex. fn(98, 1000) = (prefix || 1000 || 0)
+func BloomLogsKey(depth int, blockNum uint64) []byte {
-func mipmapKey(num, level uint64) []byte {
+	level := blockNum / AddrBloomMapLevels[depth] * AddrBloomMapLevels[depth]
-	lkey := make([]byte, 8)
-	binary.BigEndian.PutUint64(lkey, level)
-	key := new(big.Int).SetUint64(num / level * level)
 
-	return append(mipmapPre, append(lkey, key.Bytes()...)...)
+	lkey := make([]byte, 10)
+	binary.BigEndian.PutUint16(lkey[:2], uint16(depth))
+	binary.BigEndian.PutUint64(lkey[2:], level)
+
+	return append(AddrBloomMapPre, lkey...)
+}
+
+var (
+	addMipMapCacheMu sync.RWMutex
+	addMipMapCache   = make(map[string]*boom.BloomFilter)
+)
+
+func GetBloomLogs(db ethdb.Database, blockNum uint64, depth int) ([]byte, *boom.BloomFilter, error) {
+	key := BloomLogsKey(depth, blockNum)
+
+	addMipMapCacheMu.Lock()
+	defer addMipMapCacheMu.Unlock()
+
+	f, found := addMipMapCache[string(key)]
+	if !found {
+		count, ratio := AddrBloomMapCount[depth], AddrBloomMapRatio[depth]
+		f = boom.NewBloomFilter(count, ratio)
+		if data, err := db.Get(key); err == nil {
+			if err := f.GobDecode(data); err != nil {
+				return nil, nil, err
+			}
+		}
+		addMipMapCache[string(key)] = f
+	}
+
+	return key, f, nil
+}
+
+// WriteMipmapBloomBatch writes each address included in the receipts' logs to the batch.
+func WriteMipmapBloomBatch(db ethdb.Database, batch ethdb.Batch, number uint64, receipts types.Receipts) error {
+	for depth, _ := range AddrBloomMapLevels {
+		key, bloom, err := GetBloomLogs(db, number, depth)
+		if err != nil {
+			return err
+		}
+
+		for _, receipt := range receipts {
+			for _, log := range receipt.Logs {
+				bloom.Add(log.Address.Bytes())
+			}
+		}
+
+		data, err := bloom.GobEncode()
+		if err != nil {
+			return err
+		}
+
+		batch.Put(key, data)
+		addMipMapCacheMu.Lock()
+		addMipMapCache[string(key)] = bloom
+		addMipMapCacheMu.Unlock()
+	}
+
+	return nil
 }
 
 // WriteMapmapBloom writes each address included in the receipts' logs to the
 // MIP bloom bin.
 func WriteMipmapBloom(db ethdb.Database, number uint64, receipts types.Receipts) error {
-	mipmapBloomMu.Lock()
-	defer mipmapBloomMu.Unlock()
-
 	batch := db.NewBatch()
-	for _, level := range MIPMapLevels {
+	err := WriteMipmapBloomBatch(db, batch, number, receipts)
-		key := mipmapKey(number, level)
+	if err != nil {
-		bloomDat, _ := db.Get(key)
+		return err
-		bloom := types.BytesToBloom(bloomDat)
-		for _, receipt := range receipts {
-			for _, log := range receipt.Logs {
-				bloom.Add(log.Address.Big())
-			}
-		}
-		batch.Put(key, bloom.Bytes())
 	}
+
 	if err := batch.Write(); err != nil {
-		return fmt.Errorf("mipmap write fail for: %d: %v", number, err)
+		glog.Fatalf("mipmap write fail for: %d: %v", number, err)
 	}
 	return nil
 }
 
-// GetMipmapBloom returns a bloom filter using the number and level as input
-// parameters. For available levels see MIPMapLevels.
-func GetMipmapBloom(db ethdb.Database, number, level uint64) types.Bloom {
-	bloomDat, _ := db.Get(mipmapKey(number, level))
-	return types.BytesToBloom(bloomDat)
-}
-
 // PreimageTable returns a Database instance with the key prefix for preimage entries.
 func PreimageTable(db ethdb.Database) ethdb.Database {
 	return ethdb.NewTable(db, preimagePrefix)

core/database_util_test.go

@@ -517,10 +517,12 @@ func TestMipmapBloom(t *testing.T) {
 	WriteMipmapBloom(db, 1, types.Receipts{receipt1})
 	WriteMipmapBloom(db, 2, types.Receipts{receipt2})
 
-	for _, level := range MIPMapLevels {
+	for depth, _ := range AddrBloomMapLevels {
-		bloom := GetMipmapBloom(db, 2, level)
+		_, bloom, err := GetBloomLogs(db, 2, depth)
-		if !bloom.Test(new(big.Int).SetBytes([]byte("address1"))) {
+		if err != nil {
-			t.Error("expected test to be included on level:", level)
+			t.Errorf("could not load bloom: %v", err)
+		} else if !bloom.Test(common.BytesToAddress([]byte("address")).Bytes()) {
+			t.Error("expected test to be included on depth:", depth)
 		}
 	}
 
@@ -538,8 +540,12 @@ func TestMipmapBloom(t *testing.T) {
 	}
 	WriteMipmapBloom(db, 1000, types.Receipts{receipt})
 
-	bloom := GetMipmapBloom(db, 1000, 1000)
+	_, bloom, err := GetBloomLogs(db, 1000, 3)
-	if bloom.TestBytes([]byte("test")) {
+	if err != nil {
+		t.Fatalf("could not load bloom: %v", err)
+	}
+
+	if bloom.Test([]byte("test")) {
 		t.Error("test should not have been included")
 	}
 }
@@ -598,8 +604,11 @@ func TestMipmapChain(t *testing.T) {
 		}
 	}
 
-	bloom := GetMipmapBloom(db, 0, 1000)
+	_, bloom, err := GetBloomLogs(db, AddrBloomMapLevels[0]+1, 0)
-	if bloom.TestBytes(addr2[:]) {
+	if err != nil {
+		t.Fatalf("could not load bloom: %v", err)
+	}
+	if bloom.Test(addr2.Bytes()) {
 		t.Error("address was included in bloom and should not have")
 	}
 }

eth/backend.go

@@ -180,9 +180,12 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
 	if err := upgradeChainDatabase(chainDb); err != nil {
 		return nil, err
 	}
+
+	if !config.LightMode {
 		if err := addMipmapBloomBins(chainDb); err != nil {
 			return nil, err
 		}
+	}
 
 	glog.V(logger.Info).Infof("Protocol Versions: %v, Network Id: %v", ProtocolVersions, config.NetworkId)
 

eth/backend_test.go

@@ -71,9 +71,13 @@ func TestMipmapUpgrade(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	bloom := core.GetMipmapBloom(db, 1, core.MIPMapLevels[0])
+	_, bloom, err := core.GetBloomLogs(db, 1, 0)
-	if (bloom == types.Bloom{}) {
+	if err != nil {
-		t.Error("got empty bloom filter")
+		t.Fatalf("could not load bloom: %v", err)
+	}
+
+	if !bloom.Test(addr.Bytes()) {
+		t.Errorf("expected address to be in bloom")
 	}
 
 	data, _ := db.Get([]byte("setting-mipmap-version"))

eth/db_upgrade.go

@@ -20,8 +20,9 @@ package eth
 import (
 	"bytes"
 	"encoding/binary"
-	"fmt"
 	"math/big"
+	"runtime"
+	"sync"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -31,6 +32,7 @@ import (
 	"github.com/ethereum/go-ethereum/logger"
 	"github.com/ethereum/go-ethereum/logger/glog"
 	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/tylertreat/BoomFilters"
 )
 
 var useSequentialKeys = []byte("dbUpgrade_20160530sequentialKeys")
@@ -313,7 +315,7 @@ func upgradeChainDatabase(db ethdb.Database) error {
 }
 
 func addMipmapBloomBins(db ethdb.Database) (err error) {
-	const mipmapVersion uint = 2
+	const mipmapVersion uint = 3
 
 	// check if the version is set. We ignore data for now since there's
 	// only one version so we can easily ignore it for now
@@ -336,21 +338,96 @@ func addMipmapBloomBins(db ethdb.Database) (err error) {
 			return
 		}
 	}()
+
 	latestHash := core.GetHeadBlockHash(db)
 	latestBlock := core.GetBlock(db, latestHash, core.GetBlockNumber(db, latestHash))
 	if latestBlock == nil { // clean database
 		return
 	}
 
+	type task struct {
+		Start uint64
+		End   uint64
+	}
+
+	var (
+		step        = core.AddrBloomMapLevels[0]
+		processTill = latestBlock.NumberU64() + 1
+		tasks       = make(chan task, 1+ (latestBlock.NumberU64()/step))
+		wg          = sync.WaitGroup{}
+	)
+
+	for i := uint64(0); i <= processTill; i += step {
+		if i+step <= processTill {
+			tasks <- task{i, i + step}
+		} else {
+			tasks <- task{i, processTill}
+		}
+	}
+	close(tasks)
+
+	process := func() {
+		for t := range tasks {
+			type LevelFilter struct {
+				start, end uint64
+				filter     *boom.BloomFilter
+			}
+			filters := make([][]LevelFilter, len(core.AddrBloomMapLevels))
+
+			// 1. create filters
+			for i, level := range core.AddrBloomMapLevels {
+				for n := uint64(0); n < (core.AddrBloomMapLevels[0] / level); n++ {
+					count, ratio := core.AddrBloomMapCount[i], core.AddrBloomMapRatio[i]
+					fil := boom.NewBloomFilter(count, ratio)
+					filters[i] = append(filters[i], LevelFilter{t.Start + (n * level), t.Start + ((n + 1) * level), fil})
+				}
+			}
+
+			// 2. add log addresses to bloom filters
+			for blockNum := t.Start; blockNum < t.End; blockNum++ {
+				hash := core.GetCanonicalHash(db, blockNum)
+				if (hash == common.Hash{}) {
+					glog.Fatalf("chain db corrupted, could not find block %d", blockNum)
+				}
+
+				for depth, lvl := range filters {
+					receipts := core.GetBlockReceipts(db, hash, blockNum)
+					f := lvl[(blockNum-t.Start)/core.AddrBloomMapLevels[depth]].filter
+					for _, receipt := range receipts {
+						for _, log := range receipt.Logs {
+							f.Add(log.Address.Bytes())
+						}
+					}
+				}
+			}
+
+			// 3. update in database
+			batch := db.NewBatch()
+			for depth, lvl := range filters {
+				for _, filterLvl := range lvl {
+					data, err := filterLvl.filter.GobEncode()
+					if err != nil {
+						glog.Fatalf("Could not serialize bloom filter")
+					}
+					batch.Put(core.BloomLogsKey(depth, filterLvl.start), data)
+				}
+			}
+
+			if err := batch.Write(); err != nil {
+				glog.Fatalf("Could not update database: %v", err)
+			}
+		}
+		wg.Done()
+	}
+
 	tstart := time.Now()
 	glog.V(logger.Info).Infoln("upgrading db log bloom bins")
-	for i := uint64(0); i <= latestBlock.NumberU64(); i++ {
+	for i := 0; i < runtime.GOMAXPROCS(0); i++ {
-		hash := core.GetCanonicalHash(db, i)
+		wg.Add(1)
-		if (hash == common.Hash{}) {
+		go process()
-			return fmt.Errorf("chain db corrupted. Could not find block %d.", i)
-		}
-		core.WriteMipmapBloom(db, i, core.GetBlockReceipts(db, hash, i))
 	}
+	wg.Wait()
 	glog.V(logger.Info).Infoln("upgrade completed in", time.Since(tstart))
+
 	return nil
 }

eth/filters/api.go

@@ -334,11 +334,7 @@ func (api *PublicFilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([
 		crit.ToBlock = big.NewInt(rpc.LatestBlockNumber.Int64())
 	}
 
-	filter := New(api.backend, api.useMipMap)
+	filter := New(api.backend, crit, api.useMipMap)
-	filter.SetBeginBlock(crit.FromBlock.Int64())
-	filter.SetEndBlock(crit.ToBlock.Int64())
-	filter.SetAddresses(crit.Addresses)
-	filter.SetTopics(crit.Topics)
 
 	logs, err := filter.Find(ctx)
 	return returnLogs(logs), err
@@ -374,20 +370,15 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*ty
 		return nil, fmt.Errorf("filter not found")
 	}
 
-	filter := New(api.backend, api.useMipMap)
+	copy := f.crit
-	if f.crit.FromBlock != nil {
+	if copy.FromBlock == nil {
-		filter.SetBeginBlock(f.crit.FromBlock.Int64())
+		copy.FromBlock = big.NewInt(rpc.LatestBlockNumber.Int64())
-	} else {
-		filter.SetBeginBlock(rpc.LatestBlockNumber.Int64())
 	}
-	if f.crit.ToBlock != nil {
+	if copy.ToBlock == nil {
-		filter.SetEndBlock(f.crit.ToBlock.Int64())
+		copy.ToBlock = big.NewInt(rpc.LatestBlockNumber.Int64())
-	} else {
-		filter.SetEndBlock(rpc.LatestBlockNumber.Int64())
 	}
-	filter.SetAddresses(f.crit.Addresses)
-	filter.SetTopics(f.crit.Topics)
 
+	filter := New(api.backend, copy, api.useMipMap)
 	logs, err := filter.Find(ctx)
 	if err != nil {
 		return nil, err

eth/filters/filter.go

@@ -17,9 +17,6 @@
 package filters
 
 import (
-	"math"
-	"time"
-
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -28,9 +25,11 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/tylertreat/BoomFilters"
 	"golang.org/x/net/context"
 )
 
+// Backend defines the interface the filter packages needs to retrieve logs.
 type Backend interface {
 	ChainDb() ethdb.Database
 	EventMux() *event.TypeMux
@@ -38,172 +37,209 @@ type Backend interface {
 	GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error)
 }
 
-// Filter can be used to retrieve and filter logs.
+// Filter can be used to search for particular logs
 type Filter struct {
-	backend   Backend
+	be        Backend
+	crit      FilterCriteria
 	useMipMap bool
-
-	created time.Time
-
-	db         ethdb.Database
-	begin, end int64
-	addresses  []common.Address
-	topics     [][]common.Hash
 }
 
-// New creates a new filter which uses a bloom filter on blocks to figure out whether
+// filterRange helper structure that keeps track of a bloom filter and the range it covers.
-// a particular block is interesting or not.
+type filterRange struct {
-// MipMaps allow past blocks to be searched much more efficiently, but are not available
+	begin uint64
-// to light clients.
+	end   uint64
-func New(backend Backend, useMipMap bool) *Filter {
+	bloom boom.Filter
+}
+
+// New creates a filter that can be used to search for logs that match the given criteria.
+// When `useMipMap` is `true` it uses pre-calculated bloom filters to optimize searching.
+// These pre-calculated bloom filters are not available in light mode and require at least
+// one address in the criteria.
+func New(be Backend, crit FilterCriteria, useMipMap bool) *Filter {
+	if crit.FromBlock == nil || crit.FromBlock.Int64() < 0 {
+		crit.FromBlock = big.NewInt(0)
+	}
 	return &Filter{
-		backend:   backend,
+		be:        be,
 		useMipMap: useMipMap,
-		db:        backend.ChainDb(),
+		crit:      crit,
 	}
 }
 
-// SetBeginBlock sets the earliest block for filtering.
+// Find returns all logs matching the filter criteria.
-// -1 = latest block (i.e., the current block)
+func (f *Filter) Find(ctx context.Context) ([]*types.Log, error) {
-// hash = particular hash from-to
+	if f.useMipMap && len(f.crit.Addresses) > 0 {
-func (f *Filter) SetBeginBlock(begin int64) {
+		return f.findByAddressIdx(ctx)
-	f.begin = begin
+	}
+	return f.findFullRangeScan(ctx)
 }
 
-// SetEndBlock sets the latest block for filtering.
+// findByAddressIdx returns all logs matching the criteria. It uses an bloom filter based
-func (f *Filter) SetEndBlock(end int64) {
+// index on log addresses for faster searching (not available in light mode).
-	f.end = end
+func (f *Filter) findByAddressIdx(ctx context.Context) ([]*types.Log, error) {
+	var (
+		depth      = 0
+		start, end uint64
+		level      = core.AddrBloomMapLevels[depth]
+		logs       []*types.Log
+		filters    = []filterRange{}
+	)
+
+	start = f.crit.FromBlock.Uint64()
+
+	if f.crit.ToBlock.Int64() < 0 {
+		h, err := f.be.HeaderByNumber(ctx, rpc.BlockNumber(f.crit.ToBlock.Int64()))
+		if err != nil {
+			return nil, err
+		}
+		end = h.Number.Uint64()
+	} else {
+		end = f.crit.ToBlock.Uint64()
+	}
+
+	// find top level block ranges that (probably) contains logs from one of the target addresses.
+	for num := (start / level) * level; num <= end; num += level {
+		_, bloom, err := core.GetBloomLogs(f.be.ChainDb(), num, depth)
+		if err != nil {
+			return logs, nil
+		}
+		if f.matchAddress(bloom) {
+			filters = append(filters, filterRange{num, num + core.AddrBloomMapLevels[depth], bloom})
+		}
+	}
+
+	return f.filterRange(ctx, depth+1, filters, start, end)
 }
 
-// SetAddresses matches only logs that are generated from addresses that are included
+// filterRange returns the set of logs that match the filter criteria in bf within the block range
-// in the given addresses.
+// [start, end]. This is a depth first search to guarantee logs are sorted by block number.
-func (f *Filter) SetAddresses(addr []common.Address) {
+func (f *Filter) filterRange(ctx context.Context, depth int, filters []filterRange, start, end uint64) ([]*types.Log, error) {
-	f.addresses = addr
+	var logs []*types.Log
-}
+	for _, filter := range filters {
-
+		// reached lowest level, do a full block scan on the filter range
-// SetTopics matches only logs that have topics matching the given topics.
+		if depth == len(core.AddrBloomMapLevels) {
-func (f *Filter) SetTopics(topics [][]common.Hash) {
+			return f.scanRangeWithFilter(ctx, filter, start, end)
-	f.topics = topics
-}
-
-// FindOnce searches the blockchain for matching log entries, returning
-// all matching entries from the first block that contains matches,
-// updating the start point of the filter accordingly. If no results are
-// found, a nil slice is returned.
-func (f *Filter) FindOnce(ctx context.Context) ([]*types.Log, error) {
-	head, _ := f.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber)
-	if head == nil {
-		return nil, nil
-	}
-	headBlockNumber := head.Number.Uint64()
-
-	var beginBlockNo uint64 = uint64(f.begin)
-	if f.begin == -1 {
-		beginBlockNo = headBlockNumber
-	}
-	var endBlockNo uint64 = uint64(f.end)
-	if f.end == -1 {
-		endBlockNo = headBlockNumber
 		}
 
-	// if no addresses are present we can't make use of fast search which
+		// narrow search by evicting branches that are guaranteed not to include logs.
-	// uses the mipmap bloom filters to check for fast inclusion and uses
+		level := core.AddrBloomMapLevels[depth]
-	// higher range probability in order to ensure at least a false positive
+		for num := (filter.begin / level) * level; num <= end && num < filter.end; num += level {
-	if !f.useMipMap || len(f.addresses) == 0 {
+			if num+level < start {
-		logs, blockNumber, err := f.getLogs(ctx, beginBlockNo, endBlockNo)
+				continue
-		f.begin = int64(blockNumber + 1)
+			}
-		return logs, err
+			_, bloom, err := core.GetBloomLogs(f.be.ChainDb(), num, depth)
+			if err != nil {
+				return nil, err
 			}
 
-	logs, blockNumber := f.mipFind(beginBlockNo, endBlockNo, 0)
+			if f.matchAddress(bloom) {
-	f.begin = int64(blockNumber + 1)
+				fr := filterRange{num, num + level, bloom}
+				l, err := f.filterRange(ctx, depth+1, []filterRange{fr}, start, end)
+				if err != nil {
+					return nil, err
+				}
+				logs = append(logs, l...)
+			}
+		}
+	}
 	return logs, nil
 }
 
-// Run filters logs with the current parameters set
+// logs is a helper that returns all logs in the specified block that matches the filter criteria.
-func (f *Filter) Find(ctx context.Context) (logs []*types.Log, err error) {
+func (f *Filter) logs(ctx context.Context, blockNum uint64) ([]*types.Log, error) {
-	for {
+	var logs []*types.Log
-		newLogs, err := f.FindOnce(ctx)
+	blockNumber := rpc.BlockNumber(blockNum)
-		if len(newLogs) == 0 || err != nil {
+	header, err := f.be.HeaderByNumber(ctx, blockNumber)
+	if header == nil || err != nil {
 		return logs, err
 	}
-		logs = append(logs, newLogs...)
-	}
-}
 
-func (f *Filter) mipFind(start, end uint64, depth int) (logs []*types.Log, blockNumber uint64) {
+	if BloomFilter(header.Bloom, f.crit.Addresses, f.crit.Topics) {
-	level := core.MIPMapLevels[depth]
+		receipts, err := f.be.GetReceipts(ctx, header.Hash())
-	// normalise numerator so we can work in level specific batches and
-	// work with the proper range checks
-	for num := start / level * level; num <= end; num += level {
-		// find addresses in bloom filters
-		bloom := core.GetMipmapBloom(f.db, num, level)
-		// Don't bother checking the first time through the loop - we're probably picking
-		// up where a previous run left off.
-		first := true
-		for _, addr := range f.addresses {
-			if first || bloom.TestBytes(addr[:]) {
-				first = false
-				// range check normalised values and make sure that
-				// we're resolving the correct range instead of the
-				// normalised values.
-				start := uint64(math.Max(float64(num), float64(start)))
-				end := uint64(math.Min(float64(num+level-1), float64(end)))
-				if depth+1 == len(core.MIPMapLevels) {
-					l, blockNumber, _ := f.getLogs(context.Background(), start, end)
-					if len(l) > 0 {
-						return l, blockNumber
-					}
-				} else {
-					l, blockNumber := f.mipFind(start, end, depth+1)
-					if len(l) > 0 {
-						return l, blockNumber
-					}
-				}
-			}
-		}
-	}
-
-	return nil, end
-}
-
-func (f *Filter) getLogs(ctx context.Context, start, end uint64) (logs []*types.Log, blockNumber uint64, err error) {
-	for i := start; i <= end; i++ {
-		blockNumber := rpc.BlockNumber(i)
-		header, err := f.backend.HeaderByNumber(ctx, blockNumber)
-		if header == nil || err != nil {
-			return logs, end, err
-		}
-
-		// Use bloom filtering to see if this block is interesting given the
-		// current parameters
-		if f.bloomFilter(header.Bloom) {
-			// Get the logs of the block
-			receipts, err := f.backend.GetReceipts(ctx, header.Hash())
 		if err != nil {
-				return nil, end, err
+			return nil, err
 		}
+
 		var unfiltered []*types.Log
 		for _, receipt := range receipts {
 			unfiltered = append(unfiltered, ([]*types.Log)(receipt.Logs)...)
 		}
-			logs = filterLogs(unfiltered, nil, nil, f.addresses, f.topics)
+		logs = append(logs, filterLogs(unfiltered, nil, nil, f.crit.Addresses, f.crit.Topics)...)
-			if len(logs) > 0 {
-				return logs, uint64(blockNumber), nil
 	}
-		}
+	return logs, nil
-	}
-
-	return logs, end, nil
 }
 
+// scanRangeWithFilter returns all logs matching the bf filter criteria.
+func (f *Filter) scanRangeWithFilter(ctx context.Context, filter filterRange, start, end uint64) ([]*types.Log, error) {
+	if start < filter.begin {
+		start = filter.begin
+	}
+	// in case the toBlock is within the filter range make it inclusive.
+	if end > filter.begin && end <= filter.end {
+		end += 1
+	} else if end > filter.end {
+		end = filter.end
+	}
+
+	var logs []*types.Log
+	for i := start; i < end; i++ {
+		if l, err := f.logs(ctx, i); err == nil {
+			logs = append(logs, l...)
+		} else {
+			return nil, err
+		}
+	}
+
+	return logs, nil
+}
+
+// findFullRangeScan performance a naive full blocks scan in the range [start, end] and
+// returns logs matching the filters criteria.
+func (f *Filter) findFullRangeScan(ctx context.Context) ([]*types.Log, error) {
+	var (
+		start = f.crit.FromBlock.Uint64()
+		end   uint64
+		logs  []*types.Log
+	)
+
+	if f.crit.ToBlock.Int64() < 0 {
+		h, err := f.be.HeaderByNumber(ctx, rpc.BlockNumber(f.crit.ToBlock.Int64()))
+		if err != nil {
+			return nil, err
+		}
+		end = h.Number.Uint64()
+	} else {
+		end = f.crit.ToBlock.Uint64()
+	}
+
+	for i := start; i < end; i++ {
+		if l, err := f.logs(ctx, i); err == nil {
+			logs = append(logs, l...)
+		} else {
+			return nil, err
+		}
+	}
+	return logs, nil
+}
+
+// matchAddress returns an indication if at least one of the addresses
+// in bf.addresses is stored in the given bloom.
+func (f *Filter) matchAddress(bloom boom.Filter) bool {
+	for _, addr := range f.crit.Addresses {
+		if bloom.Test(addr.Bytes()) {
+			return true
+		}
+	}
+	return false
+}
+
+// includes returns an indication if a is included in addresses.
 func includes(addresses []common.Address, a common.Address) bool {
 	for _, addr := range addresses {
 		if addr == a {
 			return true
 		}
 	}
-
 	return false
 }
 
@@ -251,11 +287,8 @@ Logs:
 	return ret
 }
 
-func (f *Filter) bloomFilter(bloom types.Bloom) bool {
+// BloomFilter returns an indication if the given addresses and topics match with the given bloom
-	return bloomFilter(bloom, f.addresses, f.topics)
+func BloomFilter(bloom types.Bloom, addresses []common.Address, topics [][]common.Hash) bool {
-}
-
-func bloomFilter(bloom types.Bloom, addresses []common.Address, topics [][]common.Hash) bool {
 	if len(addresses) > 0 {
 		var included bool
 		for _, addr := range addresses {
@@ -264,7 +297,6 @@ func bloomFilter(bloom types.Bloom, addresses []common.Address, topics [][]commo
 				break
 			}
 		}
-
 		if !included {
 			return false
 		}
@@ -282,6 +314,5 @@ func bloomFilter(bloom types.Bloom, addresses []common.Address, topics [][]commo
 			return false
 		}
 	}
-
 	return true
 }

eth/filters/filter_system.go

@@ -370,7 +370,7 @@ func (es *EventSystem) lightFilterNewHead(newHeader *types.Header, callBack func
 
 // filter logs of a single header in light client mode
 func (es *EventSystem) lightFilterLogs(header *types.Header, addresses []common.Address, topics [][]common.Hash, remove bool) []*types.Log {
-	if bloomFilter(header.Bloom, addresses, topics) {
+	if BloomFilter(header.Bloom, addresses, topics) {
 		// Get the logs of the block
 		ctx, _ := context.WithTimeout(context.Background(), time.Second*5)
 		receipts, err := es.backend.GetReceipts(ctx, header.Hash())

eth/filters/filter_test.go

@@ -105,11 +105,13 @@ func BenchmarkMipmaps(b *testing.B) {
 	}
 	b.ResetTimer()
 
-	filter := New(backend, true)
+	crit := FilterCriteria{
-	filter.SetAddresses([]common.Address{addr1, addr2, addr3, addr4})
+		FromBlock: big.NewInt(0),
-	filter.SetBeginBlock(0)
+		ToBlock:   big.NewInt(-1),
-	filter.SetEndBlock(-1)
+		Addresses: []common.Address{addr1, addr2, addr3, addr4},
+	}
 
+	filter := New(backend, crit, true)
 	for i := 0; i < b.N; i++ {
 		logs, _ := filter.Find(context.Background())
 		if len(logs) != 4 {
@@ -208,22 +210,22 @@ func TestFilters(t *testing.T) {
 		}
 	}
 
-	filter := New(backend, true)
+	toCrit := func(addresses []common.Address, topics [][]common.Hash, begin, end int64) FilterCriteria {
-	filter.SetAddresses([]common.Address{addr})
+		return FilterCriteria{
-	filter.SetTopics([][]common.Hash{{hash1, hash2, hash3, hash4}})
+			FromBlock: big.NewInt(begin),
-	filter.SetBeginBlock(0)
+			ToBlock:   big.NewInt(end),
-	filter.SetEndBlock(-1)
+			Addresses: addresses,
+			Topics:    topics,
+		}
+	}
 
+	filter := New(backend, toCrit([]common.Address{addr}, [][]common.Hash{{hash1, hash2, hash3, hash4}}, 0, -1), true)
 	logs, _ := filter.Find(context.Background())
 	if len(logs) != 4 {
 		t.Error("expected 4 log, got", len(logs))
 	}
 
-	filter = New(backend, true)
+	filter = New(backend, toCrit([]common.Address{addr}, [][]common.Hash{{hash3}}, 900, 999), true)
-	filter.SetAddresses([]common.Address{addr})
-	filter.SetTopics([][]common.Hash{{hash3}})
-	filter.SetBeginBlock(900)
-	filter.SetEndBlock(999)
 	logs, _ = filter.Find(context.Background())
 	if len(logs) != 1 {
 		t.Error("expected 1 log, got", len(logs))
@@ -232,11 +234,7 @@ func TestFilters(t *testing.T) {
 		t.Errorf("expected log[0].Topics[0] to be %x, got %x", hash3, logs[0].Topics[0])
 	}
 
-	filter = New(backend, true)
+	filter = New(backend, toCrit([]common.Address{addr}, [][]common.Hash{{hash3}}, 990, -1), true)
-	filter.SetAddresses([]common.Address{addr})
-	filter.SetTopics([][]common.Hash{{hash3}})
-	filter.SetBeginBlock(990)
-	filter.SetEndBlock(-1)
 	logs, _ = filter.Find(context.Background())
 	if len(logs) != 1 {
 		t.Error("expected 1 log, got", len(logs))
@@ -245,43 +243,27 @@ func TestFilters(t *testing.T) {
 		t.Errorf("expected log[0].Topics[0] to be %x, got %x", hash3, logs[0].Topics[0])
 	}
 
-	filter = New(backend, true)
+	filter = New(backend, toCrit([]common.Address{}, [][]common.Hash{{hash1, hash2}}, 1, 10), true)
-	filter.SetTopics([][]common.Hash{{hash1, hash2}})
-	filter.SetBeginBlock(1)
-	filter.SetEndBlock(10)
-
 	logs, _ = filter.Find(context.Background())
 	if len(logs) != 2 {
 		t.Error("expected 2 log, got", len(logs))
 	}
 
 	failHash := common.BytesToHash([]byte("fail"))
-	filter = New(backend, true)
+	filter = New(backend, toCrit([]common.Address{}, [][]common.Hash{{failHash}}, 0, -1), true)
-	filter.SetTopics([][]common.Hash{{failHash}})
-	filter.SetBeginBlock(0)
-	filter.SetEndBlock(-1)
-
 	logs, _ = filter.Find(context.Background())
 	if len(logs) != 0 {
 		t.Error("expected 0 log, got", len(logs))
 	}
 
 	failAddr := common.BytesToAddress([]byte("failmenow"))
-	filter = New(backend, true)
+	filter = New(backend, toCrit([]common.Address{failAddr}, [][]common.Hash{{}}, 0, -1), true)
-	filter.SetAddresses([]common.Address{failAddr})
-	filter.SetBeginBlock(0)
-	filter.SetEndBlock(-1)
-
 	logs, _ = filter.Find(context.Background())
 	if len(logs) != 0 {
 		t.Error("expected 0 log, got", len(logs))
 	}
 
-	filter = New(backend, true)
+	filter = New(backend, toCrit([]common.Address{}, [][]common.Hash{{failHash}, {hash1}}, 0, -1), true)
-	filter.SetTopics([][]common.Hash{{failHash}, {hash1}})
-	filter.SetBeginBlock(0)
-	filter.SetEndBlock(-1)
-
 	logs, _ = filter.Find(context.Background())
 	if len(logs) != 0 {
 		t.Error("expected 0 log, got", len(logs))

vendor/github.com/tylertreat/BoomFilters/.gitignore

@@ -0,0 +1,24 @@
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+*.test
+*.prof

vendor/github.com/tylertreat/BoomFilters/.travis.yml

@@ -0,0 +1,12 @@
+language: go
+
+go:
+  - 1.3
+  - 1.4
+  - tip
+
+before_install: go get golang.org/x/tools/cmd/cover
+script: go test -cover ./...
+
+notifications:
+  email: false

vendor/github.com/tylertreat/BoomFilters/LICENSE

@@ -0,0 +1,202 @@
+Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+

vendor/github.com/tylertreat/BoomFilters/README.md

@@ -0,0 +1,429 @@
+# Boom Filters
+[![Build Status](https://travis-ci.org/tylertreat/BoomFilters.svg?branch=master)](https://travis-ci.org/tylertreat/BoomFilters) [![GoDoc](https://godoc.org/github.com/tylertreat/BoomFilters?status.png)](https://godoc.org/github.com/tylertreat/BoomFilters)
+
+**Boom Filters** are probabilistic data structures for [processing continuous, unbounded streams](http://www.bravenewgeek.com/stream-processing-and-probabilistic-methods/). This includes **Stable Bloom Filters**, **Scalable Bloom Filters**, **Counting Bloom Filters**, **Inverse Bloom Filters**, **Cuckoo Filters**, several variants of **traditional Bloom filters**, **HyperLogLog**, **Count-Min Sketch**, and **MinHash**.
+
+Classic Bloom filters generally require a priori knowledge of the data set in order to allocate an appropriately sized bit array. This works well for offline processing, but online processing typically involves unbounded data streams. With enough data, a traditional Bloom filter "fills up", after which it has a false-positive probability of 1.
+
+Boom Filters are useful for situations where the size of the data set isn't known ahead of time. For example, a Stable Bloom Filter can be used to deduplicate events from an unbounded event stream with a specified upper bound on false positives and minimal false negatives. Alternatively, an Inverse Bloom Filter is ideal for deduplicating a stream where duplicate events are relatively close together. This results in no false positives and, depending on how close together duplicates are, a small probability of false negatives. Scalable Bloom Filters place a tight upper bound on false positives while avoiding false negatives but require allocating memory proportional to the size of the data set. Counting Bloom Filters and Cuckoo Filters are useful for cases which require adding and removing elements to and from a set.
+
+For large or unbounded data sets, calculating the exact cardinality is impractical. HyperLogLog uses a fraction of the memory while providing an accurate approximation. Similarly, Count-Min Sketch provides an efficient way to estimate event frequency for data streams, while Top-K tracks the top-k most frequent elements.
+
+MinHash is a probabilistic algorithm to approximate the similarity between two sets. This can be used to cluster or compare documents by splitting the corpus into a bag of words.
+
+## Installation 
+
+```
+$ go get github.com/tylertreat/BoomFilters
+```
+
+## Stable Bloom Filter
+
+This is an implementation of Stable Bloom Filters as described by Deng and Rafiei in [Approximately Detecting Duplicates for Streaming Data using Stable Bloom Filters](http://webdocs.cs.ualberta.ca/~drafiei/papers/DupDet06Sigmod.pdf).
+
+A Stable Bloom Filter (SBF) continuously evicts stale information so that it has room for more recent elements. Like traditional Bloom filters, an SBF has a non-zero probability of false positives, which is controlled by several parameters. Unlike the classic Bloom filter, an SBF has a tight upper bound on the rate of false positives while introducing a non-zero rate of false negatives. The false-positive rate of a classic Bloom filter eventually reaches 1, after which all queries result in a false positive. The stable-point property of an SBF means the false-positive rate asymptotically approaches a configurable fixed constant. A classic Bloom filter is actually a special case of SBF where the eviction rate is zero and the cell size is one, so this provides support for them as well (in addition to bitset-based Bloom filters).
+
+Stable Bloom Filters are useful for cases where the size of the data set isn't known a priori and memory is bounded. For example, an SBF can be used to deduplicate events from an unbounded event stream with a specified upper bound on false positives and minimal false negatives.
+
+### Usage
+
+```go
+package main
+
+import (
+    "fmt"
+    "github.com/tylertreat/BoomFilters"
+)
+
+func main() {
+    sbf := boom.NewDefaultStableBloomFilter(10000, 0.01)
+    fmt.Println("stable point", sbf.StablePoint())
+    
+    sbf.Add([]byte(`a`))
+    if sbf.Test([]byte(`a`)) {
+        fmt.Println("contains a")
+    }
+    
+    if !sbf.TestAndAdd([]byte(`b`)) {
+        fmt.Println("doesn't contain b")
+    }
+    
+    if sbf.Test([]byte(`b`)) {
+        fmt.Println("now it contains b!")
+    }
+    
+    // Restore to initial state.
+    sbf.Reset()
+}
+```
+
+## Scalable Bloom Filter
+
+This is an implementation of a Scalable Bloom Filter as described by Almeida, Baquero, Preguica, and Hutchison in [Scalable Bloom Filters](http://gsd.di.uminho.pt/members/cbm/ps/dbloom.pdf).
+
+A Scalable Bloom Filter (SBF) dynamically adapts to the size of the data set while enforcing a tight upper bound on the rate of false positives and a false-negative probability of zero. This works by adding Bloom filters with geometrically decreasing false-positive rates as filters become full. A tightening ratio, r, controls the filter growth. The compounded probability over the whole series converges to a target value, even accounting for an infinite series.
+
+Scalable Bloom Filters are useful for cases where the size of the data set isn't known a priori and memory constraints aren't of particular concern. For situations where memory is bounded, consider using Inverse or Stable Bloom Filters.
+
+The core parts of this implementation were originally written by Jian Zhen as discussed in [Benchmarking Bloom Filters and Hash Functions in Go](http://zhen.org/blog/benchmarking-bloom-filters-and-hash-functions-in-go/).
+
+### Usage
+
+```go
+package main
+
+import (
+    "fmt"
+    "github.com/tylertreat/BoomFilters"
+)
+
+func main() {
+    sbf := boom.NewDefaultScalableBloomFilter(0.01)
+    
+    sbf.Add([]byte(`a`))
+    if sbf.Test([]byte(`a`)) {
+        fmt.Println("contains a")
+    }
+    
+    if !sbf.TestAndAdd([]byte(`b`)) {
+        fmt.Println("doesn't contain b")
+    }
+    
+    if sbf.Test([]byte(`b`)) {
+        fmt.Println("now it contains b!")
+    }
+    
+    // Restore to initial state.
+    sbf.Reset()
+}
+```
+
+## Inverse Bloom Filter
+
+An Inverse Bloom Filter, or "the opposite of a Bloom filter", is a concurrent, probabilistic data structure used to test whether an item has been observed or not. This implementation, [originally described and written by Jeff Hodges](http://www.somethingsimilar.com/2012/05/21/the-opposite-of-a-bloom-filter/), replaces the use of MD5 hashing with a non-cryptographic FNV-1 function.
+
+The Inverse Bloom Filter may report a false negative but can never report a false positive. That is, it may report that an item has not been seen when it actually has, but it will never report an item as seen which it hasn't come across. This behaves in a similar manner to a fixed-size hashmap which does not handle conflicts.
+
+This structure is particularly well-suited to streams in which duplicates are relatively close together. It uses a CAS-style approach, which makes it thread-safe.
+
+### Usage
+
+```go
+package main
+
+import (
+    "fmt"
+    "github.com/tylertreat/BoomFilters"
+)
+
+func main() {
+    ibf := boom.NewInverseBloomFilter(10000)
+    
+    ibf.Add([]byte(`a`))
+    if ibf.Test([]byte(`a`)) {
+        fmt.Println("contains a")
+    }
+    
+    if !ibf.TestAndAdd([]byte(`b`)) {
+        fmt.Println("doesn't contain b")
+    }
+    
+    if ibf.Test([]byte(`b`)) {
+        fmt.Println("now it contains b!")
+    }
+}
+```
+
+## Counting Bloom Filter
+
+This is an implementation of a Counting Bloom Filter as described by Fan, Cao, Almeida, and Broder in [Summary Cache: A Scalable Wide-Area Web Cache Sharing Protocol](http://pages.cs.wisc.edu/~jussara/papers/00ton.pdf).
+
+A Counting Bloom Filter (CBF) provides a way to remove elements by using an array of n-bit buckets. When an element is added, the respective buckets are incremented. To remove an element, the respective buckets are decremented. A query checks that each of the respective buckets are non-zero. Because CBFs allow elements to be removed, they introduce a non-zero probability of false negatives in addition to the possibility of false positives.
+
+Counting Bloom Filters are useful for cases where elements are both added and removed from the data set. Since they use n-bit buckets, CBFs use roughly n-times more memory than traditional Bloom filters.
+
+See Deletable Bloom Filter for an alternative which avoids false negatives.
+
+### Usage
+
+```go
+package main
+
+import (
+    "fmt"
+    "github.com/tylertreat/BoomFilters"
+)
+
+func main() {
+    bf := boom.NewDefaultCountingBloomFilter(1000, 0.01)
+    
+    bf.Add([]byte(`a`))
+    if bf.Test([]byte(`a`)) {
+        fmt.Println("contains a")
+    }
+    
+    if !bf.TestAndAdd([]byte(`b`)) {
+        fmt.Println("doesn't contain b")
+    }
+    
+    if bf.TestAndRemove([]byte(`b`)) {
+        fmt.Println("removed b")
+    }
+    
+    // Restore to initial state.
+    bf.Reset()
+}
+```
+
+## Cuckoo Filter
+
+This is an implementation of a Cuckoo Filter as described by Andersen, Kaminsky, and Mitzenmacher in [Cuckoo Filter: Practically Better Than Bloom](http://www.pdl.cmu.edu/PDL-FTP/FS/cuckoo-conext2014.pdf). The Cuckoo Filter is similar to the Counting Bloom Filter in that it supports adding and removing elements, but it does so in a way that doesn't significantly degrade space and performance.
+
+It works by using a cuckoo hashing scheme for inserting items. Instead of storing the elements themselves, it stores their fingerprints which also allows for item removal without false negatives (if you don't attempt to remove an item not contained in the filter).
+
+For applications that store many items and target moderately low false-positive rates, cuckoo filters have lower space overhead than space-optimized Bloom filters.
+
+### Usage
+
+```go
+package main
+
+import (
+    "fmt"
+    "github.com/tylertreat/BoomFilters"
+)
+
+func main() {
+    cf := boom.NewCuckooFilter(1000, 0.01)
+    
+    cf.Add([]byte(`a`))
+    if cf.Test([]byte(`a`)) {
+        fmt.Println("contains a")
+    }
+    
+    if contains, _ := cf.TestAndAdd([]byte(`b`)); !contains {
+        fmt.Println("doesn't contain b")
+    }
+    
+    if cf.TestAndRemove([]byte(`b`)) {
+        fmt.Println("removed b")
+    }
+    
+    // Restore to initial state.
+    cf.Reset()
+}
+```
+
+## Classic Bloom Filter
+
+A classic Bloom filter is a special case of a Stable Bloom Filter whose eviction rate is zero and cell size is one. We call this special case an Unstable Bloom Filter. Because cells require more memory overhead, this package also provides two bitset-based Bloom filter variations. The first variation is the traditional implementation consisting of a single bit array. The second implementation is a partitioned approach which uniformly distributes the probability of false positives across all elements.
+
+Bloom filters have a limited capacity, depending on the configured size. Once all bits are set, the probability of a false positive is 1. However, traditional Bloom filters cannot return a false negative.
+
+A Bloom filter is ideal for cases where the data set is known a priori because the false-positive rate can be configured by the size and number of hash functions.
+
+### Usage
+
+```go
+package main
+
+import (
+    "fmt"
+    "github.com/tylertreat/BoomFilters"
+)
+
+func main() {
+    // We could also use boom.NewUnstableBloomFilter or boom.NewPartitionedBloomFilter.
+    bf := boom.NewBloomFilter(1000, 0.01)
+    
+    bf.Add([]byte(`a`))
+    if bf.Test([]byte(`a`)) {
+        fmt.Println("contains a")
+    }
+    
+    if !bf.TestAndAdd([]byte(`b`)) {
+        fmt.Println("doesn't contain b")
+    }
+    
+    if bf.Test([]byte(`b`)) {
+        fmt.Println("now it contains b!")
+    }
+    
+    // Restore to initial state.
+    bf.Reset()
+}
+```
+
+## Count-Min Sketch
+
+This is an implementation of a Count-Min Sketch as described by Cormode and Muthukrishnan in [An Improved Data Stream Summary: The Count-Min Sketch and its Applications](http://dimacs.rutgers.edu/~graham/pubs/papers/cm-full.pdf).
+
+A Count-Min Sketch (CMS) is a probabilistic data structure which approximates the frequency of events in a data stream. Unlike a hash map, a CMS uses sub-linear space at the expense of a configurable error factor. Similar to Counting Bloom filters, items are hashed to a series of buckets, which increment a counter. The frequency of an item is estimated by taking the minimum of each of the item's respective counter values.
+
+Count-Min Sketches are useful for counting the frequency of events in massive data sets or unbounded streams online. In these situations, storing the entire data set or allocating counters for every event in memory is impractical. It may be possible for offline processing, but real-time processing requires fast, space-efficient solutions like the CMS. For approximating set cardinality, refer to the HyperLogLog.
+
+### Usage
+
+```go
+package main
+
+import (
+    "fmt"
+    "github.com/tylertreat/BoomFilters"
+)
+
+func main() {
+    cms := boom.NewCountMinSketch(0.001, 0.99)
+    
+    cms.Add([]byte(`alice`)).Add([]byte(`bob`)).Add([]byte(`bob`)).Add([]byte(`frank`))
+    fmt.Println("frequency of alice", cms.Count([]byte(`alice`)))
+    fmt.Println("frequency of bob", cms.Count([]byte(`bob`)))
+    fmt.Println("frequency of frank", cms.Count([]byte(`frank`)))
+    
+
+    // Serialization example
+    buf := new(bytes.Buffer)
+    n, err := cms.WriteDataTo(buf)
+    if err != nil {
+       fmt.Println(err, n)
+    }
+
+    // Restore to initial state.
+    cms.Reset()
+
+    newCMS := boom.NewCountMinSketch(0.001, 0.99)
+    n, err = newCMS.ReadDataFrom(buf)
+    if err != nil {
+       fmt.Println(err, n)
+    }
+
+    fmt.Println("frequency of frank", newCMS.Count([]byte(`frank`)))
+
+   
+}
+```
+
+## Top-K
+
+Top-K uses a Count-Min Sketch and min-heap to track the top-k most frequent elements in a stream.
+
+### Usage
+
+```go
+package main
+
+import (
+    "fmt"
+    "github.com/tylertreat/BoomFilters"
+)
+
+func main() {
+	topk := NewTopK(0.001, 0.99, 5)
+
+	topk.Add([]byte(`bob`)).Add([]byte(`bob`)).Add([]byte(`bob`))
+	topk.Add([]byte(`tyler`)).Add([]byte(`tyler`)).Add([]byte(`tyler`)).Add([]byte(`tyler`))
+	topk.Add([]byte(`fred`))
+	topk.Add([]byte(`alice`)).Add([]byte(`alice`)).Add([]byte(`alice`)).Add([]byte(`alice`))
+	topk.Add([]byte(`james`))
+	topk.Add([]byte(`fred`))
+	topk.Add([]byte(`sara`)).Add([]byte(`sara`))
+	topk.Add([]byte(`bill`))
+
+	for i, element := range topk.Elements() {
+		fmt.Println(i, string(element.Data), element.Freq)
+	}
+	
+	// Restore to initial state.
+	topk.Reset()
+}
+```
+
+## HyperLogLog
+
+This is an implementation of HyperLogLog as described by Flajolet, Fusy, Gandouet, and Meunier in [HyperLogLog: the analysis of a near-optimal cardinality estimation algorithm](http://algo.inria.fr/flajolet/Publications/FlFuGaMe07.pdf).
+
+HyperLogLog is a probabilistic algorithm which approximates the number of distinct elements in a multiset. It works by hashing values and calculating the maximum number of leading zeros in the binary representation of each hash. If the maximum number of leading zeros is n, the estimated number of distinct elements in the set is 2^n. To minimize variance, the multiset is split into a configurable number of registers, the maximum number of leading zeros is calculated in the numbers in each register, and a harmonic mean is used to combine the estimates.
+
+For large or unbounded data sets, calculating the exact cardinality is impractical. HyperLogLog uses a fraction of the memory while providing an accurate approximation.
+
+This implementation was [originally written by Eric Lesh](https://github.com/eclesh/hyperloglog). Some small changes and additions have been made, including a way to construct a HyperLogLog optimized for a particular relative accuracy and adding FNV hashing. For counting element frequency, refer to the Count-Min Sketch.
+
+### Usage
+
+```go
+package main
+
+import (
+    "fmt"
+    "github.com/tylertreat/BoomFilters"
+)
+
+func main() {
+    hll, err := boom.NewDefaultHyperLogLog(0.1)
+    if err != nil {
+        panic(err)
+    }
+    
+    hll.Add([]byte(`alice`)).Add([]byte(`bob`)).Add([]byte(`bob`)).Add([]byte(`frank`))
+    fmt.Println("count", hll.Count())
+
+    // Serialization example
+    buf := new(bytes.Buffer)
+    _, err := hll.WriteDataTo(buf)
+    if err != nil {
+       fmt.Println(err)
+    }
+    
+    // Restore to initial state.
+    hll.Reset()
+
+    newHll, err := boom.NewDefaultHyperLogLog(0.1)
+    if err != nil {
+       fmt.Println(err)
+    }
+
+    _, err := newHll.ReadDataFrom(buf)
+    if err != nil {
+       fmt.Println(err)
+    }
+    fmt.Println("count", newHll.Count())
+
+}
+```
+
+## MinHash
+
+This is a variation of the technique for estimating similarity between two sets as presented by Broder in [On the resemblance and containment of documents](http://gatekeeper.dec.com/ftp/pub/dec/SRC/publications/broder/positano-final-wpnums.pdf).
+
+MinHash is a probabilistic algorithm which can be used to cluster or compare documents by splitting the corpus into a bag of words. MinHash returns the approximated similarity ratio of the two bags. The similarity is less accurate for very small bags of words.
+
+### Usage
+
+```go
+package main
+
+import (
+    "fmt"
+    "github.com/tylertreat/BoomFilters"
+)
+
+func main() {
+    bag1 := []string{"bill", "alice", "frank", "bob", "sara", "tyler", "james"}
+	bag2 := []string{"bill", "alice", "frank", "bob", "sara"}
+	
+	fmt.Println("similarity", boom.MinHash(bag1, bag2))
+}
+```
+
+## References
+
+- [Approximately Detecting Duplicates for Streaming Data using Stable Bloom Filters](http://webdocs.cs.ualberta.ca/~drafiei/papers/DupDet06Sigmod.pdf)
+- [Scalable Bloom Filters](http://gsd.di.uminho.pt/members/cbm/ps/dbloom.pdf)
+- [The Opposite of a Bloom Filter](http://www.somethingsimilar.com/2012/05/21/the-opposite-of-a-bloom-filter/)
+- [Benchmarking Bloom Filters and Hash Functions in Go](http://zhen.org/blog/benchmarking-bloom-filters-and-hash-functions-in-go/)
+- [Summary Cache: A Scalable Wide-Area Web Cache Sharing Protocol](http://pages.cs.wisc.edu/~jussara/papers/00ton.pdf)
+- [An Improved Data Stream Summary: The Count-Min Sketch and its Applications](http://dimacs.rutgers.edu/~graham/pubs/papers/cm-full.pdf)
+- [HyperLogLog: the analysis of a near-optimal cardinality estimation algorithm](http://algo.inria.fr/flajolet/Publications/FlFuGaMe07.pdf)
+- [Package hyperloglog](https://github.com/eclesh/hyperloglog)
+- [On the resemblance and containment of documents](http://gatekeeper.dec.com/ftp/pub/dec/SRC/publications/broder/positano-final-wpnums.pdf)
+- [Cuckoo Filter: Practically Better Than Bloom](http://www.pdl.cmu.edu/PDL-FTP/FS/cuckoo-conext2014.pdf)

vendor/github.com/tylertreat/BoomFilters/boom.go

@@ -0,0 +1,83 @@
+/*
+Package boom implements probabilistic data structures for processing
+continuous, unbounded data streams. This includes Stable Bloom Filters,
+Scalable Bloom Filters, Counting Bloom Filters, Inverse Bloom Filters, several
+variants of traditional Bloom filters, HyperLogLog, Count-Min Sketch, and
+MinHash.
+
+Classic Bloom filters generally require a priori knowledge of the data set
+in order to allocate an appropriately sized bit array. This works well for
+offline processing, but online processing typically involves unbounded data
+streams. With enough data, a traditional Bloom filter "fills up", after
+which it has a false-positive probability of 1.
+
+Boom Filters are useful for situations where the size of the data set isn't
+known ahead of time. For example, a Stable Bloom Filter can be used to
+deduplicate events from an unbounded event stream with a specified upper
+bound on false positives and minimal false negatives. Alternatively, an
+Inverse Bloom Filter is ideal for deduplicating a stream where duplicate
+events are relatively close together. This results in no false positives
+and, depending on how close together duplicates are, a small probability of
+false negatives. Scalable Bloom Filters place a tight upper bound on false
+positives while avoiding false negatives but require allocating memory
+proportional to the size of the data set. Counting Bloom Filters and Cuckoo
+Filters are useful for cases which require adding and removing elements to and
+from a set.
+
+For large or unbounded data sets, calculating the exact cardinality is
+impractical. HyperLogLog uses a fraction of the memory while providing an
+accurate approximation. Similarly, Count-Min Sketch provides an efficient way
+to estimate event frequency for data streams. TopK tracks the top-k most
+frequent elements.
+
+MinHash is a probabilistic algorithm to approximate the similarity between two
+sets. This can be used to cluster or compare documents by splitting the corpus
+into a bag of words.
+*/
+package boom
+
+import (
+	"encoding/binary"
+	"hash"
+	"math"
+)
+
+const fillRatio = 0.5
+
+// Filter is a probabilistic data structure which is used to test the
+// membership of an element in a set.
+type Filter interface {
+	// Test will test for membership of the data and returns true if it is a
+	// member, false if not.
+	Test([]byte) bool
+
+	// Add will add the data to the Bloom filter. It returns the filter to
+	// allow for chaining.
+	Add([]byte) Filter
+
+	// TestAndAdd is equivalent to calling Test followed by Add. It returns
+	// true if the data is a member, false if not.
+	TestAndAdd([]byte) bool
+}
+
+// OptimalM calculates the optimal Bloom filter size, m, based on the number of
+// items and the desired rate of false positives.
+func OptimalM(n uint, fpRate float64) uint {
+	return uint(math.Ceil(float64(n) / ((math.Log(fillRatio) *
+		math.Log(1-fillRatio)) / math.Abs(math.Log(fpRate)))))
+}
+
+// OptimalK calculates the optimal number of hash functions to use for a Bloom
+// filter based on the desired rate of false positives.
+func OptimalK(fpRate float64) uint {
+	return uint(math.Ceil(math.Log2(1 / fpRate)))
+}
+
+// hashKernel returns the upper and lower base hash values from which the k
+// hashes are derived.
+func hashKernel(data []byte, hash hash.Hash64) (uint32, uint32) {
+	hash.Write(data)
+	sum := hash.Sum(nil)
+	hash.Reset()
+	return binary.BigEndian.Uint32(sum[4:8]), binary.BigEndian.Uint32(sum[0:4])
+}

vendor/github.com/tylertreat/BoomFilters/buckets.go

@@ -0,0 +1,182 @@
+package boom
+
+import (
+	"bytes"
+	"encoding/binary"
+	"io"
+)
+
+// Buckets is a fast, space-efficient array of buckets where each bucket can
+// store up to a configured maximum value.
+type Buckets struct {
+	data       []byte
+	bucketSize uint8
+	max        uint8
+	count      uint
+}
+
+// NewBuckets creates a new Buckets with the provided number of buckets where
+// each bucket is the specified number of bits.
+func NewBuckets(count uint, bucketSize uint8) *Buckets {
+	return &Buckets{
+		count:      count,
+		data:       make([]byte, (count*uint(bucketSize)+7)/8),
+		bucketSize: bucketSize,
+		max:        (1 << bucketSize) - 1,
+	}
+}
+
+// MaxBucketValue returns the maximum value that can be stored in a bucket.
+func (b *Buckets) MaxBucketValue() uint8 {
+	return b.max
+}
+
+// Count returns the number of buckets.
+func (b *Buckets) Count() uint {
+	return b.count
+}
+
+// Increment will increment the value in the specified bucket by the provided
+// delta. A bucket can be decremented by providing a negative delta. The value
+// is clamped to zero and the maximum bucket value. Returns itself to allow for
+// chaining.
+func (b *Buckets) Increment(bucket uint, delta int32) *Buckets {
+	val := int32(b.getBits(bucket*uint(b.bucketSize), uint(b.bucketSize))) + delta
+	if val > int32(b.max) {
+		val = int32(b.max)
+	} else if val < 0 {
+		val = 0
+	}
+
+	b.setBits(uint32(bucket)*uint32(b.bucketSize), uint32(b.bucketSize), uint32(val))
+	return b
+}
+
+// Set will set the bucket value. The value is clamped to zero and the maximum
+// bucket value. Returns itself to allow for chaining.
+func (b *Buckets) Set(bucket uint, value uint8) *Buckets {
+	if value > b.max {
+		value = b.max
+	}
+
+	b.setBits(uint32(bucket)*uint32(b.bucketSize), uint32(b.bucketSize), uint32(value))
+	return b
+}
+
+// Get returns the value in the specified bucket.
+func (b *Buckets) Get(bucket uint) uint32 {
+	return b.getBits(bucket*uint(b.bucketSize), uint(b.bucketSize))
+}
+
+// Reset restores the Buckets to the original state. Returns itself to allow
+// for chaining.
+func (b *Buckets) Reset() *Buckets {
+	b.data = make([]byte, (b.count*uint(b.bucketSize)+7)/8)
+	return b
+}
+
+// getBits returns the bits at the specified offset and length.
+func (b *Buckets) getBits(offset, length uint) uint32 {
+	byteIndex := offset / 8
+	byteOffset := offset % 8
+	if byteOffset+length > 8 {
+		rem := 8 - byteOffset
+		return b.getBits(offset, rem) | (b.getBits(offset+rem, length-rem) << rem)
+	}
+	bitMask := uint32((1 << length) - 1)
+	return (uint32(b.data[byteIndex]) & (bitMask << byteOffset)) >> byteOffset
+}
+
+// setBits sets bits at the specified offset and length.
+func (b *Buckets) setBits(offset, length, bits uint32) {
+	byteIndex := offset / 8
+	byteOffset := offset % 8
+	if byteOffset+length > 8 {
+		rem := 8 - byteOffset
+		b.setBits(offset, rem, bits)
+		b.setBits(offset+rem, length-rem, bits>>rem)
+		return
+	}
+	bitMask := uint32((1 << length) - 1)
+	b.data[byteIndex] = byte(uint32(b.data[byteIndex]) & ^(bitMask << byteOffset))
+	b.data[byteIndex] = byte(uint32(b.data[byteIndex]) | ((bits & bitMask) << byteOffset))
+}
+
+// WriteTo writes a binary representation of Buckets to an i/o stream.
+// It returns the number of bytes written.
+func (b *Buckets) WriteTo(stream io.Writer) (int64, error) {
+	err := binary.Write(stream, binary.BigEndian, b.bucketSize)
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Write(stream, binary.BigEndian, b.max)
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Write(stream, binary.BigEndian, uint64(b.count))
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Write(stream, binary.BigEndian, uint64(len(b.data)))
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Write(stream, binary.BigEndian, b.data)
+	if err != nil {
+		return 0, err
+	}
+	return int64(len(b.data) + 2*binary.Size(uint8(0)) + 2*binary.Size(uint64(0))), err
+}
+
+// ReadFrom reads a binary representation of Buckets (such as might
+// have been written by WriteTo()) from an i/o stream. It returns the number
+// of bytes read.
+func (b *Buckets) ReadFrom(stream io.Reader) (int64, error) {
+	var bucketSize, max uint8
+	var count, len uint64
+	err := binary.Read(stream, binary.BigEndian, &bucketSize)
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Read(stream, binary.BigEndian, &max)
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Read(stream, binary.BigEndian, &count)
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Read(stream, binary.BigEndian, &len)
+	if err != nil {
+		return 0, err
+	}
+	data := make([]byte, len)
+	err = binary.Read(stream, binary.BigEndian, &data)
+	if err != nil {
+		return 0, err
+	}
+	b.bucketSize = bucketSize
+	b.max = max
+	b.count = uint(count)
+	b.data = data
+	return int64(int(len) + 2*binary.Size(uint8(0)) + 2*binary.Size(uint64(0))), nil
+}
+
+// GobEncode implements gob.GobEncoder interface.
+func (b *Buckets) GobEncode() ([]byte, error) {
+	var buf bytes.Buffer
+	_, err := b.WriteTo(&buf)
+	if err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}
+
+// GobDecode implements gob.GobDecoder interface.
+func (b *Buckets) GobDecode(data []byte) error {
+	buf := bytes.NewBuffer(data)
+	_, err := b.ReadFrom(buf)
+
+	return err
+}

vendor/github.com/tylertreat/BoomFilters/classic.go

@@ -0,0 +1,199 @@
+package boom
+
+import (
+	"bytes"
+	"encoding/binary"
+	"hash"
+	"hash/fnv"
+	"io"
+	"math"
+)
+
+// BloomFilter implements a classic Bloom filter. A Bloom filter has a non-zero
+// probability of false positives and a zero probability of false negatives.
+type BloomFilter struct {
+	buckets *Buckets    // filter data
+	hash    hash.Hash64 // hash function (kernel for all k functions)
+	m       uint        // filter size
+	k       uint        // number of hash functions
+	count   uint        // number of items added
+}
+
+// NewBloomFilter creates a new Bloom filter optimized to store n items with a
+// specified target false-positive rate.
+func NewBloomFilter(n uint, fpRate float64) *BloomFilter {
+	m := OptimalM(n, fpRate)
+	return &BloomFilter{
+		buckets: NewBuckets(m, 1),
+		hash:    fnv.New64(),
+		m:       m,
+		k:       OptimalK(fpRate),
+	}
+}
+
+// Capacity returns the Bloom filter capacity, m.
+func (b *BloomFilter) Capacity() uint {
+	return b.m
+}
+
+// K returns the number of hash functions.
+func (b *BloomFilter) K() uint {
+	return b.k
+}
+
+// Count returns the number of items added to the filter.
+func (b *BloomFilter) Count() uint {
+	return b.count
+}
+
+// EstimatedFillRatio returns the current estimated ratio of set bits.
+func (b *BloomFilter) EstimatedFillRatio() float64 {
+	return 1 - math.Exp((-float64(b.count)*float64(b.k))/float64(b.m))
+}
+
+// FillRatio returns the ratio of set bits.
+func (b *BloomFilter) FillRatio() float64 {
+	sum := uint32(0)
+	for i := uint(0); i < b.buckets.Count(); i++ {
+		sum += b.buckets.Get(i)
+	}
+	return float64(sum) / float64(b.m)
+}
+
+// Test will test for membership of the data and returns true if it is a
+// member, false if not. This is a probabilistic test, meaning there is a
+// non-zero probability of false positives but a zero probability of false
+// negatives.
+func (b *BloomFilter) Test(data []byte) bool {
+	lower, upper := hashKernel(data, b.hash)
+
+	// If any of the K bits are not set, then it's not a member.
+	for i := uint(0); i < b.k; i++ {
+		if b.buckets.Get((uint(lower)+uint(upper)*i)%b.m) == 0 {
+			return false
+		}
+	}
+
+	return true
+}
+
+// Add will add the data to the Bloom filter. It returns the filter to allow
+// for chaining.
+func (b *BloomFilter) Add(data []byte) Filter {
+	lower, upper := hashKernel(data, b.hash)
+
+	// Set the K bits.
+	for i := uint(0); i < b.k; i++ {
+		b.buckets.Set((uint(lower)+uint(upper)*i)%b.m, 1)
+	}
+
+	b.count++
+	return b
+}
+
+// TestAndAdd is equivalent to calling Test followed by Add. It returns true if
+// the data is a member, false if not.
+func (b *BloomFilter) TestAndAdd(data []byte) bool {
+	lower, upper := hashKernel(data, b.hash)
+	member := true
+
+	// If any of the K bits are not set, then it's not a member.
+	for i := uint(0); i < b.k; i++ {
+		idx := (uint(lower) + uint(upper)*i) % b.m
+		if b.buckets.Get(idx) == 0 {
+			member = false
+		}
+		b.buckets.Set(idx, 1)
+	}
+
+	b.count++
+	return member
+}
+
+// Reset restores the Bloom filter to its original state. It returns the filter
+// to allow for chaining.
+func (b *BloomFilter) Reset() *BloomFilter {
+	b.buckets.Reset()
+	return b
+}
+
+// SetHash sets the hashing function used in the filter.
+// For the effect on false positive rates see: https://github.com/tylertreat/BoomFilters/pull/1
+func (b *BloomFilter) SetHash(h hash.Hash64) {
+	b.hash = h
+}
+
+// WriteTo writes a binary representation of the BloomFilter to an i/o stream.
+// It returns the number of bytes written.
+func (b *BloomFilter) WriteTo(stream io.Writer) (int64, error) {
+	err := binary.Write(stream, binary.BigEndian, uint64(b.count))
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Write(stream, binary.BigEndian, uint64(b.m))
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Write(stream, binary.BigEndian, uint64(b.k))
+	if err != nil {
+		return 0, err
+	}
+
+	writtenSize, err := b.buckets.WriteTo(stream)
+	if err != nil {
+		return 0, err
+	}
+
+	return writtenSize + int64(3 * binary.Size(uint64(0))), err
+}
+
+// ReadFrom reads a binary representation of BloomFilter (such as might
+// have been written by WriteTo()) from an i/o stream. It returns the number
+// of bytes read.
+func (b *BloomFilter) ReadFrom(stream io.Reader) (int64, error) {
+	var count, m, k uint64
+	var buckets Buckets
+
+	err := binary.Read(stream, binary.BigEndian, &count)
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Read(stream, binary.BigEndian, &m)
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Read(stream, binary.BigEndian, &k)
+	if err != nil {
+		return 0, err
+	}
+
+	readSize, err := buckets.ReadFrom(stream)
+	if err != nil {
+		return 0, err
+	}
+
+	b.count = uint(count)
+	b.m = uint(m)
+	b.k = uint(k)
+	b.buckets = &buckets
+	return readSize + int64(3 * binary.Size(uint64(0))), nil
+}
+
+// GobEncode implements gob.GobEncoder interface.
+func (b *BloomFilter) GobEncode() ([]byte, error) {
+	var buf bytes.Buffer
+	_, err := b.WriteTo(&buf)
+	if err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}
+
+// GobDecode implements gob.GobDecoder interface.
+func (b *BloomFilter) GobDecode(data []byte) error {
+	buf := bytes.NewBuffer(data)
+	_, err := b.ReadFrom(buf)
+
+	return err
+}

vendor/github.com/tylertreat/BoomFilters/counting.go

@@ -0,0 +1,158 @@
+package boom
+
+import (
+	"hash"
+	"hash/fnv"
+)
+
+// CountingBloomFilter implements a Counting Bloom Filter as described by Fan,
+// Cao, Almeida, and Broder in Summary Cache: A Scalable Wide-Area Web Cache
+// Sharing Protocol:
+//
+// http://pages.cs.wisc.edu/~jussara/papers/00ton.pdf
+//
+// A Counting Bloom Filter (CBF) provides a way to remove elements by using an
+// array of n-bit buckets. When an element is added, the respective buckets are
+// incremented. To remove an element, the respective buckets are decremented. A
+// query checks that each of the respective buckets are non-zero. Because CBFs
+// allow elements to be removed, they introduce a non-zero probability of false
+// negatives in addition to the possibility of false positives.
+//
+// Counting Bloom Filters are useful for cases where elements are both added
+// and removed from the data set. Since they use n-bit buckets, CBFs use
+// roughly n-times more memory than traditional Bloom filters.
+type CountingBloomFilter struct {
+	buckets     *Buckets    // filter data
+	hash        hash.Hash64 // hash function (kernel for all k functions)
+	m           uint        // number of buckets
+	k           uint        // number of hash functions
+	count       uint        // number of items in the filter
+	indexBuffer []uint      // buffer used to cache indices
+}
+
+// NewCountingBloomFilter creates a new Counting Bloom Filter optimized to
+// store n items with a specified target false-positive rate and bucket size.
+// If you don't know how many bits to use for buckets, use
+// NewDefaultCountingBloomFilter for a sensible default.
+func NewCountingBloomFilter(n uint, b uint8, fpRate float64) *CountingBloomFilter {
+	var (
+		m = OptimalM(n, fpRate)
+		k = OptimalK(fpRate)
+	)
+	return &CountingBloomFilter{
+		buckets:     NewBuckets(m, b),
+		hash:        fnv.New64(),
+		m:           m,
+		k:           k,
+		indexBuffer: make([]uint, k),
+	}
+}
+
+// NewDefaultCountingBloomFilter creates a new Counting Bloom Filter optimized
+// to store n items with a specified target false-positive rate. Buckets are
+// allocated four bits.
+func NewDefaultCountingBloomFilter(n uint, fpRate float64) *CountingBloomFilter {
+	return NewCountingBloomFilter(n, 4, fpRate)
+}
+
+// Capacity returns the Bloom filter capacity, m.
+func (c *CountingBloomFilter) Capacity() uint {
+	return c.m
+}
+
+// K returns the number of hash functions.
+func (c *CountingBloomFilter) K() uint {
+	return c.k
+}
+
+// Count returns the number of items in the filter.
+func (c *CountingBloomFilter) Count() uint {
+	return c.count
+}
+
+// Test will test for membership of the data and returns true if it is a
+// member, false if not. This is a probabilistic test, meaning there is a
+// non-zero probability of false positives and false negatives.
+func (c *CountingBloomFilter) Test(data []byte) bool {
+	lower, upper := hashKernel(data, c.hash)
+
+	// If any of the K bits are not set, then it's not a member.
+	for i := uint(0); i < c.k; i++ {
+		if c.buckets.Get((uint(lower)+uint(upper)*i)%c.m) == 0 {
+			return false
+		}
+	}
+
+	return true
+}
+
+// Add will add the data to the Bloom filter. It returns the filter to allow
+// for chaining.
+func (c *CountingBloomFilter) Add(data []byte) Filter {
+	lower, upper := hashKernel(data, c.hash)
+
+	// Set the K bits.
+	for i := uint(0); i < c.k; i++ {
+		c.buckets.Increment((uint(lower)+uint(upper)*i)%c.m, 1)
+	}
+
+	c.count++
+	return c
+}
+
+// TestAndAdd is equivalent to calling Test followed by Add. It returns true if
+// the data is a member, false if not.
+func (c *CountingBloomFilter) TestAndAdd(data []byte) bool {
+	lower, upper := hashKernel(data, c.hash)
+	member := true
+
+	// If any of the K bits are not set, then it's not a member.
+	for i := uint(0); i < c.k; i++ {
+		idx := (uint(lower) + uint(upper)*i) % c.m
+		if c.buckets.Get(idx) == 0 {
+			member = false
+		}
+		c.buckets.Increment(idx, 1)
+	}
+
+	c.count++
+	return member
+}
+
+// TestAndRemove will test for membership of the data and remove it from the
+// filter if it exists. Returns true if the data was a member, false if not.
+func (c *CountingBloomFilter) TestAndRemove(data []byte) bool {
+	lower, upper := hashKernel(data, c.hash)
+	member := true
+
+	// Set the K bits.
+	for i := uint(0); i < c.k; i++ {
+		c.indexBuffer[i] = (uint(lower) + uint(upper)*i) % c.m
+		if c.buckets.Get(c.indexBuffer[i]) == 0 {
+			member = false
+		}
+	}
+
+	if member {
+		for _, idx := range c.indexBuffer {
+			c.buckets.Increment(idx, -1)
+		}
+		c.count--
+	}
+
+	return member
+}
+
+// Reset restores the Bloom filter to its original state. It returns the filter
+// to allow for chaining.
+func (c *CountingBloomFilter) Reset() *CountingBloomFilter {
+	c.buckets.Reset()
+	c.count = 0
+	return c
+}
+
+// SetHash sets the hashing function used in the filter.
+// For the effect on false positive rates see: https://github.com/tylertreat/BoomFilters/pull/1
+func (c *CountingBloomFilter) SetHash(h hash.Hash64) {
+	c.hash = h
+}

vendor/github.com/tylertreat/BoomFilters/countmin.go

@@ -0,0 +1,217 @@
+package boom
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"hash"
+	"hash/fnv"
+	"io"
+	"math"
+)
+
+// CountMinSketch implements a Count-Min Sketch as described by Cormode and
+// Muthukrishnan in An Improved Data Stream Summary: The Count-Min Sketch and
+// its Applications:
+//
+// http://dimacs.rutgers.edu/~graham/pubs/papers/cm-full.pdf
+//
+// A Count-Min Sketch (CMS) is a probabilistic data structure which
+// approximates the frequency of events in a data stream. Unlike a hash map, a
+// CMS uses sub-linear space at the expense of a configurable error factor.
+// Similar to Counting Bloom filters, items are hashed to a series of buckets,
+// which increment a counter. The frequency of an item is estimated by taking
+// the minimum of each of the item's respective counter values.
+//
+// Count-Min Sketches are useful for counting the frequency of events in
+// massive data sets or unbounded streams online. In these situations, storing
+// the entire data set or allocating counters for every event in memory is
+// impractical. It may be possible for offline processing, but real-time
+// processing requires fast, space-efficient solutions like the CMS. For
+// approximating set cardinality, refer to the HyperLogLog.
+type CountMinSketch struct {
+	matrix  [][]uint64  // count matrix
+	width   uint        // matrix width
+	depth   uint        // matrix depth
+	count   uint64      // number of items added
+	epsilon float64     // relative-accuracy factor
+	delta   float64     // relative-accuracy probability
+	hash    hash.Hash64 // hash function (kernel for all depth functions)
+}
+
+// NewCountMinSketch creates a new Count-Min Sketch whose relative accuracy is
+// within a factor of epsilon with probability delta. Both of these parameters
+// affect the space and time complexity.
+func NewCountMinSketch(epsilon, delta float64) *CountMinSketch {
+	var (
+		width  = uint(math.Ceil(math.E / epsilon))
+		depth  = uint(math.Ceil(math.Log(1 / delta)))
+		matrix = make([][]uint64, depth)
+	)
+
+	for i := uint(0); i < depth; i++ {
+		matrix[i] = make([]uint64, width)
+	}
+
+	return &CountMinSketch{
+		matrix:  matrix,
+		width:   width,
+		depth:   depth,
+		epsilon: epsilon,
+		delta:   delta,
+		hash:    fnv.New64(),
+	}
+}
+
+// Epsilon returns the relative-accuracy factor, epsilon.
+func (c *CountMinSketch) Epsilon() float64 {
+	return c.epsilon
+}
+
+// Delta returns the relative-accuracy probability, delta.
+func (c *CountMinSketch) Delta() float64 {
+	return c.delta
+}
+
+// TotalCount returns the number of items added to the sketch.
+func (c *CountMinSketch) TotalCount() uint64 {
+	return c.count
+}
+
+// Add will add the data to the set. Returns the CountMinSketch to allow for
+// chaining.
+func (c *CountMinSketch) Add(data []byte) *CountMinSketch {
+	lower, upper := hashKernel(data, c.hash)
+
+	// Increment count in each row.
+	for i := uint(0); i < c.depth; i++ {
+		c.matrix[i][(uint(lower)+uint(upper)*i)%c.width]++
+	}
+
+	c.count++
+	return c
+}
+
+// Count returns the approximate count for the specified item, correct within
+// epsilon * total count with a probability of delta.
+func (c *CountMinSketch) Count(data []byte) uint64 {
+	var (
+		lower, upper = hashKernel(data, c.hash)
+		count        = uint64(math.MaxUint64)
+	)
+
+	for i := uint(0); i < c.depth; i++ {
+		count = uint64(math.Min(float64(count),
+			float64(c.matrix[i][(uint(lower)+uint(upper)*i)%c.width])))
+	}
+
+	return count
+}
+
+// Merge combines this CountMinSketch with another. Returns an error if the
+// matrix width and depth are not equal.
+func (c *CountMinSketch) Merge(other *CountMinSketch) error {
+	if c.depth != other.depth {
+		return errors.New("matrix depth must match")
+	}
+
+	if c.width != other.width {
+		return errors.New("matrix width must match")
+	}
+
+	for i := uint(0); i < c.depth; i++ {
+		for j := uint(0); j < c.width; j++ {
+			c.matrix[i][j] += other.matrix[i][j]
+		}
+	}
+
+	c.count += other.count
+	return nil
+}
+
+// Reset restores the CountMinSketch to its original state. It returns itself
+// to allow for chaining.
+func (c *CountMinSketch) Reset() *CountMinSketch {
+	matrix := make([][]uint64, c.depth)
+	for i := uint(0); i < c.depth; i++ {
+		matrix[i] = make([]uint64, c.width)
+	}
+
+	c.matrix = matrix
+	c.count = 0
+	return c
+}
+
+// SetHash sets the hashing function used.
+func (c *CountMinSketch) SetHash(h hash.Hash64) {
+	c.hash = h
+}
+
+// WriteDataTo writes a binary representation of the CMS data to
+// an io stream. It returns the number of bytes written and error
+func (c *CountMinSketch) WriteDataTo(stream io.Writer) (int, error) {
+	buf := new(bytes.Buffer)
+	// serialize epsilon and delta as cms configuration check
+	err := binary.Write(buf, binary.LittleEndian, c.epsilon)
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Write(buf, binary.LittleEndian, c.delta)
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Write(buf, binary.LittleEndian, c.count)
+	if err != nil {
+		return 0, err
+	}
+	// encode matrix
+	for i := range c.matrix {
+		err = binary.Write(buf, binary.LittleEndian, c.matrix[i])
+		if err != nil {
+			return 0, err
+		}
+	}
+
+	return stream.Write(buf.Bytes())
+}
+
+// ReadDataFrom reads a binary representation of the CMS data written
+// by WriteDataTo() from io stream. It returns the number of bytes read
+// and error
+// If serialized CMS configuration is different it returns error with expected params
+func (c *CountMinSketch) ReadDataFrom(stream io.Reader) (int, error) {
+	var (
+		count          uint64
+		epsilon, delta float64
+	)
+
+	err := binary.Read(stream, binary.LittleEndian, &epsilon)
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Read(stream, binary.LittleEndian, &delta)
+	if err != nil {
+		return 0, err
+	}
+
+	// check if serialized and target cms configurations are same
+	if c.epsilon != epsilon || c.delta != delta {
+		return 0, fmt.Errorf("expected cms values for epsilon %f and delta %f", epsilon, delta)
+	}
+
+	err = binary.Read(stream, binary.LittleEndian, &count)
+	if err != nil {
+		return 0, err
+	}
+
+	for i := uint(0); i < uint(c.depth); i++ {
+		err = binary.Read(stream, binary.LittleEndian, c.matrix[i])
+	}
+	// count size of matrix and count
+	size := int(c.depth*c.width)*binary.Size(uint64(0)) + binary.Size(count) + 2*binary.Size(float64(0))
+
+	c.count = count
+
+	return size, err
+}

vendor/github.com/tylertreat/BoomFilters/cuckoo.go

@@ -0,0 +1,269 @@
+package boom
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"hash"
+	"hash/fnv"
+	"math"
+	"math/rand"
+)
+
+// maxNumKicks is the maximum number of relocations to attempt when inserting
+// an element before considering the filter full.
+const maxNumKicks = 500
+
+// bucket consists of a set of []byte entries.
+type bucket [][]byte
+
+// contains indicates if the given fingerprint is contained in one of the
+// bucket's entries.
+func (b bucket) contains(f []byte) bool {
+	return b.indexOf(f) != -1
+}
+
+// indexOf returns the entry index of the given fingerprint or -1 if it's not
+// in the bucket.
+func (b bucket) indexOf(f []byte) int {
+	for i, fingerprint := range b {
+		if bytes.Equal(f, fingerprint) {
+			return i
+		}
+	}
+	return -1
+}
+
+// getEmptyEntry returns the index of the next available entry in the bucket or
+// an error if it's full.
+func (b bucket) getEmptyEntry() (int, error) {
+	for i, fingerprint := range b {
+		if fingerprint == nil {
+			return i, nil
+		}
+	}
+	return -1, errors.New("full")
+}
+
+// CuckooFilter implements a Cuckoo Bloom filter as described by Andersen,
+// Kaminsky, and Mitzenmacher in Cuckoo Filter: Practically Better Than Bloom:
+//
+// http://www.pdl.cmu.edu/PDL-FTP/FS/cuckoo-conext2014.pdf
+//
+// A Cuckoo Filter is a Bloom filter variation which provides support for
+// removing elements without significantly degrading space and performance. It
+// works by using a cuckoo hashing scheme for inserting items. Instead of
+// storing the elements themselves, it stores their fingerprints which also
+// allows for item removal without false negatives (if you don't attempt to
+// remove an item not contained in the filter).
+//
+// For applications that store many items and target moderately low
+// false-positive rates, cuckoo filters have lower space overhead than
+// space-optimized Bloom filters.
+type CuckooFilter struct {
+	buckets []bucket
+	hash    hash.Hash32 // hash function (used for fingerprint and hash)
+	m       uint        // number of buckets
+	b       uint        // number of entries per bucket
+	f       uint        // length of fingerprints (in bytes)
+	count   uint        // number of items in the filter
+	n       uint        // filter capacity
+}
+
+// NewCuckooFilter creates a new Cuckoo Bloom filter optimized to store n items
+// with a specified target false-positive rate.
+func NewCuckooFilter(n uint, fpRate float64) *CuckooFilter {
+	var (
+		b       = uint(4)
+		f       = calculateF(b, fpRate)
+		m       = power2(n / uint(f) * 8)
+		buckets = make([]bucket, m)
+	)
+
+	for i := uint(0); i < m; i++ {
+		buckets[i] = make(bucket, b)
+	}
+
+	return &CuckooFilter{
+		buckets: buckets,
+		hash:    fnv.New32(),
+		m:       m,
+		b:       b,
+		f:       uint(f),
+		n:       n,
+	}
+}
+
+// Buckets returns the number of buckets.
+func (c *CuckooFilter) Buckets() uint {
+	return c.m
+}
+
+// Capacity returns the number of items the filter can store.
+func (c *CuckooFilter) Capacity() uint {
+	return c.n
+}
+
+// Count returns the number of items in the filter.
+func (c *CuckooFilter) Count() uint {
+	return c.count
+}
+
+// Test will test for membership of the data and returns true if it is a
+// member, false if not. This is a probabilistic test, meaning there is a
+// non-zero probability of false positives.
+func (c *CuckooFilter) Test(data []byte) bool {
+	i1, i2, f := c.components(data)
+
+	// If either bucket contains f, it's a member.
+	return c.buckets[i1%c.m].contains(f) || c.buckets[i2%c.m].contains(f)
+}
+
+// Add will add the data to the Cuckoo Filter. It returns an error if the
+// filter is full. If the filter is full, an item is removed to make room for
+// the new item. This introduces a possibility for false negatives. To avoid
+// this, use Count and Capacity to check if the filter is full before adding an
+// item.
+func (c *CuckooFilter) Add(data []byte) error {
+	return c.add(c.components(data))
+}
+
+// TestAndAdd is equivalent to calling Test followed by Add. It returns true if
+// the data is a member, false if not. An error is returned if the filter is
+// full. If the filter is full, an item is removed to make room for the new
+// item. This introduces a possibility for false negatives. To avoid this, use
+// Count and Capacity to check if the filter is full before adding an item.
+func (c *CuckooFilter) TestAndAdd(data []byte) (bool, error) {
+	i1, i2, f := c.components(data)
+
+	// If either bucket contains f, it's a member.
+	if c.buckets[i1%c.m].contains(f) || c.buckets[i2%c.m].contains(f) {
+		return true, nil
+	}
+
+	return false, c.add(i1, i2, f)
+}
+
+// TestAndRemove will test for membership of the data and remove it from the
+// filter if it exists. Returns true if the data was a member, false if not.
+func (c *CuckooFilter) TestAndRemove(data []byte) bool {
+	i1, i2, f := c.components(data)
+
+	// Try to remove from bucket[i1].
+	b1 := c.buckets[i1%c.m]
+	if idx := b1.indexOf(f); idx != -1 {
+		b1[idx] = nil
+		c.count--
+		return true
+	}
+
+	// Try to remove from bucket[i2].
+	b2 := c.buckets[i2%c.m]
+	if idx := b2.indexOf(f); idx != -1 {
+		b2[idx] = nil
+		c.count--
+		return true
+	}
+
+	return false
+}
+
+// Reset restores the Bloom filter to its original state. It returns the filter
+// to allow for chaining.
+func (c *CuckooFilter) Reset() *CuckooFilter {
+	buckets := make([]bucket, c.m)
+	for i := uint(0); i < c.m; i++ {
+		buckets[i] = make(bucket, c.b)
+	}
+	c.buckets = buckets
+	c.count = 0
+	return c
+}
+
+// add will insert the fingerprint into the filter returning an error if the
+// filter is full.
+func (c *CuckooFilter) add(i1, i2 uint, f []byte) error {
+	// Try to insert into bucket[i1].
+	b1 := c.buckets[i1%c.m]
+	if idx, err := b1.getEmptyEntry(); err == nil {
+		b1[idx] = f
+		c.count++
+		return nil
+	}
+
+	// Try to insert into bucket[i2].
+	b2 := c.buckets[i2%c.m]
+	if idx, err := b2.getEmptyEntry(); err == nil {
+		b2[idx] = f
+		c.count++
+		return nil
+	}
+
+	// Must relocate existing items.
+	i := i1
+	for n := 0; n < maxNumKicks; n++ {
+		bucketIdx := i % c.m
+		entryIdx := rand.Intn(int(c.b))
+		f, c.buckets[bucketIdx][entryIdx] = c.buckets[bucketIdx][entryIdx], f
+		i = i ^ uint(binary.BigEndian.Uint32(c.computeHash(f)))
+		b := c.buckets[i%c.m]
+		if idx, err := b.getEmptyEntry(); err == nil {
+			b[idx] = f
+			c.count++
+			return nil
+		}
+	}
+
+	return errors.New("full")
+}
+
+// components returns the two hash values used to index into the buckets and
+// the fingerprint for the given element.
+func (c *CuckooFilter) components(data []byte) (uint, uint, []byte) {
+	var (
+		hash = c.computeHash(data)
+		f    = hash[0:c.f]
+		i1   = uint(binary.BigEndian.Uint32(hash))
+		i2   = i1 ^ uint(binary.BigEndian.Uint32(c.computeHash(f)))
+	)
+
+	return i1, i2, f
+}
+
+// computeHash returns a 32-bit hash value for the given data.
+func (c *CuckooFilter) computeHash(data []byte) []byte {
+	c.hash.Write(data)
+	hash := c.hash.Sum(nil)
+	c.hash.Reset()
+	return hash
+}
+
+// SetHash sets the hashing function used in the filter.
+// For the effect on false positive rates see: https://github.com/tylertreat/BoomFilters/pull/1
+func (c *CuckooFilter) SetHash(h hash.Hash32) {
+	c.hash = h
+}
+
+// calculateF returns the optimal fingerprint length in bytes for the given
+// bucket size and false-positive rate epsilon.
+func calculateF(b uint, epsilon float64) uint {
+	f := uint(math.Ceil(math.Log(2 * float64(b) / epsilon)))
+	f = f / 8
+	if f <= 0 {
+		f = 1
+	}
+	return f
+}
+
+// power2 calculates the next power of two for the given value.
+func power2(x uint) uint {
+	x--
+	x |= x >> 1
+	x |= x >> 2
+	x |= x >> 4
+	x |= x >> 8
+	x |= x >> 16
+	x |= x >> 32
+	x++
+	return x
+}

vendor/github.com/tylertreat/BoomFilters/deletable.go

@@ -0,0 +1,168 @@
+package boom
+
+import (
+	"hash"
+	"hash/fnv"
+)
+
+// DeletableBloomFilter implements a Deletable Bloom Filter as described by
+// Rothenberg, Macapuna, Verdi, Magalhaes in The Deletable Bloom filter - A new
+// member of the Bloom family:
+//
+// http://arxiv.org/pdf/1005.0352.pdf
+//
+// A Deletable Bloom Filter compactly stores information on collisions when
+// inserting elements. This information is used to determine if elements are
+// deletable. This design enables false-negative-free deletions at a fraction
+// of the cost in memory consumption.
+//
+// Deletable Bloom Filters are useful for cases which require removing elements
+// but cannot allow false negatives. This means they can be safely swapped in
+// place of traditional Bloom filters.
+type DeletableBloomFilter struct {
+	buckets     *Buckets    // filter data
+	collisions  *Buckets    // filter collision data
+	hash        hash.Hash64 // hash function (kernel for all k functions)
+	m           uint        // filter size
+	regionSize  uint        // number of bits in a region
+	k           uint        // number of hash functions
+	count       uint        // number of items added
+	indexBuffer []uint      // buffer used to cache indices
+}
+
+// NewDeletableBloomFilter creates a new DeletableBloomFilter optimized to
+// store n items with a specified target false-positive rate. The r value
+// determines the number of bits to use to store collision information. This
+// controls the deletability of an element. Refer to the paper for selecting an
+// optimal value.
+func NewDeletableBloomFilter(n, r uint, fpRate float64) *DeletableBloomFilter {
+	var (
+		m = OptimalM(n, fpRate)
+		k = OptimalK(fpRate)
+	)
+	return &DeletableBloomFilter{
+		buckets:     NewBuckets(m-r, 1),
+		collisions:  NewBuckets(r, 1),
+		hash:        fnv.New64(),
+		m:           m - r,
+		regionSize:  (m - r) / r,
+		k:           k,
+		indexBuffer: make([]uint, k),
+	}
+}
+
+// Capacity returns the Bloom filter capacity, m.
+func (d *DeletableBloomFilter) Capacity() uint {
+	return d.m
+}
+
+// K returns the number of hash functions.
+func (d *DeletableBloomFilter) K() uint {
+	return d.k
+}
+
+// Count returns the number of items added to the filter.
+func (d *DeletableBloomFilter) Count() uint {
+	return d.count
+}
+
+// Test will test for membership of the data and returns true if it is a
+// member, false if not. This is a probabilistic test, meaning there is a
+// non-zero probability of false positives but a zero probability of false
+// negatives.
+func (d *DeletableBloomFilter) Test(data []byte) bool {
+	lower, upper := hashKernel(data, d.hash)
+
+	// If any of the K bits are not set, then it's not a member.
+	for i := uint(0); i < d.k; i++ {
+		if d.buckets.Get((uint(lower)+uint(upper)*i)%d.m) == 0 {
+			return false
+		}
+	}
+
+	return true
+}
+
+// Add will add the data to the Bloom filter. It returns the filter to allow
+// for chaining.
+func (d *DeletableBloomFilter) Add(data []byte) Filter {
+	lower, upper := hashKernel(data, d.hash)
+
+	// Set the K bits.
+	for i := uint(0); i < d.k; i++ {
+		idx := (uint(lower) + uint(upper)*i) % d.m
+		if d.buckets.Get(idx) != 0 {
+			// Collision, set corresponding region bit.
+			d.collisions.Set(idx/d.regionSize, 1)
+		} else {
+			d.buckets.Set(idx, 1)
+		}
+	}
+
+	d.count++
+	return d
+}
+
+// TestAndAdd is equivalent to calling Test followed by Add. It returns true if
+// the data is a member, false if not.
+func (d *DeletableBloomFilter) TestAndAdd(data []byte) bool {
+	lower, upper := hashKernel(data, d.hash)
+	member := true
+
+	// If any of the K bits are not set, then it's not a member.
+	for i := uint(0); i < d.k; i++ {
+		idx := (uint(lower) + uint(upper)*i) % d.m
+		if d.buckets.Get(idx) == 0 {
+			member = false
+		} else {
+			// Collision, set corresponding region bit.
+			d.collisions.Set(idx/d.regionSize, 1)
+		}
+		d.buckets.Set(idx, 1)
+	}
+
+	d.count++
+	return member
+}
+
+// TestAndRemove will test for membership of the data and remove it from the
+// filter if it exists. Returns true if the data was a member, false if not.
+func (d *DeletableBloomFilter) TestAndRemove(data []byte) bool {
+	lower, upper := hashKernel(data, d.hash)
+	member := true
+
+	// Set the K bits.
+	for i := uint(0); i < d.k; i++ {
+		d.indexBuffer[i] = (uint(lower) + uint(upper)*i) % d.m
+		if d.buckets.Get(d.indexBuffer[i]) == 0 {
+			member = false
+		}
+	}
+
+	if member {
+		for _, idx := range d.indexBuffer {
+			if d.collisions.Get(idx/d.regionSize) == 0 {
+				// Clear only bits located in collision-free zones.
+				d.buckets.Set(idx, 0)
+			}
+		}
+		d.count--
+	}
+
+	return member
+}
+
+// Reset restores the Bloom filter to its original state. It returns the filter
+// to allow for chaining.
+func (d *DeletableBloomFilter) Reset() *DeletableBloomFilter {
+	d.buckets.Reset()
+	d.collisions.Reset()
+	d.count = 0
+	return d
+}
+
+// SetHash sets the hashing function used in the filter.
+// For the effect on false positive rates see: https://github.com/tylertreat/BoomFilters/pull/1
+func (d *DeletableBloomFilter) SetHash(h hash.Hash64) {
+	d.hash = h
+}

vendor/github.com/tylertreat/BoomFilters/hyperloglog.go

@@ -0,0 +1,253 @@
+/*
+Original work Copyright 2013 Eric Lesh
+Modified work Copyright 2015 Tyler Treat
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+*/
+
+package boom
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"hash"
+	"hash/fnv"
+	"io"
+	"math"
+)
+
+var exp32 = math.Pow(2, 32)
+
+// HyperLogLog implements the HyperLogLog cardinality estimation algorithm as
+// described by Flajolet, Fusy, Gandouet, and Meunier in HyperLogLog: the
+// analysis of a near-optimal cardinality estimation algorithm:
+//
+// http://algo.inria.fr/flajolet/Publications/FlFuGaMe07.pdf
+//
+// HyperLogLog is a probabilistic algorithm which approximates the number of
+// distinct elements in a multiset. It works by hashing values and calculating
+// the maximum number of leading zeros in the binary representation of each
+// hash. If the maximum number of leading zeros is n, the estimated number of
+// distinct elements in the set is 2^n. To minimize variance, the multiset is
+// split into a configurable number of registers, the maximum number of leading
+// zeros is calculated in the numbers in each register, and a harmonic mean is
+// used to combine the estimates.
+//
+// For large or unbounded data sets, calculating the exact cardinality is
+// impractical. HyperLogLog uses a fraction of the memory while providing an
+// accurate approximation. For counting element frequency, refer to the
+// Count-Min Sketch.
+type HyperLogLog struct {
+	registers []uint8     // counter registers
+	m         uint        // number of registers
+	b         uint32      // number of bits to calculate register
+	alpha     float64     // bias-correction constant
+	hash      hash.Hash32 // hash function
+}
+
+// NewHyperLogLog creates a new HyperLogLog with m registers. Returns an error
+// if m isn't a power of two.
+func NewHyperLogLog(m uint) (*HyperLogLog, error) {
+	if (m & (m - 1)) != 0 {
+		return nil, errors.New("m must be a power of two")
+	}
+
+	return &HyperLogLog{
+		registers: make([]uint8, m),
+		m:         m,
+		b:         uint32(math.Ceil(math.Log2(float64(m)))),
+		alpha:     calculateAlpha(m),
+		hash:      fnv.New32(),
+	}, nil
+}
+
+// NewDefaultHyperLogLog creates a new HyperLogLog optimized for the specified
+// standard error. Returns an error if the number of registers can't be
+// calculated for the provided accuracy.
+func NewDefaultHyperLogLog(e float64) (*HyperLogLog, error) {
+	m := math.Pow(1.04/e, 2)
+	return NewHyperLogLog(uint(math.Pow(2, math.Ceil(math.Log2(m)))))
+}
+
+// Add will add the data to the set. Returns the HyperLogLog to allow for
+// chaining.
+func (h *HyperLogLog) Add(data []byte) *HyperLogLog {
+	var (
+		hash = h.calculateHash(data)
+		k    = 32 - h.b
+		r    = calculateRho(hash<<h.b, k)
+		j    = hash >> uint(k)
+	)
+
+	if r > h.registers[j] {
+		h.registers[j] = r
+	}
+
+	return h
+}
+
+// Count returns the approximated cardinality of the set.
+func (h *HyperLogLog) Count() uint64 {
+	sum := 0.0
+	m := float64(h.m)
+	for _, val := range h.registers {
+		sum += 1.0 / math.Pow(2.0, float64(val))
+	}
+	estimate := h.alpha * m * m / sum
+	if estimate <= 5.0/2.0*m {
+		// Small range correction
+		v := 0
+		for _, r := range h.registers {
+			if r == 0 {
+				v++
+			}
+		}
+		if v > 0 {
+			estimate = m * math.Log(m/float64(v))
+		}
+	} else if estimate > 1.0/30.0*exp32 {
+		// Large range correction
+		estimate = -exp32 * math.Log(1-estimate/exp32)
+	}
+	return uint64(estimate)
+}
+
+// Merge combines this HyperLogLog with another. Returns an error if the number
+// of registers in the two HyperLogLogs are not equal.
+func (h *HyperLogLog) Merge(other *HyperLogLog) error {
+	if h.m != other.m {
+		return errors.New("number of registers must match")
+	}
+
+	for j, r := range other.registers {
+		if r > h.registers[j] {
+			h.registers[j] = r
+		}
+	}
+
+	return nil
+}
+
+// Reset restores the HyperLogLog to its original state. It returns itself to
+// allow for chaining.
+func (h *HyperLogLog) Reset() *HyperLogLog {
+	h.registers = make([]uint8, h.m)
+	return h
+}
+
+// calculateHash calculates the 32-bit hash value for the provided data.
+func (h *HyperLogLog) calculateHash(data []byte) uint32 {
+	h.hash.Write(data)
+	sum := h.hash.Sum32()
+	h.hash.Reset()
+	return sum
+}
+
+// SetHash sets the hashing function used.
+func (h *HyperLogLog) SetHash(ha hash.Hash32) {
+	h.hash = ha
+}
+
+// calculateAlpha calculates the bias-correction constant alpha based on the
+// number of registers, m.
+func calculateAlpha(m uint) (result float64) {
+	switch m {
+	case 16:
+		result = 0.673
+	case 32:
+		result = 0.697
+	case 64:
+		result = 0.709
+	default:
+		result = 0.7213 / (1.0 + 1.079/float64(m))
+	}
+	return result
+}
+
+// calculateRho calculates the position of the leftmost 1-bit.
+func calculateRho(val, max uint32) uint8 {
+	r := uint32(1)
+	for val&0x80000000 == 0 && r <= max {
+		r++
+		val <<= 1
+	}
+	return uint8(r)
+}
+
+// WriteDataTo writes a binary representation of the Hll data to
+// an io stream. It returns the number of bytes written and error
+func (h *HyperLogLog) WriteDataTo(stream io.Writer) (n int, err error) {
+	buf := new(bytes.Buffer)
+	// write register number first
+	err = binary.Write(buf, binary.LittleEndian, uint64(h.m))
+	if err != nil {
+		return
+	}
+
+	err = binary.Write(buf, binary.LittleEndian, h.b)
+	if err != nil {
+		return
+	}
+
+	err = binary.Write(buf, binary.LittleEndian, h.alpha)
+	if err != nil {
+		return
+	}
+
+	err = binary.Write(buf, binary.LittleEndian, h.registers)
+	if err != nil {
+		return
+	}
+
+	n, err = stream.Write(buf.Bytes())
+	return
+}
+
+// ReadDataFrom reads a binary representation of the Hll data written
+// by WriteDataTo() from io stream. It returns the number of bytes read
+// and error.
+// If serialized Hll configuration is different it returns error with expected params
+func (h *HyperLogLog) ReadDataFrom(stream io.Reader) (int, error) {
+	var m uint64
+	// read register number first
+	err := binary.Read(stream, binary.LittleEndian, &m)
+	if err != nil {
+		return 0, err
+	}
+	// check if register number is appropriate
+	// hll register number should be same with serialized hll
+	if uint64(h.m) != m {
+		return 0, fmt.Errorf("expected hll register number %d", m)
+	}
+	// set other values
+	err = binary.Read(stream, binary.LittleEndian, &h.b)
+	if err != nil {
+		return 0, err
+	}
+
+	err = binary.Read(stream, binary.LittleEndian, &h.alpha)
+	if err != nil {
+		return 0, err
+	}
+
+	err = binary.Read(stream, binary.LittleEndian, h.registers)
+	if err != nil {
+		return 0, err
+	}
+
+	// count size of data in registers + m, b, alpha
+	size := int(h.m)*binary.Size(uint8(0)) + binary.Size(uint64(0)) + binary.Size(uint32(0)) + binary.Size(float64(0))
+
+	return size, err
+}

vendor/github.com/tylertreat/BoomFilters/inverse.go

@@ -0,0 +1,238 @@
+/*
+Original work Copyright (c) 2012 Jeff Hodges. All rights reserved.
+Modified work Copyright (c) 2015 Tyler Treat. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Jeff Hodges nor the names of this project's
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+package boom
+
+import (
+	"bytes"
+	"encoding/binary"
+	"encoding/gob"
+	"hash"
+	"hash/fnv"
+	"io"
+	"sync"
+	"sync/atomic"
+	"unsafe"
+)
+
+// InverseBloomFilter is a concurrent "inverse" Bloom filter, which is
+// effectively the opposite of a classic Bloom filter. This was originally
+// described and written by Jeff Hodges:
+//
+// http://www.somethingsimilar.com/2012/05/21/the-opposite-of-a-bloom-filter/
+//
+// The InverseBloomFilter may report a false negative but can never report a
+// false positive. That is, it may report that an item has not been seen when
+// it actually has, but it will never report an item as seen which it hasn't
+// come across. This behaves in a similar manner to a fixed-size hashmap which
+// does not handle conflicts.
+//
+// An example use case is deduplicating events while processing a stream of
+// data. Ideally, duplicate events are relatively close together.
+type InverseBloomFilter struct {
+	array    []*[]byte
+	hashPool *sync.Pool
+	capacity uint
+}
+
+// NewInverseBloomFilter creates and returns a new InverseBloomFilter with the
+// specified capacity.
+func NewInverseBloomFilter(capacity uint) *InverseBloomFilter {
+	return &InverseBloomFilter{
+		array:    make([]*[]byte, capacity),
+		hashPool: &sync.Pool{New: func() interface{} { return fnv.New32() }},
+		capacity: capacity,
+	}
+}
+
+// Test will test for membership of the data and returns true if it is a
+// member, false if not. This is a probabilistic test, meaning there is a
+// non-zero probability of false negatives but a zero probability of false
+// positives. That is, it may return false even though the data was added, but
+// it will never return true for data that hasn't been added.
+func (i *InverseBloomFilter) Test(data []byte) bool {
+	index := i.index(data)
+	indexPtr := (*unsafe.Pointer)(unsafe.Pointer(&i.array[index]))
+	val := (*[]byte)(atomic.LoadPointer(indexPtr))
+	if val == nil {
+		return false
+	}
+	return bytes.Equal(*val, data)
+}
+
+// Add will add the data to the filter. It returns the filter to allow for
+// chaining.
+func (i *InverseBloomFilter) Add(data []byte) Filter {
+	index := i.index(data)
+	i.getAndSet(index, data)
+	return i
+}
+
+// TestAndAdd is equivalent to calling Test followed by Add atomically. It
+// returns true if the data is a member, false if not.
+func (i *InverseBloomFilter) TestAndAdd(data []byte) bool {
+	oldID := i.getAndSet(i.index(data), data)
+	return bytes.Equal(oldID, data)
+}
+
+// Capacity returns the filter capacity.
+func (i *InverseBloomFilter) Capacity() uint {
+	return i.capacity
+}
+
+// getAndSet returns the data that was in the slice at the given index after
+// putting the new data in the slice at that index, atomically.
+func (i *InverseBloomFilter) getAndSet(index uint32, data []byte) []byte {
+	indexPtr := (*unsafe.Pointer)(unsafe.Pointer(&i.array[index]))
+	keyUnsafe := unsafe.Pointer(&data)
+	var oldKey []byte
+	for {
+		oldKeyUnsafe := atomic.LoadPointer(indexPtr)
+		if atomic.CompareAndSwapPointer(indexPtr, oldKeyUnsafe, keyUnsafe) {
+			oldKeyPtr := (*[]byte)(oldKeyUnsafe)
+			if oldKeyPtr != nil {
+				oldKey = *oldKeyPtr
+			}
+			break
+		}
+	}
+	return oldKey
+}
+
+// index returns the array index for the given data.
+func (i *InverseBloomFilter) index(data []byte) uint32 {
+	hash := i.hashPool.Get().(hash.Hash32)
+	hash.Write(data)
+	index := hash.Sum32() % uint32(i.capacity)
+	hash.Reset()
+	i.hashPool.Put(hash)
+	return index
+}
+
+// SetHashFactory sets the hashing function factory used in the filter.
+func (i *InverseBloomFilter) SetHashFactory(h func() hash.Hash32) {
+	i.hashPool = &sync.Pool{New: func() interface{} { return h() }}
+}
+
+// WriteTo writes a binary representation of the InverseBloomFilter to an i/o stream.
+// It returns the number of bytes written.
+func (i *InverseBloomFilter) WriteTo(stream io.Writer) (int64, error) {
+	err := binary.Write(stream, binary.BigEndian, uint64(i.capacity))
+	if err != nil {
+		return 0, err
+	}
+
+	// Dereference all pointers to []byte
+	array := make([][]byte, int(i.capacity))
+	for b := range i.array {
+		if i.array[b] != nil {
+			array[b] = *i.array[b]
+		} else {
+			array[b] = nil
+		}
+	}
+
+	// Encode array into a []byte
+	var buf bytes.Buffer
+	gob.NewEncoder(&buf).Encode(array)
+	serialized := buf.Bytes()
+
+	// Write the length of encoded slice
+	err = binary.Write(stream, binary.BigEndian, int64(len(serialized)))
+	if err != nil {
+		return 0, err
+	}
+
+	// Write the serialized bytes
+	written, err := stream.Write(serialized)
+	if err != nil {
+		return 0, err
+	}
+
+	return int64(written) + int64(2*binary.Size(uint64(0))), err
+}
+
+// ReadFrom reads a binary representation of InverseBloomFilter (such as might
+// have been written by WriteTo()) from an i/o stream. It returns the number
+// of bytes read.
+func (i *InverseBloomFilter) ReadFrom(stream io.Reader) (int64, error) {
+	var capacity, size uint64
+
+	err := binary.Read(stream, binary.BigEndian, &capacity)
+	if err != nil {
+		return 0, err
+	}
+
+	err = binary.Read(stream, binary.BigEndian, &size)
+	if err != nil {
+		return 0, err
+	}
+
+	// Read the encoded slice and decode into [][]byte
+	encoded := make([]byte, size)
+	stream.Read(encoded)
+	buf := bytes.NewBuffer(encoded)
+	dec := gob.NewDecoder(buf)
+	decoded := make([][]byte, capacity)
+	dec.Decode(&decoded)
+
+	// Create []*[]byte and point to each item in decoded
+	decodedWithPointers := make([]*[]byte, capacity)
+	for p := range decodedWithPointers {
+		if len(decoded[p]) == 0 {
+			decodedWithPointers[p] = nil
+		} else {
+			decodedWithPointers[p] = &decoded[p]
+		}
+	}
+
+	i.array = decodedWithPointers
+	i.capacity = uint(capacity)
+	return int64(len(encoded)) + int64(2*binary.Size(uint64(0))), nil
+}
+
+// GobEncode implements gob.GobEncoder interface.
+func (i *InverseBloomFilter) GobEncode() ([]byte, error) {
+	var buf bytes.Buffer
+	_, err := i.WriteTo(&buf)
+	if err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}
+
+// GobDecode implements gob.GobDecoder interface.
+func (i *InverseBloomFilter) GobDecode(data []byte) error {
+	buf := bytes.NewBuffer(data)
+	_, err := i.ReadFrom(buf)
+	return err
+}

vendor/github.com/tylertreat/BoomFilters/minhash.go

@@ -0,0 +1,104 @@
+package boom
+
+import (
+	"math"
+	"math/rand"
+)
+
+// MinHash is a variation of the technique for estimating similarity between
+// two sets as presented by Broder in On the resemblance and containment of
+// documents:
+//
+// http://gatekeeper.dec.com/ftp/pub/dec/SRC/publications/broder/positano-final-wpnums.pdf
+//
+// This can be used to cluster or compare documents by splitting the corpus
+// into a bag of words. MinHash returns the approximated similarity ratio of
+// the two bags. The similarity is less accurate for very small bags of words.
+func MinHash(bag1, bag2 []string) float32 {
+	k := len(bag1) + len(bag2)
+	hashes := make([]int, k)
+	for i := 0; i < k; i++ {
+		a := uint(rand.Int())
+		b := uint(rand.Int())
+		c := uint(rand.Int())
+		x := computeHash(a*b*c, a, b, c)
+		hashes[i] = int(x)
+	}
+
+	bitMap := bitMap(bag1, bag2)
+	minHashValues := hashBuckets(2, k)
+	minHash(bag1, 0, minHashValues, bitMap, k, hashes)
+	minHash(bag2, 1, minHashValues, bitMap, k, hashes)
+	return similarity(minHashValues, k)
+}
+
+func minHash(bag []string, bagIndex int, minHashValues [][]int,
+	bitArray map[string][]bool, k int, hashes []int) {
+	index := 0
+	for element := range bitArray {
+		for i := 0; i < k; i++ {
+			if contains(bag, element) {
+				hindex := hashes[index]
+				if hindex < minHashValues[bagIndex][index] {
+					minHashValues[bagIndex][index] = hindex
+				}
+			}
+		}
+		index++
+	}
+}
+
+func contains(bag []string, element string) bool {
+	for _, e := range bag {
+		if e == element {
+			return true
+		}
+	}
+	return false
+}
+
+func bitMap(bag1, bag2 []string) map[string][]bool {
+	bitArray := map[string][]bool{}
+	for _, element := range bag1 {
+		bitArray[element] = []bool{true, false}
+	}
+
+	for _, element := range bag2 {
+		if _, ok := bitArray[element]; ok {
+			bitArray[element] = []bool{true, true}
+		} else if _, ok := bitArray[element]; !ok {
+			bitArray[element] = []bool{false, true}
+		}
+	}
+
+	return bitArray
+}
+
+func hashBuckets(numSets, k int) [][]int {
+	minHashValues := make([][]int, numSets)
+	for i := 0; i < numSets; i++ {
+		minHashValues[i] = make([]int, k)
+	}
+
+	for i := 0; i < numSets; i++ {
+		for j := 0; j < k; j++ {
+			minHashValues[i][j] = math.MaxInt32
+		}
+	}
+	return minHashValues
+}
+
+func computeHash(x, a, b, u uint) uint {
+	return (a*x + b) >> (32 - u)
+}
+
+func similarity(minHashValues [][]int, k int) float32 {
+	identicalMinHashes := 0
+	for i := 0; i < k; i++ {
+		if minHashValues[0][i] == minHashValues[1][i] {
+			identicalMinHashes++
+		}
+	}
+
+	return (1.0 * float32(identicalMinHashes)) / float32(k)
+}

vendor/github.com/tylertreat/BoomFilters/partitioned.go

@@ -0,0 +1,263 @@
+/*
+Original work Copyright (c) 2013 zhenjl
+Modified work Copyright (c) 2015 Tyler Treat
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+*/
+
+package boom
+
+import (
+	"bytes"
+	"encoding/binary"
+	"hash"
+	"hash/fnv"
+	"io"
+	"math"
+)
+
+// PartitionedBloomFilter implements a variation of a classic Bloom filter as
+// described by Almeida, Baquero, Preguica, and Hutchison in Scalable Bloom
+// Filters:
+//
+// http://gsd.di.uminho.pt/members/cbm/ps/dbloom.pdf
+//
+// This filter works by partitioning the M-sized bit array into k slices of
+// size m = M/k bits. Each hash function produces an index over m for its
+// respective slice. Thus, each element is described by exactly k bits, meaning
+// the distribution of false positives is uniform across all elements.
+type PartitionedBloomFilter struct {
+	partitions []*Buckets  // partitioned filter data
+	hash       hash.Hash64 // hash function (kernel for all k functions)
+	m          uint        // filter size (divided into k partitions)
+	k          uint        // number of hash functions (and partitions)
+	s          uint        // partition size (m / k)
+	count      uint        // number of items added
+}
+
+// NewPartitionedBloomFilter creates a new partitioned Bloom filter optimized
+// to store n items with a specified target false-positive rate.
+func NewPartitionedBloomFilter(n uint, fpRate float64) *PartitionedBloomFilter {
+	var (
+		m          = OptimalM(n, fpRate)
+		k          = OptimalK(fpRate)
+		partitions = make([]*Buckets, k)
+		s          = uint(math.Ceil(float64(m) / float64(k)))
+	)
+
+	for i := uint(0); i < k; i++ {
+		partitions[i] = NewBuckets(s, 1)
+	}
+
+	return &PartitionedBloomFilter{
+		partitions: partitions,
+		hash:       fnv.New64(),
+		m:          m,
+		k:          k,
+		s:          s,
+	}
+}
+
+// Capacity returns the Bloom filter capacity, m.
+func (p *PartitionedBloomFilter) Capacity() uint {
+	return p.m
+}
+
+// K returns the number of hash functions.
+func (p *PartitionedBloomFilter) K() uint {
+	return p.k
+}
+
+// Count returns the number of items added to the filter.
+func (p *PartitionedBloomFilter) Count() uint {
+	return p.count
+}
+
+// EstimatedFillRatio returns the current estimated ratio of set bits.
+func (p *PartitionedBloomFilter) EstimatedFillRatio() float64 {
+	return 1 - math.Exp(-float64(p.count)/float64(p.s))
+}
+
+// FillRatio returns the average ratio of set bits across all partitions.
+func (p *PartitionedBloomFilter) FillRatio() float64 {
+	t := float64(0)
+	for i := uint(0); i < p.k; i++ {
+		sum := uint32(0)
+		for j := uint(0); j < p.partitions[i].Count(); j++ {
+			sum += p.partitions[i].Get(j)
+		}
+		t += (float64(sum) / float64(p.s))
+	}
+	return t / float64(p.k)
+}
+
+// Test will test for membership of the data and returns true if it is a
+// member, false if not. This is a probabilistic test, meaning there is a
+// non-zero probability of false positives but a zero probability of false
+// negatives. Due to the way the filter is partitioned, the probability of
+// false positives is uniformly distributed across all elements.
+func (p *PartitionedBloomFilter) Test(data []byte) bool {
+	lower, upper := hashKernel(data, p.hash)
+
+	// If any of the K partition bits are not set, then it's not a member.
+	for i := uint(0); i < p.k; i++ {
+		if p.partitions[i].Get((uint(lower)+uint(upper)*i)%p.s) == 0 {
+			return false
+		}
+	}
+
+	return true
+}
+
+// Add will add the data to the Bloom filter. It returns the filter to allow
+// for chaining.
+func (p *PartitionedBloomFilter) Add(data []byte) Filter {
+	lower, upper := hashKernel(data, p.hash)
+
+	// Set the K partition bits.
+	for i := uint(0); i < p.k; i++ {
+		p.partitions[i].Set((uint(lower)+uint(upper)*i)%p.s, 1)
+	}
+
+	p.count++
+	return p
+}
+
+// TestAndAdd is equivalent to calling Test followed by Add. It returns true if
+// the data is a member, false if not.
+func (p *PartitionedBloomFilter) TestAndAdd(data []byte) bool {
+	lower, upper := hashKernel(data, p.hash)
+	member := true
+
+	// If any of the K partition bits are not set, then it's not a member.
+	for i := uint(0); i < p.k; i++ {
+		idx := (uint(lower) + uint(upper)*i) % p.s
+		if p.partitions[i].Get(idx) == 0 {
+			member = false
+		}
+		p.partitions[i].Set(idx, 1)
+	}
+
+	p.count++
+	return member
+}
+
+// Reset restores the Bloom filter to its original state. It returns the filter
+// to allow for chaining.
+func (p *PartitionedBloomFilter) Reset() *PartitionedBloomFilter {
+	for _, partition := range p.partitions {
+		partition.Reset()
+	}
+	return p
+}
+
+// SetHash sets the hashing function used in the filter.
+// For the effect on false positive rates see: https://github.com/tylertreat/BoomFilters/pull/1
+func (p *PartitionedBloomFilter) SetHash(h hash.Hash64) {
+	p.hash = h
+}
+
+// WriteTo writes a binary representation of the PartitionedBloomFilter to an i/o stream.
+// It returns the number of bytes written.
+func (p *PartitionedBloomFilter) WriteTo(stream io.Writer) (int64, error) {
+	err := binary.Write(stream, binary.BigEndian, uint64(p.m))
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Write(stream, binary.BigEndian, uint64(p.k))
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Write(stream, binary.BigEndian, uint64(p.s))
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Write(stream, binary.BigEndian, uint64(p.count))
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Write(stream, binary.BigEndian, uint64(len(p.partitions)))
+	if err != nil {
+		return 0, err
+	}
+	var numBytes int64
+	for _, partition := range p.partitions {
+		num, err := partition.WriteTo(stream)
+		if err != nil {
+			return 0, err
+		}
+		numBytes += num
+	}
+	return numBytes + int64(5*binary.Size(uint64(0))), err
+}
+
+// ReadFrom reads a binary representation of PartitionedBloomFilter (such as might
+// have been written by WriteTo()) from an i/o stream. It returns the number
+// of bytes read.
+func (p *PartitionedBloomFilter) ReadFrom(stream io.Reader) (int64, error) {
+	var m, k, s, count, len uint64
+	err := binary.Read(stream, binary.BigEndian, &m)
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Read(stream, binary.BigEndian, &k)
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Read(stream, binary.BigEndian, &s)
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Read(stream, binary.BigEndian, &count)
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Read(stream, binary.BigEndian, &len)
+	if err != nil {
+		return 0, err
+	}
+	var numBytes int64
+	partitions := make([]*Buckets, len)
+	for i, _ := range partitions {
+		buckets := &Buckets{}
+		num, err := buckets.ReadFrom(stream)
+		if err != nil {
+			return 0, err
+		}
+		numBytes += num
+		partitions[i] = buckets
+	}
+	p.m = uint(m)
+	p.k = uint(k)
+	p.s = uint(s)
+	p.count = uint(count)
+	p.partitions = partitions
+	return numBytes + int64(5*binary.Size(uint64(0))), nil
+}
+
+// GobEncode implements gob.GobEncoder interface.
+func (p *PartitionedBloomFilter) GobEncode() ([]byte, error) {
+	var buf bytes.Buffer
+	_, err := p.WriteTo(&buf)
+	if err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}
+
+// GobDecode implements gob.GobDecoder interface.
+func (p *PartitionedBloomFilter) GobDecode(data []byte) error {
+	buf := bytes.NewBuffer(data)
+	_, err := p.ReadFrom(buf)
+
+	return err
+}

vendor/github.com/tylertreat/BoomFilters/scalable.go

@@ -0,0 +1,259 @@
+/*
+Original work Copyright (c) 2013 zhenjl
+Modified work Copyright (c) 2015 Tyler Treat
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+*/
+
+package boom
+
+import (
+	"bytes"
+	"encoding/binary"
+	"hash"
+	"io"
+	"math"
+)
+
+// ScalableBloomFilter implements a Scalable Bloom Filter as described by
+// Almeida, Baquero, Preguica, and Hutchison in Scalable Bloom Filters:
+//
+// http://gsd.di.uminho.pt/members/cbm/ps/dbloom.pdf
+//
+// A Scalable Bloom Filter dynamically adapts to the number of elements in the
+// data set while enforcing a tight upper bound on the false-positive rate.
+// This works by adding Bloom filters with geometrically decreasing
+// false-positive rates as filters become full. The tightening ratio, r,
+// controls the filter growth. The compounded probability over the whole series
+// converges to a target value, even accounting for an infinite series.
+//
+// Scalable Bloom Filters are useful for cases where the size of the data set
+// isn't known a priori and memory constraints aren't of particular concern.
+// For situations where memory is bounded, consider using Inverse or Stable
+// Bloom Filters.
+type ScalableBloomFilter struct {
+	filters []*PartitionedBloomFilter // filters with geometrically decreasing error rates
+	r       float64                   // tightening ratio
+	fp      float64                   // target false-positive rate
+	p       float64                   // partition fill ratio
+	hint    uint                      // filter size hint
+}
+
+// NewScalableBloomFilter creates a new Scalable Bloom Filter with the
+// specified target false-positive rate and tightening ratio. Use
+// NewDefaultScalableBloomFilter if you don't want to calculate these
+// parameters.
+func NewScalableBloomFilter(hint uint, fpRate, r float64) *ScalableBloomFilter {
+	s := &ScalableBloomFilter{
+		filters: make([]*PartitionedBloomFilter, 0, 1),
+		r:       r,
+		fp:      fpRate,
+		p:       fillRatio,
+		hint:    hint,
+	}
+
+	s.addFilter()
+	return s
+}
+
+// NewDefaultScalableBloomFilter creates a new Scalable Bloom Filter with the
+// specified target false-positive rate and an optimal tightening ratio.
+func NewDefaultScalableBloomFilter(fpRate float64) *ScalableBloomFilter {
+	return NewScalableBloomFilter(10000, fpRate, 0.8)
+}
+
+// Capacity returns the current Scalable Bloom Filter capacity, which is the
+// sum of the capacities for the contained series of Bloom filters.
+func (s *ScalableBloomFilter) Capacity() uint {
+	capacity := uint(0)
+	for _, bf := range s.filters {
+		capacity += bf.Capacity()
+	}
+	return capacity
+}
+
+// K returns the number of hash functions used in each Bloom filter.
+func (s *ScalableBloomFilter) K() uint {
+	// K is the same across every filter.
+	return s.filters[0].K()
+}
+
+// FillRatio returns the average ratio of set bits across every filter.
+func (s *ScalableBloomFilter) FillRatio() float64 {
+	sum := 0.0
+	for _, filter := range s.filters {
+		sum += filter.FillRatio()
+	}
+	return sum / float64(len(s.filters))
+}
+
+// Test will test for membership of the data and returns true if it is a
+// member, false if not. This is a probabilistic test, meaning there is a
+// non-zero probability of false positives but a zero probability of false
+// negatives.
+func (s *ScalableBloomFilter) Test(data []byte) bool {
+	// Querying is made by testing for the presence in each filter.
+	for _, bf := range s.filters {
+		if bf.Test(data) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// Add will add the data to the Bloom filter. It returns the filter to allow
+// for chaining.
+func (s *ScalableBloomFilter) Add(data []byte) Filter {
+	idx := len(s.filters) - 1
+
+	// If the last filter has reached its fill ratio, add a new one.
+	if s.filters[idx].EstimatedFillRatio() >= s.p {
+		s.addFilter()
+		idx++
+	}
+
+	s.filters[idx].Add(data)
+	return s
+}
+
+// TestAndAdd is equivalent to calling Test followed by Add. It returns true if
+// the data is a member, false if not.
+func (s *ScalableBloomFilter) TestAndAdd(data []byte) bool {
+	member := s.Test(data)
+	s.Add(data)
+	return member
+}
+
+// Reset restores the Bloom filter to its original state. It returns the filter
+// to allow for chaining.
+func (s *ScalableBloomFilter) Reset() *ScalableBloomFilter {
+	s.filters = make([]*PartitionedBloomFilter, 0, 1)
+	s.addFilter()
+	return s
+}
+
+// addFilter adds a new Bloom filter with a restricted false-positive rate to
+// the Scalable Bloom Filter
+func (s *ScalableBloomFilter) addFilter() {
+	fpRate := s.fp * math.Pow(s.r, float64(len(s.filters)))
+	p := NewPartitionedBloomFilter(s.hint, fpRate)
+	if len(s.filters) > 0 {
+		p.SetHash(s.filters[0].hash)
+	}
+	s.filters = append(s.filters, p)
+}
+
+// SetHash sets the hashing function used in the filter.
+// For the effect on false positive rates see: https://github.com/tylertreat/BoomFilters/pull/1
+func (s *ScalableBloomFilter) SetHash(h hash.Hash64) {
+	for _, bf := range s.filters {
+		bf.SetHash(h)
+	}
+}
+
+// WriteTo writes a binary representation of the ScalableBloomFilter to an i/o stream.
+// It returns the number of bytes written.
+func (s *ScalableBloomFilter) WriteTo(stream io.Writer) (int64, error) {
+	err := binary.Write(stream, binary.BigEndian, s.r)
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Write(stream, binary.BigEndian, s.fp)
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Write(stream, binary.BigEndian, s.p)
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Write(stream, binary.BigEndian, uint64(s.hint))
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Write(stream, binary.BigEndian, uint64(len(s.filters)))
+	if err != nil {
+		return 0, err
+	}
+	var numBytes int64
+	for _, filter := range s.filters {
+		num, err := filter.WriteTo(stream)
+		if err != nil {
+			return 0, err
+		}
+		numBytes += num
+	}
+	return numBytes + int64(5*binary.Size(uint64(0))), err
+}
+
+// ReadFrom reads a binary representation of ScalableBloomFilter (such as might
+// have been written by WriteTo()) from an i/o stream. It returns the number
+// of bytes read.
+func (s *ScalableBloomFilter) ReadFrom(stream io.Reader) (int64, error) {
+	var r, fp, p float64
+	var hint, len uint64
+	err := binary.Read(stream, binary.BigEndian, &r)
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Read(stream, binary.BigEndian, &fp)
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Read(stream, binary.BigEndian, &p)
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Read(stream, binary.BigEndian, &hint)
+	if err != nil {
+		return 0, err
+	}
+	err = binary.Read(stream, binary.BigEndian, &len)
+	if err != nil {
+		return 0, err
+	}
+	var numBytes int64
+	filters := make([]*PartitionedBloomFilter, len)
+	for i, _ := range filters {
+		filter := NewPartitionedBloomFilter(0, fp)
+		num, err := filter.ReadFrom(stream)
+		if err != nil {
+			return 0, err
+		}
+		numBytes += num
+		filters[i] = filter
+	}
+	s.r = r
+	s.fp = fp
+	s.p = p
+	s.hint = uint(hint)
+	s.filters = filters
+	return numBytes + int64(5*binary.Size(uint64(0))), nil
+}
+
+// GobEncode implements gob.GobEncoder interface.
+func (s *ScalableBloomFilter) GobEncode() ([]byte, error) {
+	var buf bytes.Buffer
+	_, err := s.WriteTo(&buf)
+	if err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}
+
+// GobDecode implements gob.GobDecoder interface.
+func (s *ScalableBloomFilter) GobDecode(data []byte) error {
+	buf := bytes.NewBuffer(data)
+	_, err := s.ReadFrom(buf)
+
+	return err
+}

vendor/github.com/tylertreat/BoomFilters/stable.go

@@ -0,0 +1,227 @@
+package boom
+
+import (
+	"hash"
+	"hash/fnv"
+	"math"
+	"math/rand"
+)
+
+// StableBloomFilter implements a Stable Bloom Filter as described by Deng and
+// Rafiei in Approximately Detecting Duplicates for Streaming Data using Stable
+// Bloom Filters:
+//
+// http://webdocs.cs.ualberta.ca/~drafiei/papers/DupDet06Sigmod.pdf
+//
+// A Stable Bloom Filter (SBF) continuously evicts stale information so that it
+// has room for more recent elements. Like traditional Bloom filters, an SBF
+// has a non-zero probability of false positives, which is controlled by
+// several parameters. Unlike the classic Bloom filter, an SBF has a tight
+// upper bound on the rate of false positives while introducing a non-zero rate
+// of false negatives. The false-positive rate of a classic Bloom filter
+// eventually reaches 1, after which all queries result in a false positive.
+// The stable-point property of an SBF means the false-positive rate
+// asymptotically approaches a configurable fixed constant. A classic Bloom
+// filter is actually a special case of SBF where the eviction rate is zero, so
+// this package provides support for them as well.
+//
+// Stable Bloom Filters are useful for cases where the size of the data set
+// isn't known a priori, which is a requirement for traditional Bloom filters,
+// and memory is bounded.  For example, an SBF can be used to deduplicate
+// events from an unbounded event stream with a specified upper bound on false
+// positives and minimal false negatives.
+type StableBloomFilter struct {
+	cells       *Buckets    // filter data
+	hash        hash.Hash64 // hash function (kernel for all k functions)
+	m           uint        // number of cells
+	p           uint        // number of cells to decrement
+	k           uint        // number of hash functions
+	max         uint8       // cell max value
+	indexBuffer []uint      // buffer used to cache indices
+}
+
+// NewStableBloomFilter creates a new Stable Bloom Filter with m cells and d
+// bits allocated per cell optimized for the target false-positive rate. Use
+// NewDefaultStableFilter if you don't want to calculate d.
+func NewStableBloomFilter(m uint, d uint8, fpRate float64) *StableBloomFilter {
+	k := OptimalK(fpRate) / 2
+	if k > m {
+		k = m
+	} else if k <= 0 {
+		k = 1
+	}
+
+	cells := NewBuckets(m, d)
+
+	return &StableBloomFilter{
+		hash:        fnv.New64(),
+		m:           m,
+		k:           k,
+		p:           optimalStableP(m, k, d, fpRate),
+		max:         cells.MaxBucketValue(),
+		cells:       cells,
+		indexBuffer: make([]uint, k),
+	}
+}
+
+// NewDefaultStableBloomFilter creates a new Stable Bloom Filter with m 1-bit
+// cells and which is optimized for cases where there is no prior knowledge of
+// the input data stream while maintaining an upper bound using the provided
+// rate of false positives.
+func NewDefaultStableBloomFilter(m uint, fpRate float64) *StableBloomFilter {
+	return NewStableBloomFilter(m, 1, fpRate)
+}
+
+// NewUnstableBloomFilter creates a new special case of Stable Bloom Filter
+// which is a traditional Bloom filter with m bits and an optimal number of
+// hash functions for the target false-positive rate. Unlike the stable
+// variant, data is not evicted and a cell contains a maximum of 1 hash value.
+func NewUnstableBloomFilter(m uint, fpRate float64) *StableBloomFilter {
+	var (
+		cells = NewBuckets(m, 1)
+		k     = OptimalK(fpRate)
+	)
+
+	return &StableBloomFilter{
+		hash:        fnv.New64(),
+		m:           m,
+		k:           k,
+		p:           0,
+		max:         cells.MaxBucketValue(),
+		cells:       cells,
+		indexBuffer: make([]uint, k),
+	}
+}
+
+// Cells returns the number of cells in the Stable Bloom Filter.
+func (s *StableBloomFilter) Cells() uint {
+	return s.m
+}
+
+// K returns the number of hash functions.
+func (s *StableBloomFilter) K() uint {
+	return s.k
+}
+
+// P returns the number of cells decremented on every add.
+func (s *StableBloomFilter) P() uint {
+	return s.p
+}
+
+// StablePoint returns the limit of the expected fraction of zeros in the
+// Stable Bloom Filter when the number of iterations goes to infinity. When
+// this limit is reached, the Stable Bloom Filter is considered stable.
+func (s *StableBloomFilter) StablePoint() float64 {
+	var (
+		subDenom = float64(s.p) * (1/float64(s.k) - 1/float64(s.m))
+		denom    = 1 + 1/subDenom
+		base     = 1 / denom
+	)
+
+	return math.Pow(base, float64(s.max))
+}
+
+// FalsePositiveRate returns the upper bound on false positives when the filter
+// has become stable.
+func (s *StableBloomFilter) FalsePositiveRate() float64 {
+	return math.Pow(1-s.StablePoint(), float64(s.k))
+}
+
+// Test will test for membership of the data and returns true if it is a
+// member, false if not. This is a probabilistic test, meaning there is a
+// non-zero probability of false positives and false negatives.
+func (s *StableBloomFilter) Test(data []byte) bool {
+	lower, upper := hashKernel(data, s.hash)
+
+	// If any of the K cells are 0, then it's not a member.
+	for i := uint(0); i < s.k; i++ {
+		if s.cells.Get((uint(lower)+uint(upper)*i)%s.m) == 0 {
+			return false
+		}
+	}
+
+	return true
+}
+
+// Add will add the data to the Stable Bloom Filter. It returns the filter to
+// allow for chaining.
+func (s *StableBloomFilter) Add(data []byte) Filter {
+	// Randomly decrement p cells to make room for new elements.
+	s.decrement()
+
+	lower, upper := hashKernel(data, s.hash)
+
+	// Set the K cells to max.
+	for i := uint(0); i < s.k; i++ {
+		s.cells.Set((uint(lower)+uint(upper)*i)%s.m, s.max)
+	}
+
+	return s
+}
+
+// TestAndAdd is equivalent to calling Test followed by Add. It returns true if
+// the data is a member, false if not.
+func (s *StableBloomFilter) TestAndAdd(data []byte) bool {
+	lower, upper := hashKernel(data, s.hash)
+	member := true
+
+	// If any of the K cells are 0, then it's not a member.
+	for i := uint(0); i < s.k; i++ {
+		s.indexBuffer[i] = (uint(lower) + uint(upper)*i) % s.m
+		if s.cells.Get(s.indexBuffer[i]) == 0 {
+			member = false
+		}
+	}
+
+	// Randomly decrement p cells to make room for new elements.
+	s.decrement()
+
+	// Set the K cells to max.
+	for _, idx := range s.indexBuffer {
+		s.cells.Set(idx, s.max)
+	}
+
+	return member
+}
+
+// Reset restores the Stable Bloom Filter to its original state. It returns the
+// filter to allow for chaining.
+func (s *StableBloomFilter) Reset() *StableBloomFilter {
+	s.cells.Reset()
+	return s
+}
+
+// decrement will decrement a random cell and (p-1) adjacent cells by 1. This
+// is faster than generating p random numbers. Although the processes of
+// picking the p cells are not independent, each cell has a probability of p/m
+// for being picked at each iteration, which means the properties still hold.
+func (s *StableBloomFilter) decrement() {
+	r := rand.Intn(int(s.m))
+	for i := uint(0); i < s.p; i++ {
+		idx := (r + int(i)) % int(s.m)
+		s.cells.Increment(uint(idx), -1)
+	}
+}
+
+// SetHash sets the hashing function used in the filter.
+// For the effect on false positive rates see: https://github.com/tylertreat/BoomFilters/pull/1
+func (s *StableBloomFilter) SetHash(h hash.Hash64) {
+	s.hash = h
+}
+
+// optimalStableP returns the optimal number of cells to decrement, p, per
+// iteration for the provided parameters of an SBF.
+func optimalStableP(m, k uint, d uint8, fpRate float64) uint {
+	var (
+		max      = math.Pow(2, float64(d)) - 1
+		subDenom = math.Pow(1-math.Pow(fpRate, 1/float64(k)), 1/max)
+		denom    = (1/subDenom - 1) * (1/float64(k) - 1/float64(m))
+	)
+
+	p := uint(1 / denom)
+	if p <= 0 {
+		p = 1
+	}
+
+	return p
+}

vendor/github.com/tylertreat/BoomFilters/topk.go

@@ -0,0 +1,125 @@
+package boom
+
+import (
+	"bytes"
+	"container/heap"
+)
+
+type Element struct {
+	Data []byte
+	Freq uint64
+}
+
+// An elementHeap is a min-heap of elements.
+type elementHeap []*Element
+
+func (e elementHeap) Len() int           { return len(e) }
+func (e elementHeap) Less(i, j int) bool { return e[i].Freq < e[j].Freq }
+func (e elementHeap) Swap(i, j int)      { e[i], e[j] = e[j], e[i] }
+
+func (e *elementHeap) Push(x interface{}) {
+	*e = append(*e, x.(*Element))
+}
+
+func (e *elementHeap) Pop() interface{} {
+	old := *e
+	n := len(old)
+	x := old[n-1]
+	*e = old[0 : n-1]
+	return x
+}
+
+// TopK uses a Count-Min Sketch to calculate the top-K frequent elements in a
+// stream.
+type TopK struct {
+	cms      *CountMinSketch
+	k        uint
+	n        uint
+	elements *elementHeap
+}
+
+// NewTopK creates a new TopK backed by a Count-Min sketch whose relative
+// accuracy is within a factor of epsilon with probability delta. It tracks the
+// k-most frequent elements.
+func NewTopK(epsilon, delta float64, k uint) *TopK {
+	elements := make(elementHeap, 0, k)
+	heap.Init(&elements)
+	return &TopK{
+		cms:      NewCountMinSketch(epsilon, delta),
+		k:        k,
+		elements: &elements,
+	}
+}
+
+// Add will add the data to the Count-Min Sketch and update the top-k heap if
+// applicable. Returns the TopK to allow for chaining.
+func (t *TopK) Add(data []byte) *TopK {
+	t.cms.Add(data)
+	t.n++
+
+	freq := t.cms.Count(data)
+	if t.isTop(freq) {
+		t.insert(data, freq)
+	}
+
+	return t
+}
+
+// Elements returns the top-k elements from lowest to highest frequency.
+func (t *TopK) Elements() []*Element {
+	if t.elements.Len() == 0 {
+		return make([]*Element, 0)
+	}
+
+	elements := make(elementHeap, t.elements.Len())
+	copy(elements, *t.elements)
+	heap.Init(&elements)
+	topK := make([]*Element, 0, t.k)
+
+	for elements.Len() > 0 {
+		topK = append(topK, heap.Pop(&elements).(*Element))
+	}
+
+	return topK
+}
+
+// Reset restores the TopK to its original state. It returns itself to allow
+// for chaining.
+func (t *TopK) Reset() *TopK {
+	t.cms.Reset()
+	elements := make(elementHeap, 0, t.k)
+	heap.Init(&elements)
+	t.elements = &elements
+	t.n = 0
+	return t
+}
+
+// isTop indicates if the given frequency falls within the top-k heap.
+func (t *TopK) isTop(freq uint64) bool {
+	if t.elements.Len() < int(t.k) {
+		return true
+	}
+
+	return freq >= (*t.elements)[0].Freq
+}
+
+// insert adds the data to the top-k heap. If the data is already an element,
+// the frequency is updated. If the heap already has k elements, the element
+// with the minimum frequency is removed.
+func (t *TopK) insert(data []byte, freq uint64) {
+	for _, element := range *t.elements {
+		if bytes.Compare(data, element.Data) == 0 {
+			// Element already in top-k.
+			element.Freq = freq
+			return
+		}
+	}
+
+	if t.elements.Len() == int(t.k) {
+		// Remove minimum-frequency element.
+		heap.Pop(t.elements)
+	}
+
+	// Add element to top-k.
+	heap.Push(t.elements, &Element{Data: data, Freq: freq})
+}